@gscdump/cli 0.7.6 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,27 +1,32 @@
1
1
  #!/usr/bin/env node
2
+ import { t as __exportAll } from "./_chunks/rolldown-runtime.mjs";
3
+ import { t as ofetch } from "./_chunks/libs/ofetch.mjs";
4
+ import { a as loadConfig, c as setConfigDir, i as getConfigPath, n as defaultDataDir, o as resolveDataDir, r as getConfigDir, s as saveConfig } from "./_chunks/config.mjs";
5
+ import os from "node:os";
6
+ import path from "node:path";
2
7
  import process from "node:process";
3
8
  import { defineCommand, runMain } from "citty";
4
9
  import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
5
10
  import { AnalyzerCapabilityError, analyzeFromSource, createEngineQuerySource } from "@gscdump/analysis";
6
11
  import { createGscApiQuerySource } from "@gscdump/engine-gsc-api";
7
- import { cancel, isCancel, multiselect, select, text } from "@clack/prompts";
8
- import { daysAgo, fetchSitemap, formatErrorForCli, getDateRange, googleSearchConsole, progressBar } from "gscdump";
12
+ import { cancel, confirm, isCancel, multiselect, select, text } from "@clack/prompts";
13
+ import { addSite, batchInspectUrls, batchRequestIndexing, createAuth, daysAgo, deleteSite, fetchSitemap, fetchSitesWithSitemaps, formatErrorForCli, getDateRange, getIndexingMetadata, getVerificationToken, googleSearchConsole, progressBar, requestIndexing, siteUrlToVerificationSite, verificationMethodsFor, verifySite } from "gscdump";
9
14
  import fs, { readFile, rm } from "node:fs/promises";
10
15
  import { createServer } from "node:http";
11
- import path from "node:path";
12
- import { OAuth2Client } from "google-auth-library";
13
- import os from "node:os";
14
- import { consola } from "consola";
15
- import { createNodeHarness } from "@gscdump/engine/node-harness";
16
+ import { JWT, OAuth2Client } from "google-auth-library";
17
+ import { Buffer } from "node:buffer";
18
+ import fs$1 from "node:fs";
19
+ import { createConsola } from "consola";
20
+ import { createNodeHarness } from "@gscdump/engine-duckdb-node";
16
21
  import { TABLE_DIMS, transformGscRow } from "@gscdump/engine/ingest";
17
22
  import { allTables, inferTable } from "@gscdump/engine/schema";
18
- import { Buffer } from "node:buffer";
23
+ import { DuckDBInstance } from "@duckdb/node-api";
24
+ import { sqlEscape } from "@gscdump/engine/sql";
19
25
  import { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore } from "@gscdump/engine/entities";
20
26
  import { createGscMcpServer } from "@gscdump/mcp/server";
21
27
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
22
- import { SearchTypes, between, country, date, device, gsc, page, query, searchAppearance } from "gscdump/query";
23
- import { DuckDBInstance } from "@duckdb/node-api";
24
- import { sqlEscape } from "@gscdump/engine/sql";
28
+ import { SearchTypes, and, between, contains, country, date, device, eq, gsc, notRegex, page, query, regex, searchAppearance } from "gscdump/query";
29
+ import { inferLegacyTier } from "@gscdump/engine";
25
30
  import { DEFAULT_ROLLUPS, rebuildRollups } from "@gscdump/analysis/rollups";
26
31
  import { filesystemStats } from "@gscdump/engine/filesystem";
27
32
  var LocalStoreUnsupportedError = class extends Error {
@@ -63,42 +68,81 @@ async function runLiveAnalysis(client, siteUrl, params) {
63
68
  throw e;
64
69
  });
65
70
  }
66
- let configDir = path.join(os.homedir(), ".config", "gscdump");
67
- function getConfigDir() {
68
- return configDir;
71
+ const ENV_LINE_RE$1 = /^([^=]+)=(.*)$/;
72
+ function parseEnvFile(envPath) {
73
+ let content;
74
+ try {
75
+ content = fs$1.readFileSync(envPath, "utf-8");
76
+ } catch {
77
+ return null;
78
+ }
79
+ const env = {};
80
+ for (const line of content.split("\n")) {
81
+ const trimmed = line.trim();
82
+ if (!trimmed || trimmed.startsWith("#")) continue;
83
+ const match = trimmed.match(ENV_LINE_RE$1);
84
+ if (!match) continue;
85
+ const key = match[1].trim();
86
+ let value = match[2].trim();
87
+ if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
88
+ env[key] = value;
89
+ }
90
+ return env;
69
91
  }
70
- function defaultDataDir() {
71
- return path.join(os.homedir(), ".gscdump", "data");
92
+ const appliedEnvKeys = /* @__PURE__ */ new Set();
93
+ let loadedEnvPath = null;
94
+ function getAppliedEnvKeys() {
95
+ return appliedEnvKeys;
72
96
  }
73
- function resolveDataDir(config) {
74
- return expandTilde(config.dataDir ?? defaultDataDir());
97
+ function getLoadedEnvPath() {
98
+ return loadedEnvPath;
75
99
  }
76
- function expandTilde(p) {
77
- if (p === "~") return os.homedir();
78
- if (p.startsWith("~/")) return path.join(os.homedir(), p.slice(2));
79
- return p;
100
+ function loadEnvFromCwd() {
101
+ const envPath = path.join(process.cwd(), ".env");
102
+ const parsed = parseEnvFile(envPath);
103
+ if (!parsed) return [];
104
+ loadedEnvPath = envPath;
105
+ const applied = [];
106
+ for (const [key, value] of Object.entries(parsed)) if (process.env[key] === void 0) {
107
+ process.env[key] = value;
108
+ applied.push(key);
109
+ appliedEnvKeys.add(key);
110
+ }
111
+ return applied;
80
112
  }
81
- async function loadConfig() {
82
- return fs.readFile(path.join(configDir, "config.json"), "utf-8").then((data) => JSON.parse(data)).catch(() => ({}));
113
+ const VERSION = "0.8.1";
114
+ const baseLogger = createConsola({
115
+ stdout: process.stderr,
116
+ stderr: process.stderr
117
+ });
118
+ const logger = baseLogger.withTag("gscdump");
119
+ function setQuiet(quiet) {
120
+ if (quiet) baseLogger.level = 1;
83
121
  }
84
- async function saveConfig(config) {
85
- await fs.mkdir(configDir, {
86
- recursive: true,
87
- mode: 448
122
+ let colorEnabled = (() => {
123
+ if (process.env.NO_COLOR) return false;
124
+ if (process.argv.includes("--no-color")) return false;
125
+ return Boolean(process.stderr.isTTY) || Boolean(process.env.FORCE_COLOR);
126
+ })();
127
+ const ANSI_RE = /\x1B\[[0-9;]*m/g;
128
+ let stdoutWrapped = false;
129
+ function wrapStdoutForNoColor() {
130
+ if (stdoutWrapped) return;
131
+ stdoutWrapped = true;
132
+ const original = process.stdout.write.bind(process.stdout);
133
+ process.stdout.write = ((chunk, ...rest) => {
134
+ if (typeof chunk === "string") chunk = chunk.replace(ANSI_RE, "");
135
+ else if (chunk instanceof Uint8Array) chunk = Buffer.from(chunk).toString("utf8").replace(ANSI_RE, "");
136
+ return original(chunk, ...rest);
88
137
  });
89
- await fs.writeFile(path.join(configDir, "config.json"), JSON.stringify(config, null, 2), { mode: 384 });
90
- }
91
- function getConfigPath() {
92
- return path.join(configDir, "config.json");
93
138
  }
94
- const VERSION = "1.0.0";
95
- const logger = consola.withTag("gscdump");
96
- function gscErrorHandler(error) {
97
- console.error();
98
- console.error(formatErrorForCli(error));
99
- console.error();
100
- process.exit(1);
139
+ function setNoColor(disable) {
140
+ if (disable) {
141
+ colorEnabled = false;
142
+ wrapStdoutForNoColor();
143
+ }
101
144
  }
145
+ if (!colorEnabled) wrapStdoutForNoColor();
102
146
  const gradientColors = [
103
147
  (s) => `\x1B[38;2;52;211;153m${s}\x1B[0m`,
104
148
  (s) => `\x1B[38;2;45;212;191m${s}\x1B[0m`,
@@ -120,6 +164,15 @@ function showSplash() {
120
164
  function clearLine() {
121
165
  process.stdout.write("\r\x1B[K");
122
166
  }
167
+ function displayPath(absPath) {
168
+ const cwd = process.cwd();
169
+ const home = os.homedir();
170
+ if (absPath === cwd) return ".";
171
+ if (absPath.startsWith(`${cwd}/`)) return absPath.slice(cwd.length + 1);
172
+ if (absPath === home) return "~";
173
+ if (absPath.startsWith(`${home}/`)) return `~/${absPath.slice(home.length + 1)}`;
174
+ return absPath;
175
+ }
123
176
  function formatAge(ms) {
124
177
  const delta = Date.now() - ms;
125
178
  if (delta < 6e4) return "just now";
@@ -146,6 +199,16 @@ function toCSV(data, columns) {
146
199
  return str.includes(",") || str.includes("\"") || str.includes("\n") ? `"${str.replace(/"/g, "\"\"")}"` : str;
147
200
  }).join(","))].join("\n");
148
201
  }
202
+ async function readUrlList$1(args) {
203
+ if (args.file) return (await fs.readFile(String(args.file), "utf-8")).split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
204
+ if (args.urls) return (Array.isArray(args.urls) ? args.urls : [args.urls]).map(String).filter(Boolean);
205
+ if (!process.stdin.isTTY) {
206
+ const chunks = [];
207
+ for await (const chunk of process.stdin) chunks.push(chunk);
208
+ return Buffer.concat(chunks).toString("utf-8").split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
209
+ }
210
+ return [];
211
+ }
149
212
  function exportToCSV(output) {
150
213
  const sections = [];
151
214
  if (output.pages?.data) sections.push(`# Pages\n${toCSV(output.pages.data, [
@@ -178,6 +241,81 @@ function exportToCSV(output) {
178
241
  ])}`);
179
242
  return sections.join("\n\n");
180
243
  }
244
+ const SCOPES = [
245
+ "https://www.googleapis.com/auth/webmasters",
246
+ "https://www.googleapis.com/auth/indexing",
247
+ "https://www.googleapis.com/auth/siteverification"
248
+ ];
249
+ async function loadServiceAccount(jsonPath) {
250
+ const raw = await fs.readFile(jsonPath, "utf-8");
251
+ const key = JSON.parse(raw);
252
+ if (key.type !== "service_account") throw new Error(`${jsonPath} is not a service-account key (type=${key.type})`);
253
+ return new JWT({
254
+ email: key.client_email,
255
+ key: key.private_key,
256
+ scopes: SCOPES
257
+ });
258
+ }
259
+ async function resolveServiceAccount(opts = {}) {
260
+ const p = opts.path || process.env.GSC_SERVICE_ACCOUNT_JSON || process.env.GOOGLE_APPLICATION_CREDENTIALS;
261
+ if (!p) return null;
262
+ return loadServiceAccount(p);
263
+ }
264
+ async function authenticateDeviceCode(credentials) {
265
+ const init = await ofetch("https://oauth2.googleapis.com/device/code", {
266
+ method: "POST",
267
+ body: new URLSearchParams({
268
+ client_id: credentials.clientId,
269
+ scope: SCOPES.join(" ")
270
+ })
271
+ }).catch((e) => {
272
+ throw new Error(`Device-code request failed: ${e.message}`);
273
+ });
274
+ console.log();
275
+ console.log(` \x1B[1mDevice-code OAuth\x1B[0m`);
276
+ console.log(` 1. On any device, open: \x1B[36m${init.verification_url}\x1B[0m`);
277
+ console.log(` 2. Enter this code: \x1B[1m${init.user_code}\x1B[0m`);
278
+ console.log(` 3. Approve the requested scopes`);
279
+ console.log();
280
+ logger.info(`Polling for completion (expires in ${Math.floor(init.expires_in / 60)}m)...`);
281
+ const intervalMs = init.interval * 1e3;
282
+ const deadline = Date.now() + init.expires_in * 1e3;
283
+ while (Date.now() < deadline) {
284
+ await new Promise((r) => setTimeout(r, intervalMs));
285
+ const res = await ofetch("https://oauth2.googleapis.com/token", {
286
+ method: "POST",
287
+ body: new URLSearchParams({
288
+ client_id: credentials.clientId,
289
+ client_secret: credentials.clientSecret,
290
+ device_code: init.device_code,
291
+ grant_type: "urn:ietf:params:oauth:grant-type:device_code"
292
+ })
293
+ }).catch((e) => e?.data ?? { error: "request_failed" });
294
+ if (res.access_token) return {
295
+ access_token: res.access_token,
296
+ refresh_token: res.refresh_token,
297
+ expiry_date: res.expires_in ? Date.now() + res.expires_in * 1e3 : void 0
298
+ };
299
+ if (res.error === "authorization_pending" || res.error === "slow_down") continue;
300
+ if (res.error === "access_denied") throw new Error("User denied authorization.");
301
+ if (res.error === "expired_token") throw new Error("Device code expired. Re-run `gscdump auth login --no-browser`.");
302
+ if (res.error) throw new Error(`Device-code poll failed: ${res.error_description || res.error}`);
303
+ }
304
+ throw new Error("Device-code flow timed out.");
305
+ }
306
+ function resolveBYOK(opts = {}) {
307
+ const accessToken = opts.accessToken || process.env.GSC_ACCESS_TOKEN || process.env.GOOGLE_ACCESS_TOKEN;
308
+ const clientId = opts.clientId || process.env.GSC_CLIENT_ID || process.env.GOOGLE_CLIENT_ID;
309
+ const clientSecret = opts.clientSecret || process.env.GSC_CLIENT_SECRET || process.env.GOOGLE_CLIENT_SECRET;
310
+ const refreshToken = opts.refreshToken || process.env.GSC_REFRESH_TOKEN || process.env.GOOGLE_REFRESH_TOKEN;
311
+ if (clientId && clientSecret && refreshToken) return createAuth({
312
+ clientId,
313
+ clientSecret,
314
+ refreshToken
315
+ });
316
+ if (accessToken) return accessToken;
317
+ return null;
318
+ }
181
319
  const REDIRECT_URI_RE = /redirect_uri=[^&]+/;
182
320
  function getTokensPath() {
183
321
  return path.join(getConfigDir(), "tokens.json");
@@ -200,17 +338,22 @@ async function getAuthCredentials(interactive) {
200
338
  const envClientId = process.env.GOOGLE_CLIENT_ID;
201
339
  const envClientSecret = process.env.GOOGLE_CLIENT_SECRET;
202
340
  if (envClientId && envClientSecret) {
203
- logger.success("Using OAuth2 credentials from environment");
341
+ logger.info("Using OAuth client from env");
342
+ console.log(` \x1B[90m${envClientId}\x1B[0m`);
204
343
  return {
205
344
  clientId: envClientId,
206
345
  clientSecret: envClientSecret
207
346
  };
208
347
  }
209
348
  const config = await loadConfig();
210
- if (config.clientId && config.clientSecret) return {
211
- clientId: config.clientId,
212
- clientSecret: config.clientSecret
213
- };
349
+ if (config.clientId && config.clientSecret) {
350
+ logger.info(`Using OAuth client from ${displayPath(`${getConfigDir()}/config.json`)}`);
351
+ console.log(` \x1B[90m${config.clientId}\x1B[0m`);
352
+ return {
353
+ clientId: config.clientId,
354
+ clientSecret: config.clientSecret
355
+ };
356
+ }
214
357
  if (!interactive) {
215
358
  logger.error("GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET required for non-interactive mode");
216
359
  process.exit(1);
@@ -246,25 +389,31 @@ async function getAuthCredentials(interactive) {
246
389
  async function getAuthCodeViaLoopback(authUrl) {
247
390
  return new Promise((resolve, reject) => {
248
391
  let resolvedRedirectUri = "";
249
- const server = createServer((req, res) => {
392
+ let timeoutId;
393
+ let server;
394
+ const settle = (fn) => {
395
+ if (timeoutId) clearTimeout(timeoutId);
396
+ server.closeAllConnections?.();
397
+ server.close();
398
+ fn();
399
+ };
400
+ server = createServer((req, res) => {
250
401
  const url = new URL(req.url || "", `http://127.0.0.1`);
251
402
  const code = url.searchParams.get("code");
252
403
  const error = url.searchParams.get("error");
253
404
  if (error) {
254
405
  res.writeHead(400, { "Content-Type": "text/html" });
255
406
  res.end(`<html><body><h1>Authorization Failed</h1><p>${error}</p><p>You can close this window.</p></body></html>`);
256
- server.close();
257
- reject(/* @__PURE__ */ new Error(`OAuth error: ${error}`));
407
+ settle(() => reject(/* @__PURE__ */ new Error(`OAuth error: ${error}`)));
258
408
  return;
259
409
  }
260
410
  if (code) {
261
411
  res.writeHead(200, { "Content-Type": "text/html" });
262
412
  res.end(`<html><body><h1>Authorization Successful</h1><p>You can close this window and return to the terminal.</p></body></html>`);
263
- server.close();
264
- resolve({
413
+ settle(() => resolve({
265
414
  code,
266
415
  redirectUri: resolvedRedirectUri
267
- });
416
+ }));
268
417
  return;
269
418
  }
270
419
  res.writeHead(400, { "Content-Type": "text/html" });
@@ -273,7 +422,7 @@ async function getAuthCodeViaLoopback(authUrl) {
273
422
  server.listen(0, "127.0.0.1", () => {
274
423
  const addr = server.address();
275
424
  if (!addr || typeof addr === "string") {
276
- reject(/* @__PURE__ */ new Error("Failed to start local server"));
425
+ settle(() => reject(/* @__PURE__ */ new Error("Failed to start local server")));
277
426
  return;
278
427
  }
279
428
  resolvedRedirectUri = `http://127.0.0.1:${addr.port}`;
@@ -287,17 +436,16 @@ async function getAuthCodeViaLoopback(authUrl) {
287
436
  logger.warn("Could not open browser automatically");
288
437
  });
289
438
  });
290
- server.on("error", reject);
291
- setTimeout(() => {
292
- server.close();
293
- reject(/* @__PURE__ */ new Error("Authorization timed out"));
439
+ server.on("error", (err) => settle(() => reject(err)));
440
+ timeoutId = setTimeout(() => {
441
+ settle(() => reject(/* @__PURE__ */ new Error("Authorization timed out")));
294
442
  }, 300 * 1e3);
295
443
  });
296
444
  }
297
- async function authenticate(credentials, interactive) {
445
+ async function authenticate(credentials, interactive, opts = {}) {
298
446
  const oauth2Client = new OAuth2Client(credentials.clientId, credentials.clientSecret, "http://127.0.0.1");
299
- const envAccessToken = process.env.GOOGLE_ACCESS_TOKEN;
300
- const envRefreshToken = process.env.GOOGLE_REFRESH_TOKEN;
447
+ const envAccessToken = !opts.force ? process.env.GOOGLE_ACCESS_TOKEN : void 0;
448
+ const envRefreshToken = !opts.force ? process.env.GOOGLE_REFRESH_TOKEN : void 0;
301
449
  if (envAccessToken || envRefreshToken) {
302
450
  oauth2Client.setCredentials({
303
451
  access_token: envAccessToken,
@@ -309,7 +457,7 @@ async function authenticate(credentials, interactive) {
309
457
  }
310
458
  return oauth2Client;
311
459
  }
312
- const existingTokens = await loadTokens();
460
+ const existingTokens = !opts.force ? await loadTokens() : null;
313
461
  if (existingTokens) {
314
462
  oauth2Client.setCredentials(existingTokens);
315
463
  if (existingTokens.expiry_date && existingTokens.expiry_date < Date.now()) {
@@ -329,9 +477,16 @@ async function authenticate(credentials, interactive) {
329
477
  logger.error("No saved tokens. Run interactively first to authenticate.");
330
478
  process.exit(1);
331
479
  }
480
+ if (opts.noBrowser) {
481
+ const tokens = await authenticateDeviceCode(credentials);
482
+ oauth2Client.setCredentials(tokens);
483
+ await saveTokens(tokens);
484
+ logger.success(`Tokens saved to ${displayPath(getTokensPath())}`);
485
+ return oauth2Client;
486
+ }
332
487
  const authUrl = oauth2Client.generateAuthUrl({
333
488
  access_type: "offline",
334
- scope: ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/indexing"],
489
+ scope: SCOPES,
335
490
  prompt: "consent"
336
491
  });
337
492
  logger.info("Waiting for authorization...");
@@ -339,24 +494,163 @@ async function authenticate(credentials, interactive) {
339
494
  const { tokens } = await new OAuth2Client(credentials.clientId, credentials.clientSecret, redirectUri).getToken(code);
340
495
  oauth2Client.setCredentials(tokens);
341
496
  await saveTokens(tokens);
342
- logger.success(`Tokens saved to ${getTokensPath()}`);
497
+ logger.success(`Tokens saved to ${displayPath(getTokensPath())}`);
343
498
  return oauth2Client;
344
499
  }
345
500
  async function getAuth(opts = {}) {
346
- const { interactive = true } = opts;
347
- return authenticate(await getAuthCredentials(interactive), interactive);
501
+ const { interactive = true, noBrowser = false, force = false } = opts;
502
+ return authenticate(await getAuthCredentials(interactive), interactive, {
503
+ noBrowser,
504
+ force
505
+ });
506
+ }
507
+ async function resolveAuth(opts = {}) {
508
+ const sa = await resolveServiceAccount({ path: opts.serviceAccount });
509
+ if (sa) {
510
+ logger.success("Using service-account credentials");
511
+ return sa;
512
+ }
513
+ const byok = resolveBYOK(opts.byok);
514
+ if (byok) {
515
+ if (typeof byok !== "string") logger.success("Using BYOK credentials");
516
+ return byok;
517
+ }
518
+ return getAuth(opts);
519
+ }
520
+ function envSourceLabel(envVar) {
521
+ return getAppliedEnvKeys().has(envVar) ? `.env (${envVar})` : `shell env (${envVar})`;
522
+ }
523
+ function pickEnvSource(...envVars) {
524
+ for (const v of envVars) {
525
+ const value = process.env[v];
526
+ if (value) return {
527
+ envVar: v,
528
+ value
529
+ };
530
+ }
531
+ return null;
532
+ }
533
+ function redactCred(v, keepTail = 6) {
534
+ if (!v) return "<unset>";
535
+ if (v.length <= keepTail) return "***";
536
+ return `***${v.slice(-keepTail)}`;
537
+ }
538
+ async function describeAuthProvenance() {
539
+ const rows = [];
540
+ const warnings = [];
541
+ const saPath = process.env.GSC_SERVICE_ACCOUNT_JSON || process.env.GOOGLE_APPLICATION_CREDENTIALS;
542
+ if (saPath) {
543
+ const saEnv = process.env.GSC_SERVICE_ACCOUNT_JSON ? "GSC_SERVICE_ACCOUNT_JSON" : "GOOGLE_APPLICATION_CREDENTIALS";
544
+ rows.push({
545
+ field: "service_account",
546
+ source: envSourceLabel(saEnv),
547
+ value: displayPath(saPath)
548
+ });
549
+ }
550
+ const clientId = pickEnvSource("GSC_CLIENT_ID", "GOOGLE_CLIENT_ID");
551
+ const clientSecret = pickEnvSource("GSC_CLIENT_SECRET", "GOOGLE_CLIENT_SECRET");
552
+ const config = await loadConfig().catch(() => null);
553
+ if (clientId) rows.push({
554
+ field: "client_id",
555
+ source: envSourceLabel(clientId.envVar),
556
+ value: clientId.value
557
+ });
558
+ else if (config?.clientId) rows.push({
559
+ field: "client_id",
560
+ source: `${displayPath(`${getConfigDir()}/config.json`)}`,
561
+ value: config.clientId
562
+ });
563
+ else rows.push({
564
+ field: "client_id",
565
+ source: "<none>",
566
+ value: null
567
+ });
568
+ if (clientSecret) rows.push({
569
+ field: "client_secret",
570
+ source: envSourceLabel(clientSecret.envVar),
571
+ value: redactCred(clientSecret.value)
572
+ });
573
+ else if (config?.clientSecret) rows.push({
574
+ field: "client_secret",
575
+ source: `${displayPath(`${getConfigDir()}/config.json`)}`,
576
+ value: redactCred(config.clientSecret)
577
+ });
578
+ else rows.push({
579
+ field: "client_secret",
580
+ source: "<none>",
581
+ value: null
582
+ });
583
+ const accessTok = pickEnvSource("GSC_ACCESS_TOKEN", "GOOGLE_ACCESS_TOKEN");
584
+ const refreshTok = pickEnvSource("GSC_REFRESH_TOKEN", "GOOGLE_REFRESH_TOKEN");
585
+ if (accessTok) rows.push({
586
+ field: "access_token",
587
+ source: envSourceLabel(accessTok.envVar),
588
+ value: redactCred(accessTok.value)
589
+ });
590
+ if (refreshTok) rows.push({
591
+ field: "refresh_token",
592
+ source: envSourceLabel(refreshTok.envVar),
593
+ value: redactCred(refreshTok.value)
594
+ });
595
+ const tokens = await loadTokens();
596
+ if (tokens) {
597
+ const expiry = tokens.expiry_date ? new Date(tokens.expiry_date).toISOString() : "no expiry";
598
+ rows.push({
599
+ field: "saved_tokens",
600
+ source: displayPath(getTokensPath()),
601
+ value: `access=${tokens.access_token ? "present" : "missing"}, refresh=${tokens.refresh_token ? "present" : "missing"}, expiry=${expiry}`
602
+ });
603
+ }
604
+ let effective;
605
+ if (saPath) effective = "service-account";
606
+ else if (clientId && clientSecret && refreshTok) effective = "byok-refresh-token";
607
+ else if (accessTok) effective = "byok-access-token";
608
+ else if (tokens) effective = "saved-tokens";
609
+ else effective = "none";
610
+ if ((effective === "byok-refresh-token" || effective === "byok-access-token") && tokens) warnings.push("BYOK env vars shadow saved tokens. If you just ran `auth login --force`, the env tokens are stale; unset them or update them.");
611
+ if (effective === "byok-refresh-token" && clientId && refreshTok) {
612
+ if (getAppliedEnvKeys().has(clientId.envVar) !== getAppliedEnvKeys().has(refreshTok.envVar)) warnings.push(`client_id and refresh_token come from different sources (${envSourceLabel(clientId.envVar)} vs ${envSourceLabel(refreshTok.envVar)}); they may not match.`);
613
+ }
614
+ const envFile = getLoadedEnvPath();
615
+ if (envFile && getAppliedEnvKeys().size > 0) warnings.push(`Loaded ${getAppliedEnvKeys().size} var(s) from ${displayPath(envFile)}.`);
616
+ return {
617
+ rows,
618
+ effective,
619
+ warnings
620
+ };
621
+ }
622
+ async function formatAuthProvenance() {
623
+ const { rows, effective, warnings } = await describeAuthProvenance();
624
+ const lines = [];
625
+ lines.push(` \x1B[1mAuth config sources\x1B[0m \x1B[90m(effective: ${effective})\x1B[0m`);
626
+ for (const r of rows) {
627
+ const val = r.value ? ` \x1B[90m${r.value}\x1B[0m` : "";
628
+ lines.push(` ${r.field.padEnd(16)} \x1B[36m${r.source}\x1B[0m${val}`);
629
+ }
630
+ if (warnings.length > 0) {
631
+ lines.push("");
632
+ for (const w of warnings) lines.push(` \x1B[33m!\x1B[0m ${w}`);
633
+ }
634
+ return lines.join("\n");
635
+ }
636
+ function isAuthError(err) {
637
+ const msg = (err instanceof Error ? err.message : String(err ?? "")).toLowerCase();
638
+ if (!msg) return false;
639
+ return /\b(?:401|403|unauthorized|forbidden|invalid_grant|invalid_token|insufficient.*scope|invalid_client|token has been expired|token has been revoked)\b/.test(msg) || msg.includes("oauth2.googleapis.com/token");
348
640
  }
349
641
  function createLocalStore(opts) {
350
642
  return createNodeHarness(opts);
351
643
  }
644
+ var context_exports = /* @__PURE__ */ __exportAll({ createCommandContext: () => createCommandContext });
352
645
  async function createCommandContext(opts = {}) {
353
- const { needsAuth = false, needsStore = false, interactive = false } = opts;
646
+ const { needsAuth = false, needsStore = false, interactive = false, byok, fetchOptions } = opts;
354
647
  const config = await loadConfig();
355
- const auth = needsAuth ? await getAuth({
648
+ const auth = needsAuth ? await resolveAuth({
356
649
  interactive,
357
- config
650
+ config,
651
+ byok
358
652
  }) : null;
359
- const client = auth ? googleSearchConsole(auth) : null;
653
+ const client = auth ? googleSearchConsole(auth, { fetchOptions }) : null;
360
654
  const store = needsStore ? createLocalStore({ dataDir: resolveDataDir(config) }) : null;
361
655
  const loadSites = async () => {
362
656
  if (!client) throw new Error("loadSites requires needsAuth: true");
@@ -402,6 +696,16 @@ async function createCommandContext(opts = {}) {
402
696
  resolveSite
403
697
  };
404
698
  }
699
+ async function gscErrorHandler(error) {
700
+ console.error();
701
+ console.error(formatErrorForCli(error));
702
+ if (isAuthError(error)) {
703
+ console.error();
704
+ console.error(await formatAuthProvenance());
705
+ }
706
+ console.error();
707
+ process.exit(1);
708
+ }
405
709
  const ANALYSIS_TOOLS = defaultAnalyzerRegistry.listAnalyzerIds();
406
710
  const TOOL_EXTRA_ARGS = {
407
711
  brand: { "brand-terms": {
@@ -551,10 +855,7 @@ function makeToolCommand(tool) {
551
855
  renderResults(localResult.results, localResult.results.length, format);
552
856
  return;
553
857
  }
554
- const result = await runLiveAnalysis(ctx.client, siteUrl, params).catch((e) => {
555
- logger.error(`Analysis failed: ${e.message}`);
556
- process.exit(1);
557
- });
858
+ const result = await runLiveAnalysis(ctx.client, siteUrl, params).catch(gscErrorHandler);
558
859
  if (format === "json") {
559
860
  console.log(JSON.stringify(result, null, 2));
560
861
  return;
@@ -737,35 +1038,203 @@ const analyzeCommand = defineCommand({
737
1038
  },
738
1039
  subCommands: Object.fromEntries(ANALYSIS_TOOLS.map((tool) => [tool, makeToolCommand(tool)]))
739
1040
  });
1041
+ async function fetchTokenInfo(accessToken) {
1042
+ return ofetch("https://oauth2.googleapis.com/tokeninfo", { query: { access_token: accessToken } }).catch(() => null);
1043
+ }
1044
+ const statusCommand$1 = defineCommand({
1045
+ meta: {
1046
+ name: "status",
1047
+ description: "Show current authentication status"
1048
+ },
1049
+ args: {
1050
+ json: {
1051
+ type: "boolean",
1052
+ default: false,
1053
+ description: "Output as JSON"
1054
+ },
1055
+ quiet: {
1056
+ type: "boolean",
1057
+ alias: "q",
1058
+ default: false,
1059
+ description: "Suppress info/success output"
1060
+ }
1061
+ },
1062
+ async run({ args }) {
1063
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
1064
+ const tokens = await loadTokens();
1065
+ const byok = resolveBYOK();
1066
+ const byokKind = byok ? typeof byok === "string" ? "access-token" : "refresh-token" : null;
1067
+ let liveToken = null;
1068
+ if (typeof byok === "string") liveToken = byok;
1069
+ else if (byok && "getAccessToken" in byok) liveToken = await byok.getAccessToken().then((r) => r.token ?? null).catch(() => null);
1070
+ else if (tokens?.access_token) liveToken = tokens.access_token;
1071
+ const tokenInfo = liveToken ? await fetchTokenInfo(liveToken) : null;
1072
+ const scopes = tokenInfo?.scope ? tokenInfo.scope.split(/\s+/).filter(Boolean) : [];
1073
+ if (args.json) {
1074
+ console.log(JSON.stringify({
1075
+ authenticated: !!tokens || !!byok,
1076
+ source: byok ? "byok" : tokens ? "saved-tokens" : null,
1077
+ byokKind,
1078
+ scopes,
1079
+ tokenAccount: tokenInfo?.email ?? null,
1080
+ tokens: tokens ? {
1081
+ hasAccessToken: !!tokens.access_token,
1082
+ hasRefreshToken: !!tokens.refresh_token,
1083
+ expiry: tokens.expiry_date ? new Date(tokens.expiry_date).toISOString() : null,
1084
+ expired: tokens.expiry_date ? tokens.expiry_date < Date.now() : null
1085
+ } : null
1086
+ }, null, 2));
1087
+ return;
1088
+ }
1089
+ const reportScopes = () => {
1090
+ if (scopes.length === 0) return;
1091
+ console.log(` Scopes:`);
1092
+ const required = [
1093
+ "https://www.googleapis.com/auth/webmasters",
1094
+ "https://www.googleapis.com/auth/indexing",
1095
+ "https://www.googleapis.com/auth/siteverification"
1096
+ ];
1097
+ const has = (s) => scopes.includes(s) || scopes.includes(s.replace(".readonly", ""));
1098
+ for (const s of scopes) console.log(` \x1B[90m└─\x1B[0m ${s}`);
1099
+ const missing = required.filter((s) => !has(s));
1100
+ if (missing.length > 0) {
1101
+ console.log(` \x1B[33mMissing scopes:\x1B[0m`);
1102
+ for (const s of missing) console.log(` \x1B[90m└─\x1B[0m ${s}`);
1103
+ console.log(` \x1B[90mRun \`gscdump auth login --force\` to re-consent.\x1B[0m`);
1104
+ }
1105
+ };
1106
+ if (byok) {
1107
+ logger.success(`Authenticated via BYOK (${byokKind})`);
1108
+ if (tokenInfo?.email) console.log(` Account: ${tokenInfo.email}`);
1109
+ reportScopes();
1110
+ return;
1111
+ }
1112
+ if (!tokens) {
1113
+ logger.warn("Not authenticated");
1114
+ logger.info("Run `gscdump init` (full setup) or `gscdump auth login` (OAuth only)");
1115
+ logger.info("Or set GSC_ACCESS_TOKEN / GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN env vars");
1116
+ return;
1117
+ }
1118
+ const hasAccess = !!tokens.access_token;
1119
+ const hasRefresh = !!tokens.refresh_token;
1120
+ const expiry = tokens.expiry_date ? new Date(tokens.expiry_date) : null;
1121
+ const isExpired = expiry && expiry < /* @__PURE__ */ new Date();
1122
+ logger.success("Authenticated (saved tokens)");
1123
+ console.log();
1124
+ console.log(` Access token: ${hasAccess ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
1125
+ console.log(` Refresh token: ${hasRefresh ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
1126
+ if (expiry) {
1127
+ const status = isExpired ? "\x1B[33mexpired\x1B[0m" : "\x1B[32mvalid\x1B[0m";
1128
+ console.log(` Expires: ${expiry.toISOString()} (${status})`);
1129
+ }
1130
+ if (tokenInfo?.email) console.log(` Account: ${tokenInfo.email}`);
1131
+ reportScopes();
1132
+ }
1133
+ });
1134
+ const refreshCommand = defineCommand({
1135
+ meta: {
1136
+ name: "refresh",
1137
+ description: "Force-refresh saved OAuth tokens (no-op for BYOK)"
1138
+ },
1139
+ args: { quiet: {
1140
+ type: "boolean",
1141
+ alias: "q",
1142
+ default: false,
1143
+ description: "Suppress info/success output"
1144
+ } },
1145
+ async run({ args }) {
1146
+ setQuiet(Boolean(args.quiet));
1147
+ if (resolveBYOK()) {
1148
+ logger.info("BYOK detected; refresh handled per-call by the SDK");
1149
+ return;
1150
+ }
1151
+ const tokens = await loadTokens();
1152
+ if (!tokens?.refresh_token) {
1153
+ logger.error("No saved refresh token. Run `gscdump auth login`.");
1154
+ process.exit(1);
1155
+ }
1156
+ const credentials = await getAuthCredentials(false).catch((e) => {
1157
+ logger.error(`Cannot resolve credentials: ${e.message}`);
1158
+ process.exit(1);
1159
+ });
1160
+ await saveTokens({
1161
+ ...tokens,
1162
+ expiry_date: 1
1163
+ });
1164
+ if ((await authenticate(credentials, false).catch((e) => {
1165
+ logger.error(`Refresh failed: ${e.message}`);
1166
+ process.exit(1);
1167
+ })).credentials?.access_token) logger.success("Token refreshed");
1168
+ else logger.warn("Refresh completed but no new access token returned");
1169
+ }
1170
+ });
740
1171
  const authCommand = defineCommand({
741
1172
  meta: {
742
1173
  name: "auth",
743
1174
  description: "Manage authentication"
744
1175
  },
745
1176
  subCommands: {
746
- status: defineCommand({
1177
+ status: statusCommand$1,
1178
+ login: defineCommand({
747
1179
  meta: {
748
- name: "status",
749
- description: "Show current authentication status"
1180
+ name: "login",
1181
+ description: "Run OAuth flow and persist tokens (skip if BYOK env vars set)"
750
1182
  },
751
- async run() {
752
- const tokens = await loadTokens();
753
- if (!tokens) {
754
- logger.warn("Not authenticated");
755
- logger.info("Run gscdump init to authenticate");
1183
+ args: {
1184
+ "force": {
1185
+ type: "boolean",
1186
+ alias: "f",
1187
+ default: false,
1188
+ description: "Re-run OAuth even if tokens already exist"
1189
+ },
1190
+ "browser": {
1191
+ type: "boolean",
1192
+ default: true,
1193
+ description: "Use loopback browser flow. Pass --no-browser for device-code (headless)."
1194
+ },
1195
+ "service-account": {
1196
+ type: "string",
1197
+ description: "Path to a service-account JSON key (skips OAuth)"
1198
+ },
1199
+ "quiet": {
1200
+ type: "boolean",
1201
+ alias: "q",
1202
+ default: false,
1203
+ description: "Suppress info/success output"
1204
+ }
1205
+ },
1206
+ async run({ args }) {
1207
+ setQuiet(Boolean(args.quiet));
1208
+ if (resolveBYOK() && !args.force) {
1209
+ logger.info("BYOK env vars detected, no login needed (--force to override)");
756
1210
  return;
757
1211
  }
758
- const hasAccess = !!tokens.access_token;
759
- const hasRefresh = !!tokens.refresh_token;
760
- const expiry = tokens.expiry_date ? new Date(tokens.expiry_date) : null;
761
- const isExpired = expiry && expiry < /* @__PURE__ */ new Date();
762
- logger.success("Authenticated");
763
- console.log();
764
- console.log(` Access token: ${hasAccess ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
765
- console.log(` Refresh token: ${hasRefresh ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
766
- if (expiry) {
767
- const status = isExpired ? "\x1B[33mexpired\x1B[0m" : "\x1B[32mvalid\x1B[0m";
768
- console.log(` Expires: ${expiry.toISOString()} (${status})`);
1212
+ if (args["service-account"]) {
1213
+ const jwt = await loadServiceAccount(String(args["service-account"])).catch((e) => {
1214
+ logger.error(`Service-account load failed: ${e.message}`);
1215
+ process.exit(1);
1216
+ });
1217
+ await jwt.authorize().catch((e) => {
1218
+ logger.error(`Service-account auth failed: ${e.message}`);
1219
+ process.exit(1);
1220
+ });
1221
+ logger.success(`Service-account verified: ${jwt.email ?? "OK"}`);
1222
+ logger.info(`Set GOOGLE_APPLICATION_CREDENTIALS=${args["service-account"]} to use it across sessions.`);
1223
+ return;
1224
+ }
1225
+ if (args.force) await clearTokens();
1226
+ await getAuth({
1227
+ interactive: true,
1228
+ noBrowser: args.browser === false,
1229
+ force: Boolean(args.force)
1230
+ }).catch((e) => {
1231
+ logger.error(`Login failed: ${e.message}`);
1232
+ process.exit(1);
1233
+ });
1234
+ logger.success("Logged in");
1235
+ if (resolveBYOK()) {
1236
+ console.log();
1237
+ console.log(await formatAuthProvenance());
769
1238
  }
770
1239
  }
771
1240
  }),
@@ -774,35 +1243,76 @@ const authCommand = defineCommand({
774
1243
  name: "logout",
775
1244
  description: "Clear stored OAuth tokens"
776
1245
  },
777
- async run() {
1246
+ args: { quiet: {
1247
+ type: "boolean",
1248
+ alias: "q",
1249
+ default: false,
1250
+ description: "Suppress info/success output"
1251
+ } },
1252
+ async run({ args }) {
1253
+ setQuiet(Boolean(args.quiet));
778
1254
  await clearTokens();
779
1255
  }
780
- })
1256
+ }),
1257
+ refresh: refreshCommand
1258
+ }
1259
+ });
1260
+ const showCommand = defineCommand({
1261
+ meta: {
1262
+ name: "show",
1263
+ description: "Show current config"
1264
+ },
1265
+ args: {
1266
+ json: {
1267
+ type: "boolean",
1268
+ default: false,
1269
+ description: "Output config as a single JSON object (suppresses path header)"
1270
+ },
1271
+ quiet: {
1272
+ type: "boolean",
1273
+ alias: "q",
1274
+ default: false,
1275
+ description: "Suppress info/success output"
1276
+ }
1277
+ },
1278
+ async run({ args }) {
1279
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
1280
+ const config = await loadConfig();
1281
+ const configPath = getConfigPath();
1282
+ if (args.json) {
1283
+ console.log(JSON.stringify({
1284
+ path: configPath,
1285
+ config
1286
+ }, null, 2));
1287
+ return;
1288
+ }
1289
+ logger.info(`Config: ${displayPath(configPath)}`);
1290
+ console.log();
1291
+ if (Object.keys(config).length === 0) {
1292
+ logger.warn("No config set");
1293
+ return;
1294
+ }
1295
+ console.log(JSON.stringify(config, null, 2));
781
1296
  }
782
1297
  });
1298
+ const VALID_KEYS = [
1299
+ "defaultSite",
1300
+ "defaultPeriod",
1301
+ "defaultFormat",
1302
+ "defaultDb",
1303
+ "dataDir",
1304
+ "defaultLimit",
1305
+ "defaultSearchType",
1306
+ "defaultDataState"
1307
+ ];
1308
+ const NUMERIC_KEYS = new Set(["defaultLimit"]);
783
1309
  const configCommand = defineCommand({
784
1310
  meta: {
785
1311
  name: "config",
786
1312
  description: "Manage configuration"
787
1313
  },
788
1314
  subCommands: {
789
- show: defineCommand({
790
- meta: {
791
- name: "show",
792
- description: "Show current config"
793
- },
794
- async run() {
795
- const config = await loadConfig();
796
- const configPath = getConfigPath();
797
- logger.info(`Config: ${configPath}`);
798
- console.log();
799
- if (Object.keys(config).length === 0) {
800
- logger.warn("No config set");
801
- return;
802
- }
803
- console.log(JSON.stringify(config, null, 2));
804
- }
805
- }),
1315
+ show: showCommand,
806
1316
  set: defineCommand({
807
1317
  meta: {
808
1318
  name: "set",
@@ -811,31 +1321,37 @@ const configCommand = defineCommand({
811
1321
  args: {
812
1322
  key: {
813
1323
  type: "positional",
814
- description: "Config key (defaultSite, defaultPeriod, defaultFormat, defaultDb)",
1324
+ description: `Config key (${VALID_KEYS.join(", ")})`,
815
1325
  required: true
816
1326
  },
817
1327
  value: {
818
1328
  type: "positional",
819
1329
  description: "Value to set",
820
1330
  required: true
1331
+ },
1332
+ quiet: {
1333
+ type: "boolean",
1334
+ alias: "q",
1335
+ default: false,
1336
+ description: "Suppress info/success output"
821
1337
  }
822
1338
  },
823
1339
  async run({ args }) {
824
- const validKeys = [
825
- "defaultSite",
826
- "defaultPeriod",
827
- "defaultFormat",
828
- "defaultDb"
829
- ];
830
- if (!validKeys.includes(args.key)) {
1340
+ setQuiet(Boolean(args.quiet));
1341
+ if (!VALID_KEYS.includes(args.key)) {
831
1342
  logger.error(`Invalid key: ${args.key}`);
832
- logger.info(`Valid keys: ${validKeys.join(", ")}`);
1343
+ logger.info(`Valid keys: ${VALID_KEYS.join(", ")}`);
833
1344
  process.exit(1);
834
1345
  }
835
1346
  const config = await loadConfig();
836
- config[args.key] = args.value;
1347
+ const value = NUMERIC_KEYS.has(args.key) ? Number(args.value) : args.value;
1348
+ if (NUMERIC_KEYS.has(args.key) && !Number.isFinite(value)) {
1349
+ logger.error(`Invalid numeric value for ${args.key}: ${args.value}`);
1350
+ process.exit(1);
1351
+ }
1352
+ config[args.key] = value;
837
1353
  await saveConfig(config);
838
- logger.success(`Set ${args.key} = ${args.value}`);
1354
+ logger.success(`Set ${args.key} = ${value}`);
839
1355
  }
840
1356
  }),
841
1357
  unset: defineCommand({
@@ -843,12 +1359,26 @@ const configCommand = defineCommand({
843
1359
  name: "unset",
844
1360
  description: "Remove a config value"
845
1361
  },
846
- args: { key: {
847
- type: "positional",
848
- description: "Config key to remove",
849
- required: true
850
- } },
1362
+ args: {
1363
+ key: {
1364
+ type: "positional",
1365
+ description: `Config key to remove (${VALID_KEYS.join(", ")})`,
1366
+ required: true
1367
+ },
1368
+ quiet: {
1369
+ type: "boolean",
1370
+ alias: "q",
1371
+ default: false,
1372
+ description: "Suppress info/success output"
1373
+ }
1374
+ },
851
1375
  async run({ args }) {
1376
+ setQuiet(Boolean(args.quiet));
1377
+ if (!VALID_KEYS.includes(args.key)) {
1378
+ logger.error(`Invalid key: ${args.key}`);
1379
+ logger.info(`Valid keys: ${VALID_KEYS.join(", ")}`);
1380
+ process.exit(1);
1381
+ }
852
1382
  const config = await loadConfig();
853
1383
  delete config[args.key];
854
1384
  await saveConfig(config);
@@ -863,33 +1393,523 @@ const configCommand = defineCommand({
863
1393
  run() {
864
1394
  console.log(getConfigPath());
865
1395
  }
1396
+ }),
1397
+ validate: defineCommand({
1398
+ meta: {
1399
+ name: "validate",
1400
+ description: "Validate the saved config (defaultSite is verified, dataDir exists/writable)"
1401
+ },
1402
+ args: {
1403
+ json: {
1404
+ type: "boolean",
1405
+ default: false,
1406
+ description: "Output as JSON"
1407
+ },
1408
+ quiet: {
1409
+ type: "boolean",
1410
+ alias: "q",
1411
+ default: false,
1412
+ description: "Suppress info/success output"
1413
+ }
1414
+ },
1415
+ async run({ args }) {
1416
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
1417
+ const { resolveDataDir } = await import("./_chunks/config.mjs").then((n) => n.t);
1418
+ const fs = await import("node:fs/promises");
1419
+ const config = await loadConfig();
1420
+ const issues = [];
1421
+ const dataDir = resolveDataDir(config);
1422
+ const dataDirDisplay = displayPath(dataDir);
1423
+ const stat = await fs.stat(dataDir).catch(() => null);
1424
+ if (stat && !stat.isDirectory()) issues.push({
1425
+ key: "dataDir",
1426
+ level: "fail",
1427
+ message: `${dataDirDisplay} is not a directory`
1428
+ });
1429
+ else if (stat) {
1430
+ const probe = `${dataDir}/.gscdump-config-probe`;
1431
+ if (!await fs.writeFile(probe, "").then(() => fs.rm(probe)).then(() => true).catch(() => false)) issues.push({
1432
+ key: "dataDir",
1433
+ level: "fail",
1434
+ message: `${dataDirDisplay} not writable`
1435
+ });
1436
+ } else issues.push({
1437
+ key: "dataDir",
1438
+ level: "warn",
1439
+ message: `${dataDirDisplay} does not exist (will be created on first sync)`
1440
+ });
1441
+ if (config.defaultSite) if (!!config.clientId && !!config.clientSecret) {
1442
+ const { createCommandContext } = await Promise.resolve().then(() => context_exports);
1443
+ const ctx = await createCommandContext({ needsAuth: true }).catch(() => null);
1444
+ if (ctx) {
1445
+ const sites = await ctx.loadSites().catch(() => null);
1446
+ if (sites && !sites.some((s) => s.siteUrl === config.defaultSite || s.siteUrl.includes(String(config.defaultSite)))) issues.push({
1447
+ key: "defaultSite",
1448
+ level: "fail",
1449
+ message: `${config.defaultSite} is not in the verified site list`
1450
+ });
1451
+ }
1452
+ } else issues.push({
1453
+ key: "defaultSite",
1454
+ level: "warn",
1455
+ message: "set, but auth not configured — skipping verification"
1456
+ });
1457
+ if (config.defaultFormat && !["json", "csv"].includes(config.defaultFormat)) issues.push({
1458
+ key: "defaultFormat",
1459
+ level: "fail",
1460
+ message: `unknown format: ${config.defaultFormat}`
1461
+ });
1462
+ if (args.json) {
1463
+ console.log(JSON.stringify({
1464
+ ok: !issues.some((i) => i.level === "fail"),
1465
+ issues
1466
+ }, null, 2));
1467
+ return;
1468
+ }
1469
+ if (issues.length === 0) {
1470
+ logger.success("Config OK");
1471
+ return;
1472
+ }
1473
+ for (const i of issues) {
1474
+ const prefix = i.level === "fail" ? "\x1B[31m✗\x1B[0m" : "\x1B[33m!\x1B[0m";
1475
+ console.log(` ${prefix} ${i.key}: ${i.message}`);
1476
+ }
1477
+ if (issues.some((i) => i.level === "fail")) process.exit(1);
1478
+ }
866
1479
  })
867
1480
  }
868
1481
  });
869
- const DEFAULT_OUT = "./gscdump-export";
870
- const dumpCommand = defineCommand({
871
- meta: {
1482
+ const REQUIRED_SCOPES = [
1483
+ "https://www.googleapis.com/auth/webmasters",
1484
+ "https://www.googleapis.com/auth/indexing",
1485
+ "https://www.googleapis.com/auth/siteverification"
1486
+ ];
1487
+ const FETCH_TIMEOUT_MS = 5e3;
1488
+ const TIME_SKEW_WARN_MS = 5 * 6e4;
1489
+ const WATERMARK_STALE_DAYS_WARN = 7;
1490
+ const RELEVANT_ENV_KEYS = [
1491
+ "GSC_ACCESS_TOKEN",
1492
+ "GSC_CLIENT_ID",
1493
+ "GSC_CLIENT_SECRET",
1494
+ "GSC_REFRESH_TOKEN",
1495
+ "GOOGLE_ACCESS_TOKEN",
1496
+ "GOOGLE_CLIENT_ID",
1497
+ "GOOGLE_CLIENT_SECRET",
1498
+ "GOOGLE_REFRESH_TOKEN",
1499
+ "GOOGLE_APPLICATION_CREDENTIALS",
1500
+ "GSC_SERVICE_ACCOUNT_JSON"
1501
+ ];
1502
+ function redact(v) {
1503
+ if (!v) return "<missing>";
1504
+ if (v.length <= 6) return "***";
1505
+ return `***${v.slice(-6)}`;
1506
+ }
1507
+ async function checkEnv() {
1508
+ const envPath = path.join(process.cwd(), ".env");
1509
+ const parsed = parseEnvFile(envPath);
1510
+ if (!parsed) return {
1511
+ checks: [{
1512
+ name: "env",
1513
+ status: "info",
1514
+ detail: "no .env (using shell env / saved tokens)"
1515
+ }],
1516
+ envKeys: /* @__PURE__ */ new Set()
1517
+ };
1518
+ const relevant = RELEVANT_ENV_KEYS.filter((k) => parsed[k] !== void 0);
1519
+ const envKeys = new Set(relevant);
1520
+ if (relevant.length === 0) return {
1521
+ checks: [{
1522
+ name: "env",
1523
+ status: "info",
1524
+ detail: `${displayPath(envPath)} found, no auth vars`
1525
+ }],
1526
+ envKeys
1527
+ };
1528
+ const inventory = relevant.map((k) => {
1529
+ const v = parsed[k];
1530
+ if (k.endsWith("CLIENT_ID")) return `${k}=${v}`;
1531
+ return `${k}=${redact(v)}`;
1532
+ });
1533
+ return {
1534
+ checks: [{
1535
+ name: "env",
1536
+ status: "info",
1537
+ detail: `${displayPath(envPath)} → ${inventory.join(", ")} (not validated — see auth)`
1538
+ }],
1539
+ envKeys
1540
+ };
1541
+ }
1542
+ function describeAuthSource(envKeys, byok) {
1543
+ if (!byok) return "saved tokens";
1544
+ const isAccessToken = typeof byok === "string";
1545
+ const driver = isAccessToken ? "GSC_ACCESS_TOKEN" : "GSC_REFRESH_TOKEN";
1546
+ const source = (isAccessToken ? ["GSC_ACCESS_TOKEN", "GOOGLE_ACCESS_TOKEN"] : ["GSC_REFRESH_TOKEN", "GOOGLE_REFRESH_TOKEN"]).some((k) => envKeys.has(k)) ? ".env" : "shell env";
1547
+ return `BYOK ${isAccessToken ? "(access-token)" : "(refresh-token)"} from ${source} via ${driver}`;
1548
+ }
1549
+ async function checkAuth$1(envKeys) {
1550
+ const checks = [];
1551
+ const byok = resolveBYOK();
1552
+ const tokens = await loadTokens();
1553
+ if (!byok && !tokens) {
1554
+ checks.push({
1555
+ name: "auth",
1556
+ status: "fail",
1557
+ detail: "no BYOK env vars and no saved tokens; run `gscdump init`"
1558
+ });
1559
+ return {
1560
+ checks,
1561
+ liveToken: null
1562
+ };
1563
+ }
1564
+ const clientId = process.env.GSC_CLIENT_ID ?? process.env.GOOGLE_CLIENT_ID ?? (await loadConfig()).clientId ?? null;
1565
+ if (clientId) checks.push({
1566
+ name: "auth.client_id",
1567
+ status: "info",
1568
+ detail: clientId
1569
+ });
1570
+ let liveToken = null;
1571
+ let refreshError = null;
1572
+ if (typeof byok === "string") liveToken = byok;
1573
+ else if (byok && "getAccessToken" in byok) liveToken = await byok.getAccessToken().then((r) => r.token ?? null).catch((e) => {
1574
+ refreshError = e?.data?.error_description ?? e?.data?.error ?? e?.message ?? String(e);
1575
+ return null;
1576
+ });
1577
+ else if (tokens?.access_token) liveToken = tokens.access_token;
1578
+ if (!liveToken) {
1579
+ const source = describeAuthSource(envKeys, byok);
1580
+ const detail = refreshError ? `${source} — refresh failed: ${refreshError} (token revoked / invalid — re-run \`gscdump auth login --force\` and update the source above)` : `${source} — no usable access token`;
1581
+ checks.push({
1582
+ name: "auth",
1583
+ status: "fail",
1584
+ detail
1585
+ });
1586
+ return {
1587
+ checks,
1588
+ liveToken: null
1589
+ };
1590
+ }
1591
+ const info = await ofetch("https://oauth2.googleapis.com/tokeninfo", { query: { access_token: liveToken } }).catch((e) => ({ error: e.message }));
1592
+ if ("error" in info) {
1593
+ checks.push({
1594
+ name: "auth",
1595
+ status: "fail",
1596
+ detail: `tokeninfo failed: ${info.error}`
1597
+ });
1598
+ return {
1599
+ checks,
1600
+ liveToken: null
1601
+ };
1602
+ }
1603
+ checks.push({
1604
+ name: "auth",
1605
+ status: "pass",
1606
+ detail: describeAuthSource(envKeys, byok)
1607
+ });
1608
+ if (info.email) checks.push({
1609
+ name: "auth.account",
1610
+ status: "pass",
1611
+ detail: info.email
1612
+ });
1613
+ const scopes = info.scope ? info.scope.split(/\s+/) : [];
1614
+ const missing = REQUIRED_SCOPES.filter((s) => !scopes.includes(s) && !scopes.includes(s.replace(".readonly", "")));
1615
+ if (missing.length > 0) checks.push({
1616
+ name: "auth.scopes",
1617
+ status: "warn",
1618
+ detail: `missing: ${missing.join(", ")} — \`gscdump auth login --force\` to re-consent`
1619
+ });
1620
+ else checks.push({
1621
+ name: "auth.scopes",
1622
+ status: "pass",
1623
+ detail: `${scopes.length} granted`
1624
+ });
1625
+ if (tokens?.expiry_date) {
1626
+ const expiresInMs = tokens.expiry_date - Date.now();
1627
+ if (expiresInMs < 0) checks.push({
1628
+ name: "auth.expiry",
1629
+ status: "warn",
1630
+ detail: `expired ${new Date(tokens.expiry_date).toISOString()} — will refresh on next call`
1631
+ });
1632
+ else checks.push({
1633
+ name: "auth.expiry",
1634
+ status: "pass",
1635
+ detail: `valid for ${Math.floor(expiresInMs / 6e4)}m`
1636
+ });
1637
+ }
1638
+ return {
1639
+ checks,
1640
+ liveToken
1641
+ };
1642
+ }
1643
+ async function checkTimeSkew() {
1644
+ const dateHeader = await ofetch.raw("https://oauth2.googleapis.com/tokeninfo", {
1645
+ method: "GET",
1646
+ timeout: FETCH_TIMEOUT_MS
1647
+ }).then((r) => r.headers.get("date")).catch((e) => e?.response?.headers?.get("date") ?? null);
1648
+ if (!dateHeader) return [{
1649
+ name: "time",
1650
+ status: "warn",
1651
+ detail: "could not probe Google clock (no Date header)"
1652
+ }];
1653
+ const remoteMs = Date.parse(dateHeader);
1654
+ if (!Number.isFinite(remoteMs)) return [{
1655
+ name: "time",
1656
+ status: "warn",
1657
+ detail: `unparseable Date header: ${dateHeader}`
1658
+ }];
1659
+ const skewMs = Date.now() - remoteMs;
1660
+ const human = `${skewMs >= 0 ? "+" : ""}${(skewMs / 1e3).toFixed(1)}s`;
1661
+ if (Math.abs(skewMs) > TIME_SKEW_WARN_MS) return [{
1662
+ name: "time",
1663
+ status: "warn",
1664
+ detail: `local clock ${human} off Google — OAuth refresh may reject; sync your clock`
1665
+ }];
1666
+ return [{
1667
+ name: "time",
1668
+ status: "pass",
1669
+ detail: `in sync (${human})`
1670
+ }];
1671
+ }
1672
+ async function checkDataDir() {
1673
+ const dataDir = resolveDataDir(await loadConfig());
1674
+ const display = displayPath(dataDir);
1675
+ const stat = await fs.stat(dataDir).catch(() => null);
1676
+ if (!stat) return [{
1677
+ name: "dataDir",
1678
+ status: "warn",
1679
+ detail: `${display} does not exist (will be created on first sync)`
1680
+ }];
1681
+ if (!stat.isDirectory()) return [{
1682
+ name: "dataDir",
1683
+ status: "fail",
1684
+ detail: `${display} is not a directory`
1685
+ }];
1686
+ const probe = `${dataDir}/.gscdump-doctor-probe`;
1687
+ return await fs.writeFile(probe, "").then(() => fs.rm(probe)).then(() => true).catch(() => false) ? [{
1688
+ name: "dataDir",
1689
+ status: "pass",
1690
+ detail: display
1691
+ }] : [{
1692
+ name: "dataDir",
1693
+ status: "fail",
1694
+ detail: `${display} not writable`
1695
+ }];
1696
+ }
1697
+ async function checkStoreWatermarks() {
1698
+ const dataDir = resolveDataDir(await loadConfig());
1699
+ if (!(await fs.stat(dataDir).catch(() => null))?.isDirectory()) return [{
1700
+ name: "store.watermarks",
1701
+ status: "pass",
1702
+ detail: "no store yet (run `gscdump sync`)"
1703
+ }];
1704
+ const store = createLocalStore({ dataDir });
1705
+ const watermarks = await store.engine.getWatermarks({ userId: store.userId }).catch(() => null);
1706
+ if (!watermarks || watermarks.length === 0) return [{
1707
+ name: "store.watermarks",
1708
+ status: "pass",
1709
+ detail: "no watermarks (run `gscdump sync`)"
1710
+ }];
1711
+ const bySite = /* @__PURE__ */ new Map();
1712
+ for (const w of watermarks) {
1713
+ const site = w.siteId ?? "(global)";
1714
+ const existing = bySite.get(site);
1715
+ if (!existing || w.newestDateSynced > existing) bySite.set(site, w.newestDateSynced);
1716
+ }
1717
+ const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
1718
+ const stale = [];
1719
+ let freshest = null;
1720
+ for (const [site, newest] of bySite) {
1721
+ const days = Math.floor((Date.parse(today) - Date.parse(newest)) / 864e5);
1722
+ if (days > WATERMARK_STALE_DAYS_WARN) stale.push({
1723
+ site,
1724
+ days
1725
+ });
1726
+ if (!freshest || days < freshest.days) freshest = {
1727
+ site,
1728
+ days
1729
+ };
1730
+ }
1731
+ if (stale.length > 0) {
1732
+ const sample = stale.slice(0, 3).map((s) => `${s.site} (${s.days}d)`).join(", ");
1733
+ const more = stale.length > 3 ? ` +${stale.length - 3} more` : "";
1734
+ return [{
1735
+ name: "store.watermarks",
1736
+ status: "warn",
1737
+ detail: `${stale.length}/${bySite.size} site(s) stale >${WATERMARK_STALE_DAYS_WARN}d: ${sample}${more}`
1738
+ }];
1739
+ }
1740
+ const tail = freshest ? `, freshest ${freshest.days}d ago` : "";
1741
+ return [{
1742
+ name: "store.watermarks",
1743
+ status: "pass",
1744
+ detail: `${bySite.size} site(s)${tail}`
1745
+ }];
1746
+ }
1747
+ async function checkApiReachable(name, url) {
1748
+ const reachable = await ofetch.raw(url, {
1749
+ method: "GET",
1750
+ timeout: FETCH_TIMEOUT_MS
1751
+ }).then(() => true).catch(() => false);
1752
+ return [{
1753
+ name,
1754
+ status: reachable ? "pass" : "warn",
1755
+ detail: reachable ? "reachable" : `${new URL(url).hostname} unreachable (network/firewall?)`
1756
+ }];
1757
+ }
1758
+ async function checkGscSites() {
1759
+ const config = await loadConfig();
1760
+ const auth = await resolveAuth({
1761
+ interactive: false,
1762
+ config
1763
+ }).catch(() => null);
1764
+ if (!auth) return [{
1765
+ name: "gsc.sites",
1766
+ status: "warn",
1767
+ detail: "skipped (no usable auth)"
1768
+ }];
1769
+ const sites = await googleSearchConsole(auth).sites().catch((e) => e);
1770
+ if (sites instanceof Error) return [{
1771
+ name: "gsc.sites",
1772
+ status: "fail",
1773
+ detail: `sites() failed: ${sites.message}`
1774
+ }];
1775
+ const checks = [];
1776
+ const verified = sites.filter((s) => s.permissionLevel !== "siteUnverifiedUser").length;
1777
+ checks.push({
1778
+ name: "gsc.sites",
1779
+ status: "pass",
1780
+ detail: `${sites.length} site(s) accessible (${verified} verified)`
1781
+ });
1782
+ if (config.defaultSite) {
1783
+ const match = sites.find((s) => s.siteUrl === config.defaultSite || (s.siteUrl ?? "").includes(String(config.defaultSite)));
1784
+ checks.push(match ? {
1785
+ name: "config.defaultSite",
1786
+ status: "pass",
1787
+ detail: `${config.defaultSite} ✓`
1788
+ } : {
1789
+ name: "config.defaultSite",
1790
+ status: "fail",
1791
+ detail: `${config.defaultSite} not in verified site list`
1792
+ });
1793
+ }
1794
+ return checks;
1795
+ }
1796
+ const doctorCommand = defineCommand({
1797
+ meta: {
1798
+ name: "doctor",
1799
+ description: "Run health checks (env, auth, scopes, time, dataDir, store, GSC reachability + ping, defaultSite)"
1800
+ },
1801
+ args: { json: {
1802
+ type: "boolean",
1803
+ default: false,
1804
+ description: "Output as JSON"
1805
+ } },
1806
+ async run({ args }) {
1807
+ const envResult = await checkEnv();
1808
+ const [authResult, timeChecks, dataDirChecks, watermarkChecks, gscApi, indexingApi, siteVerificationApi] = await Promise.all([
1809
+ checkAuth$1(envResult.envKeys),
1810
+ checkTimeSkew(),
1811
+ checkDataDir(),
1812
+ checkStoreWatermarks(),
1813
+ checkApiReachable("gsc.api", "https://searchconsole.googleapis.com/$discovery/rest?version=v1"),
1814
+ checkApiReachable("indexing.api", "https://indexing.googleapis.com/$discovery/rest?version=v3"),
1815
+ checkApiReachable("siteverification.api", "https://www.googleapis.com/discovery/v1/apis/siteVerification/v1/rest")
1816
+ ]);
1817
+ const sitesChecks = authResult.liveToken ? await checkGscSites() : [{
1818
+ name: "gsc.sites",
1819
+ status: "warn",
1820
+ detail: "skipped (auth failed)"
1821
+ }];
1822
+ const all = [
1823
+ ...envResult.checks,
1824
+ ...authResult.checks,
1825
+ ...timeChecks,
1826
+ ...dataDirChecks,
1827
+ ...watermarkChecks,
1828
+ ...gscApi,
1829
+ ...indexingApi,
1830
+ ...siteVerificationApi,
1831
+ ...sitesChecks
1832
+ ];
1833
+ if (args.json) {
1834
+ console.log(JSON.stringify({
1835
+ checks: all,
1836
+ ok: all.every((c) => c.status !== "fail")
1837
+ }, null, 2));
1838
+ return;
1839
+ }
1840
+ const ICONS = {
1841
+ pass: "\x1B[32m✓\x1B[0m",
1842
+ warn: "\x1B[33m!\x1B[0m",
1843
+ fail: "\x1B[31m✗\x1B[0m",
1844
+ info: "\x1B[34mℹ\x1B[0m"
1845
+ };
1846
+ console.log();
1847
+ for (const c of all) {
1848
+ const detail = c.detail ? ` \x1B[90m${c.detail}\x1B[0m` : "";
1849
+ console.log(` ${ICONS[c.status]} ${c.name}${detail}`);
1850
+ }
1851
+ console.log();
1852
+ const failed = all.filter((c) => c.status === "fail");
1853
+ if (failed.length > 0) {
1854
+ logger.error(`${failed.length} check(s) failed`);
1855
+ process.exit(1);
1856
+ }
1857
+ const warned = all.filter((c) => c.status === "warn");
1858
+ if (warned.length > 0) logger.warn(`${warned.length} warning(s)`);
1859
+ else logger.success("All checks passed");
1860
+ }
1861
+ });
1862
+ const DEFAULT_OUT = "./gscdump-export";
1863
+ const FORMATS = [
1864
+ "parquet",
1865
+ "json",
1866
+ "ndjson",
1867
+ "csv"
1868
+ ];
1869
+ const dumpCommand = defineCommand({
1870
+ meta: {
872
1871
  name: "dump",
873
1872
  description: "Export live Parquet files from the local store to a directory"
874
1873
  },
875
1874
  args: {
876
- site: {
1875
+ "site": {
877
1876
  type: "string",
878
1877
  alias: "s",
879
- description: "Site URL (e.g., sc-domain:example.com)"
1878
+ description: "Site URL (e.g., sc-domain:example.com); ignored with --all-sites"
880
1879
  },
881
- out: {
1880
+ "out": {
882
1881
  type: "string",
883
1882
  alias: "o",
884
1883
  default: DEFAULT_OUT,
885
1884
  description: `Output directory (default: ${DEFAULT_OUT})`
886
1885
  },
887
- compact: {
1886
+ "format": {
1887
+ type: "string",
1888
+ alias: "F",
1889
+ default: "parquet",
1890
+ description: `Output format: ${FORMATS.join(", ")} (default: parquet copies raw files)`
1891
+ },
1892
+ "tables": {
1893
+ type: "string",
1894
+ alias: "t",
1895
+ description: `Comma-separated table list (default: all). Known: ${allTables().join(", ")}`
1896
+ },
1897
+ "all-sites": {
1898
+ type: "boolean",
1899
+ default: false,
1900
+ description: "Iterate every site with local data"
1901
+ },
1902
+ "compact": {
888
1903
  type: "boolean",
889
1904
  default: false,
890
1905
  description: "Compact every closed month into a single file before exporting"
891
1906
  },
892
- quiet: {
1907
+ "json": {
1908
+ type: "boolean",
1909
+ default: false,
1910
+ description: "Emit a JSON summary of what was exported"
1911
+ },
1912
+ "quiet": {
893
1913
  type: "boolean",
894
1914
  alias: "q",
895
1915
  default: false,
@@ -897,31 +1917,76 @@ const dumpCommand = defineCommand({
897
1917
  }
898
1918
  },
899
1919
  async run({ args }) {
1920
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
1921
+ const format = String(args.format);
1922
+ if (!FORMATS.includes(format)) {
1923
+ logger.error(`Invalid --format: ${format}. Allowed: ${FORMATS.join(", ")}`);
1924
+ process.exit(1);
1925
+ }
1926
+ const tablesFilter = args.tables ? new Set(String(args.tables).split(",").map((t) => t.trim()).filter(Boolean)) : null;
900
1927
  const ctx = await createCommandContext({
901
- needsAuth: true,
1928
+ needsAuth: !args["all-sites"],
902
1929
  needsStore: true
903
1930
  });
904
- const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
905
1931
  const store = ctx.store;
906
1932
  const outDir = path.resolve(String(args.out));
907
- if (args.compact) await compactClosedMonths(store, siteUrl, args.quiet);
908
- const entries = await listLiveEntries(store, siteUrl);
909
- if (entries.length === 0) {
910
- logger.warn(`No data for ${siteUrl}. Run \`gscdump sync\` first.`);
1933
+ const targets = args["all-sites"] ? await listSitesWithData(store) : [await ctx.resolveSite(args.site ? String(args.site) : void 0)];
1934
+ if (targets.length === 0) {
1935
+ logger.warn("No sites with local data. Run `gscdump sync` first.");
911
1936
  process.exit(0);
912
1937
  }
913
- await fs.mkdir(outDir, { recursive: true });
914
- let copied = 0;
915
- for (const entry of entries) {
916
- const bytes = await store.engine.readObject(entry.objectKey);
917
- const target = path.join(outDir, entry.objectKey);
918
- await fs.mkdir(path.dirname(target), { recursive: true });
919
- await fs.writeFile(target, Buffer.from(bytes));
920
- copied++;
1938
+ if (args.compact) for (const siteUrl of targets) await compactClosedMonths(store, siteUrl, args.quiet);
1939
+ const summary = [];
1940
+ for (const siteUrl of targets) {
1941
+ const entries = (await listLiveEntries(store, siteUrl)).filter((e) => !tablesFilter || tablesFilter.has(e.table));
1942
+ if (entries.length === 0) {
1943
+ if (!args.json && !args.quiet) logger.warn(`No data for ${siteUrl}; skipping`);
1944
+ continue;
1945
+ }
1946
+ if (format === "parquet") {
1947
+ const written = await dumpParquet(store, entries, outDir);
1948
+ summary.push({
1949
+ site: siteUrl,
1950
+ files: written,
1951
+ rows: 0,
1952
+ format,
1953
+ outPath: outDir
1954
+ });
1955
+ } else {
1956
+ const written = await dumpRowFormat(store, entries, outDir, siteUrl, format);
1957
+ summary.push({
1958
+ site: siteUrl,
1959
+ files: written.files,
1960
+ rows: written.rows,
1961
+ format,
1962
+ outPath: outDir
1963
+ });
1964
+ }
1965
+ }
1966
+ if (args.json) {
1967
+ console.log(JSON.stringify({
1968
+ outDir,
1969
+ sites: summary
1970
+ }, null, 2));
1971
+ return;
1972
+ }
1973
+ for (const s of summary) {
1974
+ const rows = s.rows ? `, ${s.rows.toLocaleString()} rows` : "";
1975
+ logger.success(`[${s.site}] ${s.files} ${s.format} file(s)${rows} → ${displayPath(s.outPath)}`);
921
1976
  }
922
- if (!args.quiet) logger.success(`Exported ${copied} file(s) to ${outDir}`);
923
1977
  }
924
1978
  });
1979
+ async function listSitesWithData(store) {
1980
+ const siteIds = /* @__PURE__ */ new Set();
1981
+ for (const table of allTables()) {
1982
+ const entries = await store.engine.listLive({
1983
+ userId: store.userId,
1984
+ table
1985
+ });
1986
+ for (const e of entries) if (e.siteId) siteIds.add(e.siteId);
1987
+ }
1988
+ return Array.from(siteIds);
1989
+ }
925
1990
  async function listLiveEntries(store, siteUrl) {
926
1991
  const siteId = store.siteIdFor(siteUrl);
927
1992
  return (await Promise.all(allTables().map((table) => store.engine.listLive({
@@ -930,6 +1995,58 @@ async function listLiveEntries(store, siteUrl) {
930
1995
  table
931
1996
  })))).flat();
932
1997
  }
1998
+ async function dumpParquet(store, entries, outDir) {
1999
+ await fs.mkdir(outDir, { recursive: true });
2000
+ let copied = 0;
2001
+ for (const entry of entries) {
2002
+ const bytes = await store.engine.readObject(entry.objectKey);
2003
+ const target = path.join(outDir, entry.objectKey);
2004
+ await fs.mkdir(path.dirname(target), { recursive: true });
2005
+ await fs.writeFile(target, Buffer.from(bytes));
2006
+ copied++;
2007
+ }
2008
+ return copied;
2009
+ }
2010
+ async function dumpRowFormat(store, entries, outDir, siteUrl, format) {
2011
+ const byTable = /* @__PURE__ */ new Map();
2012
+ for (const e of entries) {
2013
+ const arr = byTable.get(e.table) ?? [];
2014
+ arr.push(e);
2015
+ byTable.set(e.table, arr);
2016
+ }
2017
+ const safeSite = siteUrl.replace(/[^a-z0-9]+/gi, "_");
2018
+ const siteDir = path.join(outDir, safeSite);
2019
+ await fs.mkdir(siteDir, { recursive: true });
2020
+ let files = 0;
2021
+ let totalRows = 0;
2022
+ for (const [table, tableEntries] of byTable) {
2023
+ const rows = await readTableRows(tableEntries.map((e) => path.join(store.dataDir, e.objectKey)));
2024
+ const ext = format === "csv" ? "csv" : format === "ndjson" ? "ndjson" : "json";
2025
+ const target = path.join(siteDir, `${table}.${ext}`);
2026
+ let body;
2027
+ if (format === "json") body = JSON.stringify(rows, null, 2);
2028
+ else if (format === "ndjson") body = rows.map((r) => JSON.stringify(r)).join("\n");
2029
+ else body = rows.length > 0 ? toCSV(rows, Object.keys(rows[0])) : "";
2030
+ await fs.writeFile(target, body);
2031
+ files++;
2032
+ totalRows += rows.length;
2033
+ }
2034
+ return {
2035
+ files,
2036
+ rows: totalRows
2037
+ };
2038
+ }
2039
+ async function readTableRows(filePaths) {
2040
+ const instance = await DuckDBInstance.create(":memory:");
2041
+ const conn = await instance.connect();
2042
+ try {
2043
+ const fileList = filePaths.map((p) => `'${sqlEscape(p)}'`).join(", ");
2044
+ return (await conn.runAndReadAll(`SELECT * FROM read_parquet([${fileList}], union_by_name=true)`)).getRowObjects();
2045
+ } finally {
2046
+ conn.closeSync();
2047
+ instance.closeSync();
2048
+ }
2049
+ }
933
2050
  async function compactClosedMonths(store, siteUrl, quiet) {
934
2051
  const siteId = store.siteIdFor(siteUrl);
935
2052
  for (const table of allTables()) {
@@ -958,8 +2075,7 @@ const inspectSubCommand = defineCommand({
958
2075
  site: {
959
2076
  type: "string",
960
2077
  alias: "s",
961
- required: true,
962
- description: "Site URL (e.g., sc-domain:example.com)"
2078
+ description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
963
2079
  },
964
2080
  file: {
965
2081
  type: "string",
@@ -976,6 +2092,11 @@ const inspectSubCommand = defineCommand({
976
2092
  default: "4",
977
2093
  description: "Concurrent in-flight inspect calls (default: 4)"
978
2094
  },
2095
+ json: {
2096
+ type: "boolean",
2097
+ default: false,
2098
+ description: "Emit a JSON summary of inspection results"
2099
+ },
979
2100
  quiet: {
980
2101
  type: "boolean",
981
2102
  alias: "q",
@@ -984,16 +2105,17 @@ const inspectSubCommand = defineCommand({
984
2105
  }
985
2106
  },
986
2107
  async run({ args }) {
2108
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
987
2109
  const ctx = await createCommandContext({
988
2110
  needsAuth: true,
989
2111
  needsStore: true
990
2112
  });
991
2113
  const client = ctx.client;
992
2114
  const store = ctx.store;
993
- const siteUrl = String(args.site);
2115
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
994
2116
  const limit = args.limit ? Number.parseInt(String(args.limit), 10) : INSPECTION_QPD_PER_PROPERTY;
995
2117
  const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
996
- const quiet = Boolean(args.quiet);
2118
+ const quiet = Boolean(args.quiet) || Boolean(args.json);
997
2119
  const urls = (await readUrlList({ file: args.file ? String(args.file) : void 0 })).slice(0, limit);
998
2120
  if (urls.length === 0) {
999
2121
  logger.warn("No URLs to inspect.");
@@ -1041,7 +2163,14 @@ const inspectSubCommand = defineCommand({
1041
2163
  userId: store.userId,
1042
2164
  siteId: store.siteIdFor(siteUrl)
1043
2165
  }, records);
1044
- if (!quiet) {
2166
+ if (args.json) console.log(JSON.stringify({
2167
+ site: siteUrl,
2168
+ inspected: records.length,
2169
+ failed,
2170
+ failures,
2171
+ records
2172
+ }, null, 2));
2173
+ else if (!quiet) {
1045
2174
  logger.success(`Inspected ${records.length}/${urls.length} URL(s)`);
1046
2175
  if (failed > 0) {
1047
2176
  logger.warn(`${failed} failed:`);
@@ -1061,8 +2190,7 @@ const showSubCommand = defineCommand({
1061
2190
  site: {
1062
2191
  type: "string",
1063
2192
  alias: "s",
1064
- required: true,
1065
- description: "Site URL"
2193
+ description: "Site URL (defaults to config.defaultSite or prompt)"
1066
2194
  },
1067
2195
  url: {
1068
2196
  type: "positional",
@@ -1076,10 +2204,15 @@ const showSubCommand = defineCommand({
1076
2204
  }
1077
2205
  },
1078
2206
  async run({ args }) {
1079
- const store = (await createCommandContext({ needsStore: true })).store;
2207
+ const ctx = await createCommandContext({
2208
+ needsAuth: true,
2209
+ needsStore: true
2210
+ });
2211
+ const store = ctx.store;
2212
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
1080
2213
  const record = await createInspectionStore({ dataSource: store.dataSource }).getLatest({
1081
2214
  userId: store.userId,
1082
- siteId: store.siteIdFor(String(args.site))
2215
+ siteId: store.siteIdFor(siteUrl)
1083
2216
  }, String(args.url));
1084
2217
  if (!record) {
1085
2218
  logger.warn(`No inspection record for ${args.url}`);
@@ -1110,8 +2243,7 @@ const sitemapsSnapshotSubCommand = defineCommand({
1110
2243
  site: {
1111
2244
  type: "string",
1112
2245
  alias: "s",
1113
- required: true,
1114
- description: "Site URL (e.g., sc-domain:example.com)"
2246
+ description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
1115
2247
  },
1116
2248
  quiet: {
1117
2249
  type: "boolean",
@@ -1132,7 +2264,7 @@ const sitemapsSnapshotSubCommand = defineCommand({
1132
2264
  });
1133
2265
  const client = ctx.client;
1134
2266
  const store = ctx.store;
1135
- const siteUrl = String(args.site);
2267
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
1136
2268
  const quiet = Boolean(args.quiet);
1137
2269
  const apiSitemaps = await client.sitemaps.list(siteUrl);
1138
2270
  const capturedAt = (/* @__PURE__ */ new Date()).toISOString();
@@ -1185,8 +2317,7 @@ const sitemapsShowSubCommand = defineCommand({
1185
2317
  site: {
1186
2318
  type: "string",
1187
2319
  alias: "s",
1188
- required: true,
1189
- description: "Site URL"
2320
+ description: "Site URL (defaults to config.defaultSite or prompt)"
1190
2321
  },
1191
2322
  path: {
1192
2323
  type: "positional",
@@ -1200,10 +2331,15 @@ const sitemapsShowSubCommand = defineCommand({
1200
2331
  }
1201
2332
  },
1202
2333
  async run({ args }) {
1203
- const store = (await createCommandContext({ needsStore: true })).store;
2334
+ const ctx = await createCommandContext({
2335
+ needsAuth: true,
2336
+ needsStore: true
2337
+ });
2338
+ const store = ctx.store;
2339
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
1204
2340
  const record = await createSitemapStore({ dataSource: store.dataSource }).getLatest({
1205
2341
  userId: store.userId,
1206
- siteId: store.siteIdFor(String(args.site))
2342
+ siteId: store.siteIdFor(siteUrl)
1207
2343
  }, String(args.path));
1208
2344
  if (!record) {
1209
2345
  logger.warn(`No sitemap record for ${args.path}`);
@@ -1249,8 +2385,7 @@ const indexingSubCommand = defineCommand({
1249
2385
  site: {
1250
2386
  type: "string",
1251
2387
  alias: "s",
1252
- required: true,
1253
- description: "Site URL (e.g., sc-domain:example.com)"
2388
+ description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
1254
2389
  },
1255
2390
  file: {
1256
2391
  type: "string",
@@ -1277,7 +2412,7 @@ const indexingSubCommand = defineCommand({
1277
2412
  });
1278
2413
  const client = ctx.client;
1279
2414
  const store = ctx.store;
1280
- const siteUrl = String(args.site);
2415
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
1281
2416
  const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
1282
2417
  const quiet = Boolean(args.quiet);
1283
2418
  const urls = await readUrlList({ file: args.file ? String(args.file) : void 0 });
@@ -1346,124 +2481,575 @@ const entitiesCommand = defineCommand({
1346
2481
  indexing: indexingSubCommand
1347
2482
  }
1348
2483
  });
1349
- const ENV_LINE_RE = /^([^=]+)=(.*)$/;
1350
- async function promptDataDir(existing) {
1351
- const fallback = existing ?? defaultDataDir();
1352
- const answer = await text({
1353
- message: "Where should Parquet data be stored?",
1354
- placeholder: fallback,
1355
- defaultValue: fallback
1356
- });
1357
- if (isCancel(answer)) process.exit(1);
1358
- return String(answer) || fallback;
1359
- }
1360
- async function loadEnvFile() {
1361
- const envPath = path.join(process.cwd(), ".env");
1362
- const content = await fs.readFile(envPath, "utf-8").catch(() => null);
1363
- if (!content) return null;
1364
- const env = {};
1365
- for (const line of content.split("\n")) {
1366
- const trimmed = line.trim();
1367
- if (!trimmed || trimmed.startsWith("#")) continue;
1368
- const match = trimmed.match(ENV_LINE_RE);
1369
- if (match) {
1370
- const key = match[1].trim();
1371
- let value = match[2].trim();
1372
- if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
1373
- env[key] = value;
1374
- }
1375
- }
1376
- return env;
2484
+ const RETRIES_ARG = { retries: {
2485
+ type: "string",
2486
+ description: "Override per-call retry count (default: 3)"
2487
+ } };
2488
+ function parseRetries(v) {
2489
+ if (v == null || v === "") return void 0;
2490
+ const n = Number.parseInt(String(v), 10);
2491
+ return Number.isFinite(n) && n >= 0 ? n : void 0;
1377
2492
  }
1378
- const initCommand = defineCommand({
1379
- meta: {
1380
- name: "init",
1381
- description: "Set up GSCDump authentication"
1382
- },
1383
- args: { force: {
1384
- type: "boolean",
1385
- alias: "f",
1386
- description: "Force re-initialization"
1387
- } },
1388
- async run({ args }) {
1389
- const config = await loadConfig();
1390
- if (config.clientId && config.clientSecret && !args.force) {
1391
- logger.info("Already configured");
1392
- logger.info("Run with --force to reconfigure");
1393
- return;
1394
- }
1395
- const envFile = await loadEnvFile();
1396
- if (envFile?.GOOGLE_CLIENT_ID && envFile?.GOOGLE_CLIENT_SECRET && envFile?.GOOGLE_REFRESH_TOKEN) {
1397
- logger.info("Found .env file with Google credentials");
1398
- process.env.GOOGLE_CLIENT_ID = envFile.GOOGLE_CLIENT_ID;
1399
- process.env.GOOGLE_CLIENT_SECRET = envFile.GOOGLE_CLIENT_SECRET;
1400
- process.env.GOOGLE_REFRESH_TOKEN = envFile.GOOGLE_REFRESH_TOKEN;
1401
- if (envFile.GOOGLE_ACCESS_TOKEN) process.env.GOOGLE_ACCESS_TOKEN = envFile.GOOGLE_ACCESS_TOKEN;
1402
- await saveConfig({
1403
- ...config,
1404
- clientId: envFile.GOOGLE_CLIENT_ID,
1405
- clientSecret: envFile.GOOGLE_CLIENT_SECRET,
1406
- dataDir: config.dataDir ?? defaultDataDir()
1407
- });
1408
- const creds = (await authenticate({
1409
- clientId: envFile.GOOGLE_CLIENT_ID,
1410
- clientSecret: envFile.GOOGLE_CLIENT_SECRET
1411
- }, false)).credentials;
1412
- if (creds.access_token) await saveTokens({
1413
- access_token: creds.access_token,
1414
- refresh_token: creds.refresh_token || envFile.GOOGLE_REFRESH_TOKEN,
1415
- expiry_date: creds.expiry_date
1416
- });
1417
- console.log();
1418
- logger.success("Setup complete using .env credentials! Run gscdump to get started.");
1419
- return;
1420
- }
1421
- console.log();
1422
- console.log(" \x1B[1mWelcome to GSCDump!\x1B[0m");
1423
- console.log(" \x1B[90mGoogle Search Console data extraction CLI\x1B[0m");
1424
- console.log();
1425
- const dataDir = await promptDataDir(config.dataDir);
1426
- const credentials = await getAuthCredentials(true);
1427
- await saveConfig({
1428
- ...config,
1429
- dataDir,
1430
- clientId: credentials.clientId,
1431
- clientSecret: credentials.clientSecret
1432
- });
1433
- await authenticate(credentials, true);
1434
- console.log();
1435
- logger.success("Setup complete! Run gscdump to get started.");
1436
- }
1437
- });
1438
- const inspectCommand = defineCommand({
2493
+ const submitCommand$1 = defineCommand({
1439
2494
  meta: {
1440
- name: "inspect",
1441
- description: "Inspect a specific URL's indexing status"
2495
+ name: "submit",
2496
+ description: "Notify Google of a new or updated URL (URL_UPDATED)"
1442
2497
  },
1443
2498
  args: {
1444
- site: {
1445
- type: "string",
1446
- alias: "s",
1447
- required: true,
1448
- description: "Site URL (e.g., sc-domain:example.com)"
1449
- },
1450
2499
  url: {
1451
2500
  type: "positional",
1452
2501
  required: true,
1453
- description: "URL to inspect"
2502
+ description: "URL to submit"
1454
2503
  },
1455
2504
  json: {
1456
2505
  type: "boolean",
1457
2506
  default: false,
1458
2507
  description: "Output as JSON"
1459
- }
2508
+ },
2509
+ quiet: {
2510
+ type: "boolean",
2511
+ alias: "q",
2512
+ default: false,
2513
+ description: "Suppress info/success output"
2514
+ },
2515
+ ...RETRIES_ARG
1460
2516
  },
1461
2517
  async run({ args }) {
1462
- const result = await (await createCommandContext({ needsAuth: true })).client.inspect(args.site, args.url).catch((e) => {
1463
- logger.error(`Inspection failed: ${e.message}`);
1464
- process.exit(1);
1465
- });
1466
- const indexStatus = (result?.inspectionResult)?.indexStatusResult;
2518
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2519
+ const result = await requestIndexing((await createCommandContext({
2520
+ needsAuth: true,
2521
+ fetchOptions: { retry: parseRetries(args.retries) }
2522
+ })).client, args.url, { type: "URL_UPDATED" }).catch(gscErrorHandler);
2523
+ if (args.json) {
2524
+ console.log(JSON.stringify(result, null, 2));
2525
+ return;
2526
+ }
2527
+ logger.success(`Submitted: ${result.url}`);
2528
+ if (result.notifyTime) console.log(` Notified: ${result.notifyTime}`);
2529
+ }
2530
+ });
2531
+ const removeCommand = defineCommand({
2532
+ meta: {
2533
+ name: "remove",
2534
+ description: "Notify Google a URL has been removed (URL_DELETED)"
2535
+ },
2536
+ args: {
2537
+ url: {
2538
+ type: "positional",
2539
+ required: true,
2540
+ description: "URL to mark removed"
2541
+ },
2542
+ json: {
2543
+ type: "boolean",
2544
+ default: false,
2545
+ description: "Output as JSON"
2546
+ },
2547
+ quiet: {
2548
+ type: "boolean",
2549
+ alias: "q",
2550
+ default: false,
2551
+ description: "Suppress info/success output"
2552
+ },
2553
+ ...RETRIES_ARG
2554
+ },
2555
+ async run({ args }) {
2556
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2557
+ const result = await requestIndexing((await createCommandContext({
2558
+ needsAuth: true,
2559
+ fetchOptions: { retry: parseRetries(args.retries) }
2560
+ })).client, args.url, { type: "URL_DELETED" }).catch(gscErrorHandler);
2561
+ if (args.json) {
2562
+ console.log(JSON.stringify(result, null, 2));
2563
+ return;
2564
+ }
2565
+ logger.success(`Removed: ${result.url}`);
2566
+ if (result.notifyTime) console.log(` Notified: ${result.notifyTime}`);
2567
+ }
2568
+ });
2569
+ const statusCommand = defineCommand({
2570
+ meta: {
2571
+ name: "status",
2572
+ description: "Show indexing notification metadata for a URL"
2573
+ },
2574
+ args: {
2575
+ url: {
2576
+ type: "positional",
2577
+ required: true,
2578
+ description: "URL to inspect"
2579
+ },
2580
+ json: {
2581
+ type: "boolean",
2582
+ default: false,
2583
+ description: "Output as JSON"
2584
+ },
2585
+ quiet: {
2586
+ type: "boolean",
2587
+ alias: "q",
2588
+ default: false,
2589
+ description: "Suppress info/success output"
2590
+ }
2591
+ },
2592
+ async run({ args }) {
2593
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2594
+ const meta = await getIndexingMetadata((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
2595
+ if (args.json) {
2596
+ console.log(JSON.stringify(meta, null, 2));
2597
+ return;
2598
+ }
2599
+ const fmtNotification = (n) => {
2600
+ if (!n?.notifyTime) return "never";
2601
+ return n.type ? `${n.notifyTime} (${n.type})` : n.notifyTime;
2602
+ };
2603
+ const update = meta.latestUpdate;
2604
+ const remove = meta.latestRemove;
2605
+ console.log();
2606
+ console.log(` \x1B[1mURL:\x1B[0m ${meta.url}`);
2607
+ console.log(` Last update notify: ${fmtNotification(update)}`);
2608
+ console.log(` Last remove notify: ${fmtNotification(remove)}`);
2609
+ console.log();
2610
+ }
2611
+ });
2612
+ const INDEXING_DAILY_QUOTA = 200;
2613
+ const indexingCommand = defineCommand({
2614
+ meta: {
2615
+ name: "indexing",
2616
+ description: "Notify Google about URL updates/removals (Indexing API)"
2617
+ },
2618
+ subCommands: {
2619
+ submit: submitCommand$1,
2620
+ remove: removeCommand,
2621
+ status: statusCommand,
2622
+ batch: defineCommand({
2623
+ meta: {
2624
+ name: "batch",
2625
+ description: "Submit many URLs from a file or stdin (one URL per line)"
2626
+ },
2627
+ args: {
2628
+ "urls": {
2629
+ type: "positional",
2630
+ required: false,
2631
+ description: "URLs (or use --file/stdin)"
2632
+ },
2633
+ "file": {
2634
+ type: "string",
2635
+ alias: "f",
2636
+ description: "File with URLs (one per line)"
2637
+ },
2638
+ "type": {
2639
+ type: "string",
2640
+ default: "URL_UPDATED",
2641
+ description: "URL_UPDATED or URL_DELETED"
2642
+ },
2643
+ "delay-ms": {
2644
+ type: "string",
2645
+ default: "100",
2646
+ description: "Delay between requests"
2647
+ },
2648
+ "concurrency": {
2649
+ type: "string",
2650
+ alias: "c",
2651
+ default: "1",
2652
+ description: "Concurrent in-flight requests"
2653
+ },
2654
+ "quiet": {
2655
+ type: "boolean",
2656
+ alias: "q",
2657
+ default: false,
2658
+ description: "Suppress progress output"
2659
+ },
2660
+ "json": {
2661
+ type: "boolean",
2662
+ default: false,
2663
+ description: "Output as JSON"
2664
+ },
2665
+ "yes": {
2666
+ type: "boolean",
2667
+ alias: "y",
2668
+ default: false,
2669
+ description: "Skip the over-quota confirmation prompt"
2670
+ },
2671
+ "retries": {
2672
+ type: "string",
2673
+ description: "Override per-call retry count (default: 3)"
2674
+ }
2675
+ },
2676
+ async run({ args }) {
2677
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2678
+ const urls = await readUrlList$1(args);
2679
+ if (urls.length === 0) {
2680
+ logger.error("No URLs provided. Pass URLs as args, --file, or stdin.");
2681
+ process.exit(1);
2682
+ }
2683
+ const type = String(args.type);
2684
+ if (type !== "URL_UPDATED" && type !== "URL_DELETED") {
2685
+ logger.error(`Invalid --type: ${type}. Use URL_UPDATED or URL_DELETED.`);
2686
+ process.exit(1);
2687
+ }
2688
+ if (urls.length > INDEXING_DAILY_QUOTA && !args.yes && !args.json) {
2689
+ logger.warn(`Submitting ${urls.length} URLs but the Indexing API daily quota is ${INDEXING_DAILY_QUOTA}/day.`);
2690
+ logger.warn(`Excess submissions will fail with quota errors. Pass --yes to proceed anyway.`);
2691
+ process.exit(1);
2692
+ }
2693
+ if (urls.length > INDEXING_DAILY_QUOTA && !args.json && !args.quiet) logger.warn(`Proceeding with ${urls.length} URLs (over the ${INDEXING_DAILY_QUOTA}/day quota). Excess will fail.`);
2694
+ const ctx = await createCommandContext({
2695
+ needsAuth: true,
2696
+ fetchOptions: { retry: parseRetries(args.retries) }
2697
+ });
2698
+ const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
2699
+ const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
2700
+ if (!args.json && !args.quiet) logger.info(`Submitting ${urls.length} URLs (${type}) ...`);
2701
+ const results = await batchRequestIndexing(ctx.client, urls, {
2702
+ type,
2703
+ delayMs,
2704
+ concurrency,
2705
+ onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url}`)
2706
+ }).catch(gscErrorHandler);
2707
+ if (args.json) {
2708
+ console.log(JSON.stringify(results, null, 2));
2709
+ return;
2710
+ }
2711
+ if (!args.quiet) logger.success(`Submitted ${results.length}/${urls.length} URLs`);
2712
+ }
2713
+ })
2714
+ }
2715
+ });
2716
+ const ENV_LINE_RE = /^([^=]+)=(.*)$/;
2717
+ async function promptDataDir(existing) {
2718
+ const fallback = existing ?? defaultDataDir();
2719
+ const answer = await text({
2720
+ message: "Where should Parquet data be stored?",
2721
+ placeholder: fallback,
2722
+ defaultValue: fallback
2723
+ });
2724
+ if (isCancel(answer)) process.exit(1);
2725
+ return String(answer) || fallback;
2726
+ }
2727
+ async function loadEnvFile() {
2728
+ const envPath = path.join(process.cwd(), ".env");
2729
+ const content = await fs.readFile(envPath, "utf-8").catch(() => null);
2730
+ if (!content) return null;
2731
+ const env = {};
2732
+ for (const line of content.split("\n")) {
2733
+ const trimmed = line.trim();
2734
+ if (!trimmed || trimmed.startsWith("#")) continue;
2735
+ const match = trimmed.match(ENV_LINE_RE);
2736
+ if (match) {
2737
+ const key = match[1].trim();
2738
+ let value = match[2].trim();
2739
+ if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
2740
+ env[key] = value;
2741
+ }
2742
+ }
2743
+ return env;
2744
+ }
2745
+ const initCommand = defineCommand({
2746
+ meta: {
2747
+ name: "init",
2748
+ description: "Set up GSCDump authentication"
2749
+ },
2750
+ args: {
2751
+ "force": {
2752
+ type: "boolean",
2753
+ alias: "f",
2754
+ description: "Force re-initialization"
2755
+ },
2756
+ "no-store": {
2757
+ type: "boolean",
2758
+ default: false,
2759
+ description: "Skip dataDir prompt (auth-only setup)"
2760
+ },
2761
+ "quiet": {
2762
+ type: "boolean",
2763
+ alias: "q",
2764
+ default: false,
2765
+ description: "Suppress info/success output"
2766
+ }
2767
+ },
2768
+ async run({ args }) {
2769
+ setQuiet(Boolean(args.quiet));
2770
+ const config = await loadConfig();
2771
+ if (config.clientId && config.clientSecret && !args.force) {
2772
+ logger.info("Already configured");
2773
+ logger.info("Run with --force to reconfigure");
2774
+ return;
2775
+ }
2776
+ const byok = resolveBYOK();
2777
+ if (byok) {
2778
+ const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
2779
+ await saveConfig({
2780
+ ...config,
2781
+ dataDir: dataDir ?? config.dataDir
2782
+ });
2783
+ logger.success(`BYOK detected (${typeof byok === "string" ? "access-token" : "refresh-token"}) — auth setup skipped`);
2784
+ logger.success("Setup complete! Run gscdump to get started.");
2785
+ return;
2786
+ }
2787
+ const envFile = await loadEnvFile();
2788
+ const envCid = envFile?.GSC_CLIENT_ID ?? envFile?.GOOGLE_CLIENT_ID;
2789
+ const envSec = envFile?.GSC_CLIENT_SECRET ?? envFile?.GOOGLE_CLIENT_SECRET;
2790
+ const envRef = envFile?.GSC_REFRESH_TOKEN ?? envFile?.GOOGLE_REFRESH_TOKEN;
2791
+ const envAcc = envFile?.GSC_ACCESS_TOKEN ?? envFile?.GOOGLE_ACCESS_TOKEN;
2792
+ if (envCid && envSec && envRef) {
2793
+ logger.info("Found .env file with credentials");
2794
+ process.env.GOOGLE_CLIENT_ID = envCid;
2795
+ process.env.GOOGLE_CLIENT_SECRET = envSec;
2796
+ process.env.GOOGLE_REFRESH_TOKEN = envRef;
2797
+ if (envAcc) process.env.GOOGLE_ACCESS_TOKEN = envAcc;
2798
+ await saveConfig({
2799
+ ...config,
2800
+ clientId: envCid,
2801
+ clientSecret: envSec,
2802
+ dataDir: config.dataDir ?? defaultDataDir()
2803
+ });
2804
+ const creds = (await authenticate({
2805
+ clientId: envCid,
2806
+ clientSecret: envSec
2807
+ }, false)).credentials;
2808
+ if (creds.access_token) await saveTokens({
2809
+ access_token: creds.access_token,
2810
+ refresh_token: creds.refresh_token || envRef,
2811
+ expiry_date: creds.expiry_date
2812
+ });
2813
+ console.log();
2814
+ logger.success("Setup complete using .env credentials! Run gscdump to get started.");
2815
+ return;
2816
+ }
2817
+ console.log();
2818
+ console.log(" \x1B[1mWelcome to GSCDump!\x1B[0m");
2819
+ console.log(" \x1B[90mGoogle Search Console data extraction CLI\x1B[0m");
2820
+ console.log();
2821
+ const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
2822
+ const credentials = await getAuthCredentials(true);
2823
+ await saveConfig({
2824
+ ...config,
2825
+ ...dataDir ? { dataDir } : {},
2826
+ clientId: credentials.clientId,
2827
+ clientSecret: credentials.clientSecret
2828
+ });
2829
+ await smokeTest(await authenticate(credentials, true));
2830
+ await maybeWriteEnvFile(credentials.clientId, credentials.clientSecret);
2831
+ console.log();
2832
+ logger.success("Setup complete! Run gscdump to get started.");
2833
+ }
2834
+ });
2835
+ async function smokeTest(oauth) {
2836
+ const sites = await googleSearchConsole(oauth).sites().catch((e) => e);
2837
+ if (sites instanceof Error) {
2838
+ logger.warn(`Smoke test failed: ${sites.message}`);
2839
+ logger.info("Auth saved, but verify scopes via `gscdump auth status` / `gscdump doctor`.");
2840
+ return;
2841
+ }
2842
+ logger.success(`Verified: ${sites.length} GSC site(s) accessible`);
2843
+ }
2844
+ async function maybeWriteEnvFile(clientId, clientSecret) {
2845
+ const tokens = await loadTokens();
2846
+ if (!tokens?.refresh_token) return;
2847
+ const wants = await confirm({
2848
+ message: "Write a `.env` file with these credentials? (handy for CI / other machines)",
2849
+ initialValue: false
2850
+ });
2851
+ if (isCancel(wants) || !wants) return;
2852
+ const envPath = path.join(process.cwd(), ".env");
2853
+ if (await fs.stat(envPath).then(() => true).catch(() => false)) {
2854
+ const overwrite = await confirm({
2855
+ message: `${displayPath(envPath)} exists. Overwrite?`,
2856
+ initialValue: false
2857
+ });
2858
+ if (isCancel(overwrite) || !overwrite) {
2859
+ logger.info(`Skipped — keep credentials at ${displayPath(envPath)} manually if needed`);
2860
+ return;
2861
+ }
2862
+ }
2863
+ const content = [
2864
+ `GSC_CLIENT_ID=${clientId}`,
2865
+ `GSC_CLIENT_SECRET=${clientSecret}`,
2866
+ `GSC_REFRESH_TOKEN=${tokens.refresh_token}`,
2867
+ ""
2868
+ ].join("\n");
2869
+ await fs.writeFile(envPath, content, { mode: 384 });
2870
+ logger.success(`Wrote ${displayPath(envPath)}`);
2871
+ }
2872
+ function verdictTone(verdict) {
2873
+ if (verdict === "PASS") return "\x1B[32m";
2874
+ if (verdict === "NEUTRAL" || verdict === "PARTIAL") return "\x1B[33m";
2875
+ if (verdict === "FAIL") return "\x1B[31m";
2876
+ return "\x1B[90m";
2877
+ }
2878
+ function colorVerdict(verdict) {
2879
+ return `${verdictTone(verdict)}${verdict || "N/A"}\x1B[0m`;
2880
+ }
2881
+ function printInspection(url, inspection) {
2882
+ const indexStatus = inspection?.indexStatusResult;
2883
+ console.log();
2884
+ console.log(` \x1B[1mURL:\x1B[0m ${url}`);
2885
+ console.log();
2886
+ console.log(` \x1B[1mIndex status\x1B[0m`);
2887
+ console.log(` Verdict: ${colorVerdict(indexStatus?.verdict)}`);
2888
+ if (indexStatus?.coverageState) console.log(` Coverage: ${indexStatus.coverageState}`);
2889
+ if (indexStatus?.indexingState) console.log(` Indexing: ${indexStatus.indexingState}`);
2890
+ if (indexStatus?.lastCrawlTime) console.log(` Last Crawl: ${indexStatus.lastCrawlTime}`);
2891
+ if (indexStatus?.crawledAs) console.log(` Crawled As: ${indexStatus.crawledAs}`);
2892
+ if (indexStatus?.robotsTxtState) console.log(` Robots.txt: ${indexStatus.robotsTxtState}`);
2893
+ if (indexStatus?.pageFetchState) console.log(` Page Fetch: ${indexStatus.pageFetchState}`);
2894
+ if (indexStatus?.googleCanonical) console.log(` Google Canon: ${indexStatus.googleCanonical}`);
2895
+ if (indexStatus?.userCanonical) console.log(` User Canon: ${indexStatus.userCanonical}`);
2896
+ if (indexStatus?.sitemap?.length) {
2897
+ console.log(` Sitemaps:`);
2898
+ for (const sm of indexStatus.sitemap) console.log(` \x1B[90m└─\x1B[0m ${sm}`);
2899
+ }
2900
+ if (indexStatus?.referringUrls?.length) {
2901
+ const shown = indexStatus.referringUrls.slice(0, 5);
2902
+ console.log(` Referring URLs (${indexStatus.referringUrls.length}):`);
2903
+ for (const r of shown) console.log(` \x1B[90m└─\x1B[0m ${r}`);
2904
+ if (indexStatus.referringUrls.length > shown.length) console.log(` \x1B[90m… ${indexStatus.referringUrls.length - shown.length} more\x1B[0m`);
2905
+ }
2906
+ const rich = inspection?.richResultsResult;
2907
+ if (rich) {
2908
+ console.log();
2909
+ console.log(` \x1B[1mRich results\x1B[0m`);
2910
+ console.log(` Verdict: ${colorVerdict(rich.verdict)}`);
2911
+ if (rich.detectedItems?.length) for (const group of rich.detectedItems) {
2912
+ const count = group.items?.length ?? 0;
2913
+ console.log(` ${group.richResultType ?? "unknown"}: ${count} item${count === 1 ? "" : "s"}`);
2914
+ for (const item of group.items ?? []) if (item.issues?.length) for (const issue of item.issues) {
2915
+ const sev = issue.severity === "ERROR" ? "\x1B[31m" : "\x1B[33m";
2916
+ console.log(` ${sev}${issue.severity ?? "?"}\x1B[0m ${item.name ?? ""}: ${issue.issueMessage ?? ""}`);
2917
+ }
2918
+ }
2919
+ }
2920
+ const amp = inspection?.ampResult;
2921
+ if (amp) {
2922
+ console.log();
2923
+ console.log(` \x1B[1mAMP\x1B[0m`);
2924
+ console.log(` Verdict: ${colorVerdict(amp.verdict)}`);
2925
+ if (amp.ampUrl) console.log(` AMP URL: ${amp.ampUrl}`);
2926
+ if (amp.ampIndexStatusVerdict) console.log(` Index Verdict: ${amp.ampIndexStatusVerdict}`);
2927
+ if (amp.indexingState) console.log(` Indexing: ${amp.indexingState}`);
2928
+ if (amp.robotsTxtState) console.log(` Robots.txt: ${amp.robotsTxtState}`);
2929
+ if (amp.pageFetchState) console.log(` Page Fetch: ${amp.pageFetchState}`);
2930
+ if (amp.lastCrawlTime) console.log(` Last Crawl: ${amp.lastCrawlTime}`);
2931
+ if (amp.issues?.length) for (const issue of amp.issues) {
2932
+ const sev = issue.severity === "ERROR" ? "\x1B[31m" : "\x1B[33m";
2933
+ console.log(` ${sev}${issue.severity ?? "?"}\x1B[0m ${issue.issueMessage ?? ""}`);
2934
+ }
2935
+ }
2936
+ const mobile = inspection?.mobileUsabilityResult;
2937
+ if (mobile) {
2938
+ console.log();
2939
+ console.log(` \x1B[1mMobile usability\x1B[0m`);
2940
+ console.log(` Verdict: ${colorVerdict(mobile.verdict)}`);
2941
+ if (mobile.issues?.length) for (const issue of mobile.issues) console.log(` \x1B[33m${issue.issueType ?? "?"}\x1B[0m ${issue.message ?? ""}`);
2942
+ }
2943
+ if (inspection?.inspectionResultLink) {
2944
+ console.log();
2945
+ console.log(` \x1B[90mOpen in Search Console:\x1B[0m \x1B[36m${inspection.inspectionResultLink}\x1B[0m`);
2946
+ }
2947
+ console.log();
2948
+ }
2949
+ const inspectCommand = defineCommand({
2950
+ meta: {
2951
+ name: "inspect",
2952
+ description: "Inspect URL indexing status (single URL; use `inspect batch` for many)"
2953
+ },
2954
+ args: {
2955
+ site: {
2956
+ type: "string",
2957
+ alias: "s",
2958
+ description: "Site URL (defaults to config.defaultSite or prompt)"
2959
+ },
2960
+ url: {
2961
+ type: "positional",
2962
+ required: true,
2963
+ description: "URL to inspect"
2964
+ },
2965
+ json: {
2966
+ type: "boolean",
2967
+ default: false,
2968
+ description: "Output as JSON"
2969
+ },
2970
+ quiet: {
2971
+ type: "boolean",
2972
+ alias: "q",
2973
+ default: false,
2974
+ description: "Suppress info/success output"
2975
+ }
2976
+ },
2977
+ subCommands: { batch: defineCommand({
2978
+ meta: {
2979
+ name: "batch",
2980
+ description: "Inspect many URLs from a file or stdin (one URL per line)"
2981
+ },
2982
+ args: {
2983
+ "site": {
2984
+ type: "string",
2985
+ alias: "s",
2986
+ description: "Site URL (defaults to config.defaultSite or prompt)"
2987
+ },
2988
+ "urls": {
2989
+ type: "positional",
2990
+ required: false,
2991
+ description: "URLs (or use --file/stdin)"
2992
+ },
2993
+ "file": {
2994
+ type: "string",
2995
+ alias: "f",
2996
+ description: "File with URLs (one per line)"
2997
+ },
2998
+ "delay-ms": {
2999
+ type: "string",
3000
+ default: "200",
3001
+ description: "Delay between requests"
3002
+ },
3003
+ "concurrency": {
3004
+ type: "string",
3005
+ alias: "c",
3006
+ default: "1",
3007
+ description: "Concurrent in-flight requests"
3008
+ },
3009
+ "quiet": {
3010
+ type: "boolean",
3011
+ alias: "q",
3012
+ default: false,
3013
+ description: "Suppress progress output"
3014
+ },
3015
+ "json": {
3016
+ type: "boolean",
3017
+ default: false,
3018
+ description: "Output as JSON"
3019
+ }
3020
+ },
3021
+ async run({ args }) {
3022
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
3023
+ const urls = await readUrlList$1(args);
3024
+ if (urls.length === 0) {
3025
+ logger.error("No URLs provided. Pass URLs as args, --file, or stdin.");
3026
+ process.exit(1);
3027
+ }
3028
+ const ctx = await createCommandContext({ needsAuth: true });
3029
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
3030
+ const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
3031
+ const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
3032
+ if (!args.json && !args.quiet) logger.info(`Inspecting ${urls.length} URLs ...`);
3033
+ const results = await batchInspectUrls(ctx.client, siteUrl, urls, {
3034
+ delayMs,
3035
+ concurrency,
3036
+ onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url} ${r.isIndexed ? "PASS" : "FAIL"}`)
3037
+ }).catch(gscErrorHandler);
3038
+ if (args.json) {
3039
+ console.log(JSON.stringify(results, null, 2));
3040
+ return;
3041
+ }
3042
+ const indexed = results.filter((r) => r.isIndexed).length;
3043
+ if (!args.quiet) logger.success(`Inspected ${results.length} URLs (${indexed} indexed, ${results.length - indexed} not)`);
3044
+ }
3045
+ }) },
3046
+ async run({ args }) {
3047
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
3048
+ const ctx = await createCommandContext({ needsAuth: true });
3049
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
3050
+ const result = await ctx.client.inspect(siteUrl, args.url).catch(gscErrorHandler);
3051
+ const inspection = result?.inspectionResult;
3052
+ const indexStatus = inspection?.indexStatusResult;
1467
3053
  if (args.json) {
1468
3054
  console.log(JSON.stringify({
1469
3055
  url: args.url,
@@ -1476,23 +3062,11 @@ const inspectCommand = defineCommand({
1476
3062
  }, null, 2));
1477
3063
  return;
1478
3064
  }
1479
- console.log();
1480
- console.log(` \x1B[1mURL:\x1B[0m ${args.url}`);
1481
- console.log();
1482
- const verdictColor = indexStatus?.verdict === "PASS" ? "\x1B[32m" : "\x1B[31m";
1483
- console.log(` Verdict: ${verdictColor}${indexStatus?.verdict || "N/A"}\x1B[0m`);
1484
- if (indexStatus?.coverageState) console.log(` Coverage: ${indexStatus.coverageState}`);
1485
- if (indexStatus?.indexingState) console.log(` Indexing: ${indexStatus.indexingState}`);
1486
- if (indexStatus?.lastCrawlTime) console.log(` Last Crawl: ${indexStatus.lastCrawlTime}`);
1487
- if (indexStatus?.robotsTxtState) console.log(` Robots.txt: ${indexStatus.robotsTxtState}`);
1488
- if (indexStatus?.pageFetchState) console.log(` Page Fetch: ${indexStatus.pageFetchState}`);
1489
- if (indexStatus?.googleCanonical) console.log(` Google Canon: ${indexStatus.googleCanonical}`);
1490
- if (indexStatus?.userCanonical) console.log(` User Canon: ${indexStatus.userCanonical}`);
1491
- console.log();
3065
+ printInspection(args.url, inspection);
1492
3066
  }
1493
3067
  });
1494
3068
  async function checkAuth() {
1495
- if ((process.env.GOOGLE_ACCESS_TOKEN || process.env.GOOGLE_REFRESH_TOKEN) && process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET) return { ok: true };
3069
+ if (resolveBYOK()) return { ok: true };
1496
3070
  const config = await loadConfig();
1497
3071
  if (!config.clientId && !config.clientSecret) return {
1498
3072
  ok: false,
@@ -1502,7 +3076,8 @@ Run this command to set up authentication:
1502
3076
 
1503
3077
  npx @gscdump/cli init
1504
3078
 
1505
- Or provide env vars: GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, GOOGLE_ACCESS_TOKEN
3079
+ Or set BYOK env vars: GSC_ACCESS_TOKEN, or GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN
3080
+ (GOOGLE_* aliases also accepted).
1506
3081
 
1507
3082
  Then restart your MCP client.`
1508
3083
  };
@@ -1512,7 +3087,7 @@ Then restart your MCP client.`
1512
3087
 
1513
3088
  Run this command to authenticate:
1514
3089
 
1515
- npx @gscdump/cli auth
3090
+ npx @gscdump/cli auth login
1516
3091
 
1517
3092
  Then restart your MCP client.`
1518
3093
  };
@@ -1546,23 +3121,78 @@ const DIMENSIONS = [
1546
3121
  "device",
1547
3122
  "searchAppearance"
1548
3123
  ];
1549
- const DIM_COLUMNS = {
1550
- page,
3124
+ const DIM_COLUMNS = {
3125
+ page,
3126
+ query,
3127
+ date,
3128
+ country,
3129
+ device,
3130
+ searchAppearance
3131
+ };
3132
+ const FILTER_DIMS = [
3133
+ "query",
3134
+ "page",
3135
+ "country",
3136
+ "device",
3137
+ "searchAppearance"
3138
+ ];
3139
+ const FILTER_COL = {
1551
3140
  query,
1552
- date,
3141
+ page,
1553
3142
  country,
1554
3143
  device,
1555
3144
  searchAppearance
1556
3145
  };
3146
+ const ALL_SEARCH_TYPES$1 = Object.values(SearchTypes);
3147
+ const DATA_STATES = [
3148
+ "all",
3149
+ "final",
3150
+ "hourly_all"
3151
+ ];
3152
+ const AGGREGATION_TYPES = [
3153
+ "auto",
3154
+ "byPage",
3155
+ "byProperty"
3156
+ ];
3157
+ function buildFilterFromArg(col, raw) {
3158
+ if (raw.startsWith("!~")) return makeLeaf(col, "notContains", raw.slice(2));
3159
+ if (raw.startsWith("!re:")) return notRegex(col, raw.slice(4));
3160
+ if (raw.startsWith("!")) return makeLeaf(col, "notEquals", raw.slice(1));
3161
+ if (raw.startsWith("~")) return contains(col, raw.slice(1));
3162
+ if (raw.startsWith("re:")) return regex(col, raw.slice(3));
3163
+ if (raw.startsWith("contains:")) return contains(col, raw.slice(9));
3164
+ if (raw.startsWith("eq:")) return eq(col, raw.slice(3));
3165
+ return eq(col, raw);
3166
+ }
3167
+ function makeLeaf(col, operator, value) {
3168
+ return {
3169
+ _constraints: {},
3170
+ _filters: [{
3171
+ dimension: col.dimension,
3172
+ operator,
3173
+ expression: value
3174
+ }]
3175
+ };
3176
+ }
1557
3177
  async function runLiveQuery(client, siteUrl, opts) {
1558
3178
  const allRows = [];
1559
3179
  let startRow = 0;
3180
+ const baseBody = {
3181
+ startDate: opts.startDate,
3182
+ endDate: opts.endDate,
3183
+ dimensions: opts.dimensions,
3184
+ rowLimit: opts.rowLimit
3185
+ };
3186
+ if (opts.searchType) baseBody.searchType = opts.searchType;
3187
+ if (opts.dataState) baseBody.dataState = opts.dataState;
3188
+ if (opts.aggregationType) baseBody.aggregationType = opts.aggregationType;
3189
+ if (opts.dimensionFilter) {
3190
+ const groups = filterToGroups(opts.dimensionFilter);
3191
+ if (groups.length > 0) baseBody.dimensionFilterGroups = groups;
3192
+ }
1560
3193
  while (true) {
1561
3194
  const rows = ((await client._rawQuery(siteUrl, {
1562
- startDate: opts.startDate,
1563
- endDate: opts.endDate,
1564
- dimensions: opts.dimensions,
1565
- rowLimit: opts.rowLimit,
3195
+ ...baseBody,
1566
3196
  startRow
1567
3197
  })).rows || []).map((row) => {
1568
3198
  const result = {
@@ -1582,71 +3212,116 @@ async function runLiveQuery(client, siteUrl, opts) {
1582
3212
  }
1583
3213
  return { rows: allRows };
1584
3214
  }
3215
+ function filterToGroups(f) {
3216
+ if (f._filters.length === 0) return [];
3217
+ return [{ filters: f._filters.map((leaf) => ({
3218
+ dimension: leaf.dimension,
3219
+ operator: leaf.operator,
3220
+ expression: leaf.expression
3221
+ })) }];
3222
+ }
1585
3223
  const queryCommand = defineCommand({
1586
3224
  meta: {
1587
3225
  name: "query",
1588
3226
  description: "Run a search analytics query (local Parquet by default, --live hits GSC API)"
1589
3227
  },
1590
3228
  args: {
1591
- site: {
3229
+ "site": {
1592
3230
  type: "string",
1593
3231
  alias: "s",
1594
3232
  description: "Site URL (e.g., sc-domain:example.com)"
1595
3233
  },
1596
- dimensions: {
3234
+ "dimensions": {
1597
3235
  type: "string",
1598
3236
  alias: "d",
1599
3237
  description: `Dimensions: ${DIMENSIONS.join(",")}`
1600
3238
  },
1601
- start: {
3239
+ "start": {
1602
3240
  type: "string",
1603
3241
  description: "Start date (YYYY-MM-DD)"
1604
3242
  },
1605
- end: {
3243
+ "end": {
1606
3244
  type: "string",
1607
3245
  description: "End date (YYYY-MM-DD)"
1608
3246
  },
1609
- limit: {
3247
+ "limit": {
1610
3248
  type: "string",
1611
3249
  alias: "l",
1612
3250
  default: "1000",
1613
3251
  description: "Max rows (default: 1000)"
1614
3252
  },
1615
- output: {
3253
+ "output": {
1616
3254
  type: "string",
1617
3255
  alias: "o",
1618
3256
  description: "Output file path (default: stdout)"
1619
3257
  },
1620
- format: {
3258
+ "format": {
1621
3259
  type: "string",
1622
3260
  alias: "f",
1623
3261
  default: "json",
1624
3262
  description: "Output format: json or csv"
1625
3263
  },
1626
- sql: {
3264
+ "sql": {
1627
3265
  type: "string",
1628
3266
  description: "Raw DuckDB SQL using {{FILES}} as the file list placeholder (bypasses builder)"
1629
3267
  },
1630
- table: {
3268
+ "table": {
1631
3269
  type: "string",
1632
3270
  description: "Analytics table for --sql (default: pages)"
1633
3271
  },
1634
- live: {
3272
+ "live": {
1635
3273
  type: "boolean",
1636
3274
  default: false,
1637
3275
  description: "Bypass local store; hit the GSC API directly"
1638
3276
  },
1639
- quiet: {
3277
+ "quiet": {
1640
3278
  type: "boolean",
1641
3279
  alias: "q",
1642
3280
  default: false,
1643
3281
  description: "Suppress progress output"
1644
3282
  },
1645
- interactive: {
3283
+ "interactive": {
1646
3284
  type: "boolean",
1647
3285
  alias: "i",
1648
3286
  default: false,
1649
3287
  description: "Interactive mode"
3288
+ },
3289
+ "query": {
3290
+ type: "string",
3291
+ description: "Filter by query (prefix: ~contains, !exclude, re:regex, !re:not-regex)"
3292
+ },
3293
+ "page": {
3294
+ type: "string",
3295
+ description: "Filter by page (same prefix syntax as --query)"
3296
+ },
3297
+ "country": {
3298
+ type: "string",
3299
+ description: "Filter by country (ISO-3 lowercase, e.g. usa). Same prefix syntax as --query"
3300
+ },
3301
+ "device": {
3302
+ type: "string",
3303
+ description: "Filter by device (DESKTOP/MOBILE/TABLET). Same prefix syntax"
3304
+ },
3305
+ "search-appearance": {
3306
+ type: "string",
3307
+ description: "Filter by search appearance feature. Same prefix syntax"
3308
+ },
3309
+ "type": {
3310
+ type: "string",
3311
+ description: `Search type (live mode only). One of: ${ALL_SEARCH_TYPES$1.join(",")}`
3312
+ },
3313
+ "data-state": {
3314
+ type: "string",
3315
+ description: `Data state (live mode only). One of: ${DATA_STATES.join(",")} (default: final)`
3316
+ },
3317
+ "aggregation-type": {
3318
+ type: "string",
3319
+ description: `Aggregation type (live mode only). One of: ${AGGREGATION_TYPES.join(",")}`
3320
+ },
3321
+ "explain": {
3322
+ type: "boolean",
3323
+ default: false,
3324
+ description: "Print the request body / planned local SQL and exit without executing"
1650
3325
  }
1651
3326
  },
1652
3327
  async run({ args }) {
@@ -1660,10 +3335,25 @@ const queryCommand = defineCommand({
1660
3335
  });
1661
3336
  return;
1662
3337
  }
3338
+ const ctxConfig = await loadConfig();
1663
3339
  const dimNames = await resolveDimensions(args);
1664
3340
  const { startDate, endDate } = await resolveRange(args);
1665
- const rowLimit = Number.parseInt(String(args.limit), 10);
3341
+ await promptFilters(args);
3342
+ const limitArg = args.limit != null ? String(args.limit) : null;
3343
+ const rowLimit = limitArg != null && limitArg !== "1000" ? Number.parseInt(limitArg, 10) : ctxConfig.defaultLimit ?? 1e3;
1666
3344
  const format = String(args.format);
3345
+ const dimensionFilter = buildDimensionFilter(args);
3346
+ const searchType = parseSearchType(args.type ?? ctxConfig.defaultSearchType);
3347
+ const dataState = args["data-state"] ? String(args["data-state"]) : ctxConfig.defaultDataState;
3348
+ const aggregationType = args["aggregation-type"] ? String(args["aggregation-type"]) : void 0;
3349
+ if (dataState && !DATA_STATES.includes(dataState)) {
3350
+ logger.error(`Invalid --data-state: ${dataState}. Allowed: ${DATA_STATES.join(", ")}`);
3351
+ process.exit(1);
3352
+ }
3353
+ if (aggregationType && !AGGREGATION_TYPES.includes(aggregationType)) {
3354
+ logger.error(`Invalid --aggregation-type: ${aggregationType}. Allowed: ${AGGREGATION_TYPES.join(", ")}`);
3355
+ process.exit(1);
3356
+ }
1667
3357
  const ctx = await createCommandContext({
1668
3358
  needsAuth: true,
1669
3359
  needsStore: !args.live,
@@ -1671,16 +3361,34 @@ const queryCommand = defineCommand({
1671
3361
  });
1672
3362
  const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
1673
3363
  if (args.live) {
3364
+ if (args.explain) {
3365
+ const body = {
3366
+ startDate,
3367
+ endDate,
3368
+ dimensions: dimNames,
3369
+ rowLimit
3370
+ };
3371
+ if (searchType) body.searchType = searchType;
3372
+ if (dataState) body.dataState = dataState;
3373
+ if (aggregationType) body.aggregationType = aggregationType;
3374
+ if (dimensionFilter) body.dimensionFilterGroups = filterToGroups(dimensionFilter);
3375
+ console.log(JSON.stringify({
3376
+ siteUrl,
3377
+ body
3378
+ }, null, 2));
3379
+ return;
3380
+ }
1674
3381
  if (!args.quiet) logger.info(`Querying ${siteUrl} via live GSC API...`);
1675
3382
  const result = await runLiveQuery(ctx.client, siteUrl, {
1676
3383
  startDate,
1677
3384
  endDate,
1678
3385
  dimensions: dimNames,
1679
- rowLimit
1680
- }).catch((e) => {
1681
- logger.error(`Query failed: ${e.message}`);
1682
- process.exit(1);
1683
- });
3386
+ rowLimit,
3387
+ searchType,
3388
+ dataState,
3389
+ aggregationType,
3390
+ dimensionFilter
3391
+ }).catch(gscErrorHandler);
1684
3392
  await writeOutput({
1685
3393
  output: {
1686
3394
  siteUrl,
@@ -1698,10 +3406,23 @@ const queryCommand = defineCommand({
1698
3406
  });
1699
3407
  return;
1700
3408
  }
3409
+ if (searchType && searchType !== "web") {
3410
+ logger.error(`--type=${searchType} requires --live (local store query path is web-only).`);
3411
+ process.exit(1);
3412
+ }
3413
+ if (dataState || aggregationType) logger.warn("--data-state / --aggregation-type are ignored without --live");
1701
3414
  if (!args.quiet) logger.info(`Querying ${siteUrl} from local Parquet store...`);
1702
- const state = buildLocalState(dimNames, startDate, endDate, rowLimit);
3415
+ const state = buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter);
1703
3416
  const store = ctx.store;
1704
3417
  const table = inferTable(dimNames);
3418
+ if (args.explain) {
3419
+ console.log(JSON.stringify({
3420
+ siteUrl,
3421
+ table,
3422
+ state
3423
+ }, null, 2));
3424
+ return;
3425
+ }
1705
3426
  await assertRangeCovered(store, siteUrl, table, startDate, endDate);
1706
3427
  const result = await store.engine.query({
1707
3428
  userId: store.userId,
@@ -1779,9 +3500,68 @@ async function resolveRange(args) {
1779
3500
  endDate: daysAgo(3)
1780
3501
  };
1781
3502
  }
1782
- function buildLocalState(dimNames, startDate, endDate, rowLimit) {
3503
+ async function promptFilters(args) {
3504
+ if (!args.interactive) return;
3505
+ for (const dim of FILTER_DIMS) {
3506
+ if (args[dim]) continue;
3507
+ const v = await text({
3508
+ message: `Filter by ${dim} (blank to skip; prefix ~ for contains, ! for not-equals, re: for regex)`,
3509
+ placeholder: ""
3510
+ });
3511
+ if (isCancel(v)) {
3512
+ cancel("Cancelled");
3513
+ process.exit(0);
3514
+ }
3515
+ if (v && String(v).length > 0) args[dim] = String(v);
3516
+ }
3517
+ if (!args.type) {
3518
+ const t = await text({
3519
+ message: `Search type (blank for default web; allowed: ${ALL_SEARCH_TYPES$1.join(", ")})`,
3520
+ placeholder: ""
3521
+ });
3522
+ if (isCancel(t)) {
3523
+ cancel("Cancelled");
3524
+ process.exit(0);
3525
+ }
3526
+ if (t && String(t).length > 0) args.type = String(t);
3527
+ }
3528
+ if (!args["data-state"]) {
3529
+ const ds = await text({
3530
+ message: `Data state (blank for default 'final'; allowed: ${DATA_STATES.join(", ")})`,
3531
+ placeholder: ""
3532
+ });
3533
+ if (isCancel(ds)) {
3534
+ cancel("Cancelled");
3535
+ process.exit(0);
3536
+ }
3537
+ if (ds && String(ds).length > 0) args["data-state"] = String(ds);
3538
+ }
3539
+ }
3540
+ function buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter) {
1783
3541
  const dims = dimNames.map((d) => DIM_COLUMNS[d]).filter((c) => Boolean(c));
1784
- return gsc.select(...dims).where(between(date, startDate, endDate)).limit(rowLimit).getState();
3542
+ const dateFilter = between(date, startDate, endDate);
3543
+ const filter = dimensionFilter ? and(dateFilter, dimensionFilter) : dateFilter;
3544
+ return gsc.select(...dims).where(filter).limit(rowLimit).getState();
3545
+ }
3546
+ function buildDimensionFilter(args) {
3547
+ const leaves = [];
3548
+ for (const dim of FILTER_DIMS) {
3549
+ const raw = args[dim];
3550
+ if (raw == null || raw === "") continue;
3551
+ leaves.push(buildFilterFromArg(FILTER_COL[dim], String(raw)));
3552
+ }
3553
+ if (leaves.length === 0) return void 0;
3554
+ if (leaves.length === 1) return leaves[0];
3555
+ return and(...leaves);
3556
+ }
3557
+ function parseSearchType(value) {
3558
+ if (!value) return void 0;
3559
+ const v = String(value);
3560
+ if (!ALL_SEARCH_TYPES$1.includes(v)) {
3561
+ logger.error(`Invalid --type: ${v}. Allowed: ${ALL_SEARCH_TYPES$1.join(", ")}`);
3562
+ process.exit(1);
3563
+ }
3564
+ return v;
1785
3565
  }
1786
3566
  async function assertRangeCovered(store, siteUrl, table, startDate, endDate) {
1787
3567
  const wm = (await store.engine.getWatermarks({
@@ -1827,14 +3607,14 @@ async function runRawSqlMode(opts) {
1827
3607
  total: rows.length,
1828
3608
  data: rows
1829
3609
  }, null, 2);
1830
- if (opts.output) {
3610
+ if (opts.output && opts.output !== "-") {
1831
3611
  await fs.writeFile(opts.output, payload);
1832
3612
  if (!opts.quiet) logger.info(`Written to ${opts.output}`);
1833
3613
  } else console.log(payload);
1834
3614
  }
1835
3615
  async function writeOutput(opts) {
1836
3616
  const content = opts.format === "csv" ? exportToCSV(opts.output) : JSON.stringify(opts.output, null, 2);
1837
- if (opts.path) {
3617
+ if (opts.path && opts.path !== "-") {
1838
3618
  await fs.writeFile(opts.path, content);
1839
3619
  if (!opts.quiet) logger.info(`Written to ${opts.path}`);
1840
3620
  } else console.log(content);
@@ -1842,13 +3622,6 @@ async function writeOutput(opts) {
1842
3622
  function isKnownTable$1(name) {
1843
3623
  return allTables().includes(name);
1844
3624
  }
1845
- function requireSite(target) {
1846
- if (!target) {
1847
- logger.error("Site URL required (-s)");
1848
- process.exit(1);
1849
- }
1850
- return target;
1851
- }
1852
3625
  const sitemapsCommand = defineCommand({
1853
3626
  meta: {
1854
3627
  name: "sitemaps",
@@ -1870,160 +3643,480 @@ const sitemapsCommand = defineCommand({
1870
3643
  type: "boolean",
1871
3644
  default: false,
1872
3645
  description: "Output as JSON"
3646
+ },
3647
+ pending: {
3648
+ type: "boolean",
3649
+ default: false,
3650
+ description: "Show only sitemaps with isPending=true"
3651
+ },
3652
+ errored: {
3653
+ type: "boolean",
3654
+ default: false,
3655
+ description: "Show only sitemaps with errors > 0"
3656
+ }
3657
+ },
3658
+ async run({ args }) {
3659
+ const ctx = await createCommandContext({ needsAuth: true });
3660
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
3661
+ let sitemaps = (await ctx.client.sitemaps.list(siteUrl).catch((e) => {
3662
+ logger.error(`Failed to fetch sitemaps: ${e.message}`);
3663
+ process.exit(1);
3664
+ })).map((sm) => ({
3665
+ path: sm.path,
3666
+ type: sm.type || void 0,
3667
+ isPending: sm.isPending || false,
3668
+ errors: Number(sm.errors) || 0,
3669
+ warnings: Number(sm.warnings) || 0,
3670
+ lastDownloaded: sm.lastDownloaded || null,
3671
+ lastSubmitted: sm.lastSubmitted || null
3672
+ }));
3673
+ if (args.pending) sitemaps = sitemaps.filter((sm) => sm.isPending);
3674
+ if (args.errored) sitemaps = sitemaps.filter((sm) => sm.errors > 0);
3675
+ if (args.json) {
3676
+ console.log(JSON.stringify(sitemaps, null, 2));
3677
+ return;
3678
+ }
3679
+ if (sitemaps.length === 0) {
3680
+ logger.warn("No sitemaps found");
3681
+ return;
3682
+ }
3683
+ logger.success(`Found ${sitemaps.length} sitemaps:`);
3684
+ console.log();
3685
+ for (const sm of sitemaps) {
3686
+ const pending = sm.isPending ? " \x1B[33m(pending)\x1B[0m" : "";
3687
+ const errors = sm.errors ? ` \x1B[31m${sm.errors} errors\x1B[0m` : "";
3688
+ const warnings = sm.warnings ? ` \x1B[33m${sm.warnings} warnings\x1B[0m` : "";
3689
+ const submitted = sm.lastSubmitted ? ` \x1B[90msubmitted ${sm.lastSubmitted}\x1B[0m` : "";
3690
+ console.log(` ${sm.path}${pending}${errors}${warnings}${submitted}`);
3691
+ }
3692
+ }
3693
+ }),
3694
+ get: defineCommand({
3695
+ meta: {
3696
+ name: "get",
3697
+ description: "Get details for a specific sitemap"
3698
+ },
3699
+ args: {
3700
+ site: {
3701
+ type: "string",
3702
+ alias: "s",
3703
+ description: "Site URL (defaults to config.defaultSite or prompt)"
3704
+ },
3705
+ url: {
3706
+ type: "positional",
3707
+ required: true,
3708
+ description: "Sitemap URL"
3709
+ },
3710
+ json: {
3711
+ type: "boolean",
3712
+ default: false,
3713
+ description: "Output as JSON"
3714
+ }
3715
+ },
3716
+ async run({ args }) {
3717
+ const ctx = await createCommandContext({ needsAuth: true });
3718
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
3719
+ const client = ctx.client;
3720
+ const sitemap = await fetchSitemap(client, siteUrl, args.url).catch(gscErrorHandler);
3721
+ if (args.json) {
3722
+ console.log(JSON.stringify(sitemap, null, 2));
3723
+ return;
3724
+ }
3725
+ console.log();
3726
+ console.log(` \x1B[1mPath:\x1B[0m ${sitemap.path}`);
3727
+ console.log(` \x1B[1mType:\x1B[0m ${sitemap.type || "sitemap"}`);
3728
+ console.log(` \x1B[1mLast Submitted:\x1B[0m ${sitemap.lastSubmitted || "N/A"}`);
3729
+ console.log(` \x1B[1mLast Downloaded:\x1B[0m ${sitemap.lastDownloaded || "N/A"}`);
3730
+ console.log(` \x1B[1mPending:\x1B[0m ${sitemap.isPending ? "Yes" : "No"}`);
3731
+ console.log(` \x1B[1mErrors:\x1B[0m ${sitemap.errors || 0}`);
3732
+ console.log(` \x1B[1mWarnings:\x1B[0m ${sitemap.warnings || 0}`);
3733
+ if (sitemap.contents?.length) {
3734
+ console.log();
3735
+ console.log(" \x1B[1mContents:\x1B[0m");
3736
+ for (const c of sitemap.contents) console.log(` ${c.type}: ${c.submitted} submitted, ${c.indexed} indexed`);
3737
+ }
3738
+ }
3739
+ }),
3740
+ submit: defineCommand({
3741
+ meta: {
3742
+ name: "submit",
3743
+ description: "Submit a sitemap to GSC"
3744
+ },
3745
+ args: {
3746
+ site: {
3747
+ type: "string",
3748
+ alias: "s",
3749
+ description: "Site URL (defaults to config.defaultSite or prompt)"
3750
+ },
3751
+ url: {
3752
+ type: "positional",
3753
+ required: true,
3754
+ description: "Sitemap URL to submit"
3755
+ }
3756
+ },
3757
+ async run({ args }) {
3758
+ const ctx = await createCommandContext({ needsAuth: true });
3759
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
3760
+ await ctx.client.sitemaps.submit(siteUrl, args.url).catch((e) => {
3761
+ logger.error(`Submit failed: ${e.message}`);
3762
+ process.exit(1);
3763
+ });
3764
+ logger.success(`Submitted sitemap: ${args.url}`);
3765
+ }
3766
+ }),
3767
+ delete: defineCommand({
3768
+ meta: {
3769
+ name: "delete",
3770
+ description: "Delete a sitemap from GSC"
3771
+ },
3772
+ args: {
3773
+ site: {
3774
+ type: "string",
3775
+ alias: "s",
3776
+ description: "Site URL (defaults to config.defaultSite or prompt)"
3777
+ },
3778
+ url: {
3779
+ type: "positional",
3780
+ required: true,
3781
+ description: "Sitemap URL to delete"
3782
+ }
3783
+ },
3784
+ async run({ args }) {
3785
+ const ctx = await createCommandContext({ needsAuth: true });
3786
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
3787
+ await ctx.client.sitemaps.delete(siteUrl, args.url).catch((e) => {
3788
+ logger.error(`Delete failed: ${e.message}`);
3789
+ process.exit(1);
3790
+ });
3791
+ logger.success(`Deleted sitemap: ${args.url}`);
3792
+ }
3793
+ })
3794
+ }
3795
+ });
3796
+ const ALL_METHODS = [
3797
+ "META",
3798
+ "FILE",
3799
+ "DNS_TXT",
3800
+ "DNS_CNAME",
3801
+ "ANALYTICS",
3802
+ "TAG_MANAGER"
3803
+ ];
3804
+ function pickDefaultMethod(siteUrl) {
3805
+ return siteUrl.startsWith("sc-domain:") ? "DNS_TXT" : "META";
3806
+ }
3807
+ function validateMethod(siteUrl, method) {
3808
+ const upper = method.toUpperCase();
3809
+ if (!ALL_METHODS.includes(upper)) {
3810
+ logger.error(`Invalid --method: ${method}. Valid: ${ALL_METHODS.join(", ")}`);
3811
+ process.exit(1);
3812
+ }
3813
+ const site = siteUrlToVerificationSite(siteUrl);
3814
+ const allowed = verificationMethodsFor(site);
3815
+ if (!allowed.includes(upper)) {
3816
+ logger.error(`Method ${upper} not valid for ${site.type === "INET_DOMAIN" ? "domain" : "URL-prefix"} property "${siteUrl}". Valid: ${allowed.join(", ")}`);
3817
+ process.exit(1);
3818
+ }
3819
+ return upper;
3820
+ }
3821
+ function printPlacementInstructions(method, siteUrl, token) {
3822
+ console.log();
3823
+ console.log(` \x1B[1mPlacement instructions (${method})\x1B[0m`);
3824
+ console.log();
3825
+ switch (method) {
3826
+ case "META":
3827
+ console.log(` Add this tag inside the <head> of \x1B[36m${siteUrl}\x1B[0m:`);
3828
+ console.log();
3829
+ console.log(` \x1B[2m<meta name="google-site-verification" content="${token}" />\x1B[0m`);
3830
+ break;
3831
+ case "FILE":
3832
+ console.log(` Upload a file named \x1B[1m${token}\x1B[0m to the site root, accessible at:`);
3833
+ console.log();
3834
+ console.log(` \x1B[36m${siteUrl.replace(/\/?$/, "/")}${token}\x1B[0m`);
3835
+ break;
3836
+ case "DNS_TXT":
3837
+ console.log(` Add a TXT record on \x1B[1m${siteUrl.replace(/^sc-domain:/, "")}\x1B[0m with value:`);
3838
+ console.log();
3839
+ console.log(` \x1B[2m${token}\x1B[0m`);
3840
+ break;
3841
+ case "DNS_CNAME": {
3842
+ const [host, target] = token.split(/\s+/, 2);
3843
+ console.log(` Add a CNAME record:`);
3844
+ console.log();
3845
+ console.log(` \x1B[2mHost: ${host}\x1B[0m`);
3846
+ console.log(` \x1B[2mTarget: ${target ?? "(see token)"}\x1B[0m`);
3847
+ break;
3848
+ }
3849
+ case "ANALYTICS":
3850
+ console.log(` Make sure your Google Analytics tracking tag is installed on the site, then run \`gscdump sites verify\`.`);
3851
+ break;
3852
+ case "TAG_MANAGER":
3853
+ console.log(` Make sure your Google Tag Manager container snippet is installed on the site, then run \`gscdump sites verify\`.`);
3854
+ break;
3855
+ }
3856
+ console.log();
3857
+ console.log(` \x1B[90mThen run:\x1B[0m gscdump sites verify ${siteUrl} --method ${method}`);
3858
+ console.log();
3859
+ }
3860
+ const sitesCommand = defineCommand({
3861
+ meta: {
3862
+ name: "sites",
3863
+ description: "List GSC sites; manage properties (add/delete) and verify ownership"
3864
+ },
3865
+ args: {
3866
+ "json": {
3867
+ type: "boolean",
3868
+ default: false,
3869
+ description: "Output as JSON for scripting"
3870
+ },
3871
+ "with-sitemaps": {
3872
+ type: "boolean",
3873
+ default: false,
3874
+ description: "Include sitemaps for each owned site"
3875
+ },
3876
+ "owner-only": {
3877
+ type: "boolean",
3878
+ default: false,
3879
+ description: "Filter to permissionLevel=siteOwner"
3880
+ },
3881
+ "quiet": {
3882
+ type: "boolean",
3883
+ alias: "q",
3884
+ default: false,
3885
+ description: "Suppress info/success output"
3886
+ }
3887
+ },
3888
+ subCommands: {
3889
+ "add": defineCommand({
3890
+ meta: {
3891
+ name: "add",
3892
+ description: "Register a property in Search Console (unverified state — verify ownership separately)"
3893
+ },
3894
+ args: {
3895
+ url: {
3896
+ type: "positional",
3897
+ required: true,
3898
+ description: "Property URL (https://example.com/ or sc-domain:example.com)"
3899
+ },
3900
+ json: {
3901
+ type: "boolean",
3902
+ default: false,
3903
+ description: "Output as JSON"
3904
+ },
3905
+ quiet: {
3906
+ type: "boolean",
3907
+ alias: "q",
3908
+ default: false,
3909
+ description: "Suppress info/success output"
1873
3910
  }
1874
3911
  },
1875
3912
  async run({ args }) {
1876
- const config = await loadConfig();
1877
- const siteUrl = requireSite(args.site || config.defaultSite);
1878
- const sitemaps = (await (await createCommandContext({ needsAuth: true })).client.sitemaps.list(siteUrl).catch((e) => {
1879
- logger.error(`Failed to fetch sitemaps: ${e.message}`);
1880
- process.exit(1);
1881
- })).map((sm) => ({
1882
- path: sm.path,
1883
- type: sm.type || void 0,
1884
- isPending: sm.isPending || false,
1885
- errors: Number(sm.errors) || 0,
1886
- warnings: Number(sm.warnings) || 0,
1887
- lastDownloaded: sm.lastDownloaded || null
1888
- }));
3913
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
3914
+ await addSite((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
1889
3915
  if (args.json) {
1890
- console.log(JSON.stringify(sitemaps, null, 2));
1891
- return;
1892
- }
1893
- if (sitemaps.length === 0) {
1894
- logger.warn("No sitemaps found");
3916
+ console.log(JSON.stringify({
3917
+ siteUrl: args.url,
3918
+ status: "added",
3919
+ verified: false
3920
+ }, null, 2));
1895
3921
  return;
1896
3922
  }
1897
- logger.success(`Found ${sitemaps.length} sitemaps:`);
1898
- console.log();
1899
- for (const sm of sitemaps) {
1900
- const pending = sm.isPending ? " \x1B[33m(pending)\x1B[0m" : "";
1901
- const errors = sm.errors ? ` \x1B[31m${sm.errors} errors\x1B[0m` : "";
1902
- const warnings = sm.warnings ? ` \x1B[33m${sm.warnings} warnings\x1B[0m` : "";
1903
- console.log(` ${sm.path}${pending}${errors}${warnings}`);
1904
- }
3923
+ logger.success(`Added: ${args.url}`);
3924
+ logger.info(`Property is in unverified state. Verify ownership next:`);
3925
+ const method = pickDefaultMethod(args.url);
3926
+ console.log(` \x1B[2mgscdump sites verify-token ${args.url} --method ${method}\x1B[0m`);
1905
3927
  }
1906
3928
  }),
1907
- get: defineCommand({
3929
+ "delete": defineCommand({
1908
3930
  meta: {
1909
- name: "get",
1910
- description: "Get details for a specific sitemap"
3931
+ name: "delete",
3932
+ description: "Remove a property from Search Console"
1911
3933
  },
1912
3934
  args: {
1913
- site: {
1914
- type: "string",
1915
- alias: "s",
1916
- required: true,
1917
- description: "Site URL"
1918
- },
1919
3935
  url: {
1920
3936
  type: "positional",
1921
3937
  required: true,
1922
- description: "Sitemap URL"
3938
+ description: "Property URL to remove"
3939
+ },
3940
+ yes: {
3941
+ type: "boolean",
3942
+ alias: "y",
3943
+ default: false,
3944
+ description: "Skip confirmation prompt"
1923
3945
  },
1924
3946
  json: {
1925
3947
  type: "boolean",
1926
3948
  default: false,
1927
3949
  description: "Output as JSON"
3950
+ },
3951
+ quiet: {
3952
+ type: "boolean",
3953
+ alias: "q",
3954
+ default: false,
3955
+ description: "Suppress info/success output"
1928
3956
  }
1929
3957
  },
1930
3958
  async run({ args }) {
1931
- const client = (await createCommandContext({ needsAuth: true })).client;
1932
- const sitemap = await fetchSitemap(client, args.site, args.url).catch(gscErrorHandler);
3959
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
3960
+ if (!args.yes && !args.json) {
3961
+ const ok = await confirm({
3962
+ message: `Remove ${args.url} from Search Console? Local synced data is unaffected.`,
3963
+ initialValue: false
3964
+ });
3965
+ if (isCancel(ok) || !ok) {
3966
+ logger.info("Cancelled");
3967
+ process.exit(0);
3968
+ }
3969
+ }
3970
+ await deleteSite((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
1933
3971
  if (args.json) {
1934
- console.log(JSON.stringify(sitemap, null, 2));
3972
+ console.log(JSON.stringify({
3973
+ siteUrl: args.url,
3974
+ status: "deleted"
3975
+ }, null, 2));
1935
3976
  return;
1936
3977
  }
1937
- console.log();
1938
- console.log(` \x1B[1mPath:\x1B[0m ${sitemap.path}`);
1939
- console.log(` \x1B[1mType:\x1B[0m ${sitemap.type || "sitemap"}`);
1940
- console.log(` \x1B[1mLast Submitted:\x1B[0m ${sitemap.lastSubmitted || "N/A"}`);
1941
- console.log(` \x1B[1mLast Downloaded:\x1B[0m ${sitemap.lastDownloaded || "N/A"}`);
1942
- console.log(` \x1B[1mPending:\x1B[0m ${sitemap.isPending ? "Yes" : "No"}`);
1943
- console.log(` \x1B[1mErrors:\x1B[0m ${sitemap.errors || 0}`);
1944
- console.log(` \x1B[1mWarnings:\x1B[0m ${sitemap.warnings || 0}`);
1945
- if (sitemap.contents?.length) {
1946
- console.log();
1947
- console.log(" \x1B[1mContents:\x1B[0m");
1948
- for (const c of sitemap.contents) console.log(` ${c.type}: ${c.submitted} submitted, ${c.indexed} indexed`);
1949
- }
3978
+ logger.success(`Removed: ${args.url}`);
1950
3979
  }
1951
3980
  }),
1952
- submit: defineCommand({
3981
+ "verify-token": defineCommand({
1953
3982
  meta: {
1954
- name: "submit",
1955
- description: "Submit a sitemap to GSC"
3983
+ name: "verify-token",
3984
+ description: "Get a verification token to place on the site or in DNS"
1956
3985
  },
1957
3986
  args: {
1958
- site: {
1959
- type: "string",
1960
- alias: "s",
1961
- required: true,
1962
- description: "Site URL"
1963
- },
1964
3987
  url: {
1965
3988
  type: "positional",
1966
3989
  required: true,
1967
- description: "Sitemap URL to submit"
3990
+ description: "Property URL"
3991
+ },
3992
+ method: {
3993
+ type: "string",
3994
+ alias: "m",
3995
+ description: "META, FILE, DNS_TXT, DNS_CNAME, ANALYTICS, TAG_MANAGER (default: META for URL-prefix, DNS_TXT for sc-domain:)"
3996
+ },
3997
+ json: {
3998
+ type: "boolean",
3999
+ default: false,
4000
+ description: "Output as JSON"
4001
+ },
4002
+ quiet: {
4003
+ type: "boolean",
4004
+ alias: "q",
4005
+ default: false,
4006
+ description: "Suppress info/success output"
1968
4007
  }
1969
4008
  },
1970
4009
  async run({ args }) {
1971
- await (await createCommandContext({ needsAuth: true })).client.sitemaps.submit(args.site, args.url).catch((e) => {
1972
- logger.error(`Submit failed: ${e.message}`);
1973
- process.exit(1);
1974
- });
1975
- logger.success(`Submitted sitemap: ${args.url}`);
4010
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
4011
+ const method = validateMethod(args.url, args.method ?? pickDefaultMethod(args.url));
4012
+ const result = await getVerificationToken((await createCommandContext({ needsAuth: true })).client, args.url, method).catch(gscErrorHandler);
4013
+ if (args.json) {
4014
+ console.log(JSON.stringify({
4015
+ siteUrl: args.url,
4016
+ method,
4017
+ token: result.token,
4018
+ site: result.site
4019
+ }, null, 2));
4020
+ return;
4021
+ }
4022
+ printPlacementInstructions(method, args.url, result.token);
1976
4023
  }
1977
4024
  }),
1978
- delete: defineCommand({
4025
+ "verify": defineCommand({
1979
4026
  meta: {
1980
- name: "delete",
1981
- description: "Delete a sitemap from GSC"
4027
+ name: "verify",
4028
+ description: "Trigger verification Google fetches/validates the token you placed"
1982
4029
  },
1983
4030
  args: {
1984
- site: {
1985
- type: "string",
1986
- alias: "s",
1987
- required: true,
1988
- description: "Site URL"
1989
- },
1990
4031
  url: {
1991
4032
  type: "positional",
1992
4033
  required: true,
1993
- description: "Sitemap URL to delete"
4034
+ description: "Property URL"
4035
+ },
4036
+ method: {
4037
+ type: "string",
4038
+ alias: "m",
4039
+ description: "Verification method to validate (must match the one used for verify-token)"
4040
+ },
4041
+ json: {
4042
+ type: "boolean",
4043
+ default: false,
4044
+ description: "Output as JSON"
4045
+ },
4046
+ quiet: {
4047
+ type: "boolean",
4048
+ alias: "q",
4049
+ default: false,
4050
+ description: "Suppress info/success output"
1994
4051
  }
1995
4052
  },
1996
4053
  async run({ args }) {
1997
- await (await createCommandContext({ needsAuth: true })).client.sitemaps.delete(args.site, args.url).catch((e) => {
1998
- logger.error(`Delete failed: ${e.message}`);
1999
- process.exit(1);
2000
- });
2001
- logger.success(`Deleted sitemap: ${args.url}`);
4054
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
4055
+ const method = validateMethod(args.url, args.method ?? pickDefaultMethod(args.url));
4056
+ const resource = await verifySite((await createCommandContext({ needsAuth: true })).client, args.url, method).catch(gscErrorHandler);
4057
+ if (args.json) {
4058
+ console.log(JSON.stringify({
4059
+ siteUrl: args.url,
4060
+ method,
4061
+ resource
4062
+ }, null, 2));
4063
+ return;
4064
+ }
4065
+ logger.success(`Verified: ${args.url}`);
4066
+ if (resource.owners?.length) {
4067
+ console.log();
4068
+ console.log(` Owners:`);
4069
+ for (const o of resource.owners) console.log(` \x1B[90m└─\x1B[0m ${o}`);
4070
+ }
2002
4071
  }
2003
4072
  })
2004
- }
2005
- });
2006
- const sitesCommand = defineCommand({
2007
- meta: {
2008
- name: "sites",
2009
- description: "List available GSC sites"
2010
4073
  },
2011
- args: { json: {
2012
- type: "boolean",
2013
- default: false,
2014
- description: "Output as JSON for scripting"
2015
- } },
2016
4074
  async run({ args }) {
2017
- const sites = await (await createCommandContext({ needsAuth: true })).loadSites();
4075
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
4076
+ const ctx = await createCommandContext({ needsAuth: true });
4077
+ const ownerOnly = Boolean(args["owner-only"]);
4078
+ if (args["with-sitemaps"]) {
4079
+ const all = await fetchSitesWithSitemaps(ctx.client).catch(gscErrorHandler);
4080
+ const sites = ownerOnly ? all.filter((s) => s.permissionLevel === "siteOwner") : all;
4081
+ if (args.json) {
4082
+ const enriched = sites.map((s) => ({
4083
+ ...s,
4084
+ sitemapCounts: {
4085
+ total: s.sitemaps.length,
4086
+ pending: s.sitemaps.filter((sm) => sm.isPending).length,
4087
+ errored: s.sitemaps.filter((sm) => Number(sm.errors) > 0).length
4088
+ }
4089
+ }));
4090
+ console.log(JSON.stringify(enriched, null, 2));
4091
+ return;
4092
+ }
4093
+ if (sites.length === 0) {
4094
+ logger.warn(ownerOnly ? "No owned sites found" : "No verified sites found");
4095
+ return;
4096
+ }
4097
+ logger.success(`Found ${sites.length} ${ownerOnly ? "owned" : "verified"} sites:`);
4098
+ console.log();
4099
+ for (const site of sites) {
4100
+ const perm = site.permissionLevel === "siteOwner" ? "\x1B[32m" : "\x1B[90m";
4101
+ console.log(` ${site.siteUrl} ${perm}(${site.permissionLevel})\x1B[0m`);
4102
+ for (const sm of site.sitemaps) {
4103
+ const pending = sm.isPending ? " \x1B[33m(pending)\x1B[0m" : "";
4104
+ console.log(` \x1B[90m└─\x1B[0m ${sm.path}${pending}`);
4105
+ }
4106
+ }
4107
+ return;
4108
+ }
4109
+ const all = await ctx.loadSites();
4110
+ const sites = ownerOnly ? all.filter((s) => s.permissionLevel === "siteOwner") : all;
2018
4111
  if (args.json) {
2019
4112
  console.log(JSON.stringify(sites, null, 2));
2020
4113
  return;
2021
4114
  }
2022
4115
  if (sites.length === 0) {
2023
- logger.warn("No verified sites found");
4116
+ logger.warn(ownerOnly ? "No owned sites found" : "No verified sites found");
2024
4117
  return;
2025
4118
  }
2026
- logger.success(`Found ${sites.length} sites:`);
4119
+ logger.success(`Found ${sites.length} ${ownerOnly ? "owned " : ""}sites:`);
2027
4120
  console.log();
2028
4121
  for (const site of sites) {
2029
4122
  const perm = site.permissionLevel === "siteOwner" ? "\x1B[32m" : "\x1B[90m";
@@ -2054,6 +4147,16 @@ const compactCommand = defineCommand({
2054
4147
  type: "string",
2055
4148
  description: "Override d30→d90 age threshold in days (default: 90)"
2056
4149
  },
4150
+ "dry-run": {
4151
+ type: "boolean",
4152
+ default: false,
4153
+ description: "Report tier counts per (table, site) without compacting"
4154
+ },
4155
+ "json": {
4156
+ type: "boolean",
4157
+ default: false,
4158
+ description: "Output a JSON summary"
4159
+ },
2057
4160
  "quiet": {
2058
4161
  type: "boolean",
2059
4162
  alias: "q",
@@ -2062,13 +4165,43 @@ const compactCommand = defineCommand({
2062
4165
  }
2063
4166
  },
2064
4167
  async run({ args }) {
4168
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2065
4169
  const store = (await createCommandContext({ needsStore: true })).store;
2066
4170
  const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
2067
- const quiet = Boolean(args.quiet);
4171
+ const dryRun = Boolean(args["dry-run"]);
2068
4172
  const thresholds = {};
2069
4173
  if (args["raw-days"]) thresholds.raw = Number(args["raw-days"]);
2070
4174
  if (args["d7-days"]) thresholds.d7 = Number(args["d7-days"]);
2071
4175
  if (args["d30-days"]) thresholds.d30 = Number(args["d30-days"]);
4176
+ if (dryRun) {
4177
+ const report = [];
4178
+ for (const table of allTables()) {
4179
+ const bySite = groupBySite(await store.engine.listLive({
4180
+ userId: store.userId,
4181
+ siteId,
4182
+ table
4183
+ }));
4184
+ for (const [s, group] of bySite) report.push({
4185
+ table,
4186
+ siteId: s,
4187
+ ...countByTier(group)
4188
+ });
4189
+ }
4190
+ if (args.json) {
4191
+ console.log(JSON.stringify({
4192
+ thresholds,
4193
+ plan: report
4194
+ }, null, 2));
4195
+ return;
4196
+ }
4197
+ console.log();
4198
+ console.log(` table site raw d7 d30 d90`);
4199
+ for (const r of report) console.log(` ${r.table.padEnd(20)} ${(r.siteId ?? "-").padEnd(20)} ${String(r.raw).padStart(4)} ${String(r.d7).padStart(4)} ${String(r.d30).padStart(4)} ${String(r.d90).padStart(4)}`);
4200
+ console.log();
4201
+ logger.info(`compact --dry-run: ${report.length} (table, site) pair(s) — pass without --dry-run to apply`);
4202
+ return;
4203
+ }
4204
+ const summary = [];
2072
4205
  for (const table of allTables()) {
2073
4206
  const entries = await store.engine.listLive({
2074
4207
  userId: store.userId,
@@ -2077,17 +4210,56 @@ const compactCommand = defineCommand({
2077
4210
  });
2078
4211
  const siteIds = new Set(entries.map((e) => e.siteId));
2079
4212
  for (const targetSite of siteIds) {
2080
- if (!quiet) logger.info(`Compacting ${table} [${targetSite ?? "-"}] (raw→d7→d30→d90)`);
4213
+ logger.info(`Compacting ${table} [${targetSite ?? "-"}] (raw→d7→d30→d90)`);
2081
4214
  await store.engine.compactTiered({
2082
4215
  userId: store.userId,
2083
4216
  siteId: targetSite,
2084
4217
  table
2085
4218
  }, thresholds);
4219
+ summary.push({
4220
+ table,
4221
+ siteId: targetSite
4222
+ });
2086
4223
  }
2087
4224
  }
2088
- if (!quiet) logger.success(`compact: done`);
4225
+ if (args.json) {
4226
+ console.log(JSON.stringify({
4227
+ thresholds,
4228
+ compacted: summary
4229
+ }, null, 2));
4230
+ return;
4231
+ }
4232
+ logger.success(`compact: done`);
2089
4233
  }
2090
4234
  });
4235
+ function groupBySite(entries) {
4236
+ const m = /* @__PURE__ */ new Map();
4237
+ for (const e of entries) {
4238
+ const arr = m.get(e.siteId) ?? [];
4239
+ arr.push(e);
4240
+ m.set(e.siteId, arr);
4241
+ }
4242
+ return m;
4243
+ }
4244
+ function countByTier(entries) {
4245
+ let raw = 0;
4246
+ let d7 = 0;
4247
+ let d30 = 0;
4248
+ let d90 = 0;
4249
+ for (const e of entries) {
4250
+ const tier = e.tier ?? inferLegacyTier(e) ?? "raw";
4251
+ if (tier === "raw") raw++;
4252
+ else if (tier === "d7") d7++;
4253
+ else if (tier === "d30") d30++;
4254
+ else if (tier === "d90") d90++;
4255
+ }
4256
+ return {
4257
+ raw,
4258
+ d7,
4259
+ d30,
4260
+ d90
4261
+ };
4262
+ }
2091
4263
  async function exportToDuckDB(opts) {
2092
4264
  const outPath = path.resolve(opts.outPath);
2093
4265
  if (opts.force) await rm(outPath, { force: true });
@@ -2141,9 +4313,21 @@ const exportCommand = defineCommand({
2141
4313
  type: "boolean",
2142
4314
  default: false,
2143
4315
  description: "Overwrite the output file if it already exists"
4316
+ },
4317
+ json: {
4318
+ type: "boolean",
4319
+ default: false,
4320
+ description: "Output a JSON summary instead of formatted text"
4321
+ },
4322
+ quiet: {
4323
+ type: "boolean",
4324
+ alias: "q",
4325
+ default: false,
4326
+ description: "Suppress info/success output"
2144
4327
  }
2145
4328
  },
2146
4329
  async run({ args }) {
4330
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2147
4331
  const store = (await createCommandContext({ needsStore: true })).store;
2148
4332
  const siteId = args.site ? store.siteIdFor(args.site) : void 0;
2149
4333
  const result = await exportToDuckDB({
@@ -2154,12 +4338,16 @@ const exportCommand = defineCommand({
2154
4338
  outPath: args.out,
2155
4339
  force: args.force
2156
4340
  });
4341
+ if (args.json) {
4342
+ console.log(JSON.stringify(result, null, 2));
4343
+ return;
4344
+ }
2157
4345
  if (result.tables.length === 0) {
2158
4346
  console.log(`\n No data to export. Run \`gscdump sync\` first.`);
2159
4347
  return;
2160
4348
  }
2161
4349
  for (const t of result.tables) console.log(` ${t.table.padEnd(15)} ${String(t.files).padStart(4)} parquet → ${t.table} (${t.rows.toLocaleString()} rows)`);
2162
- console.log(`\n Exported ${result.tables.length} table(s), ${result.totalRows.toLocaleString()} rows → ${result.outPath}`);
4350
+ console.log(`\n Exported ${result.tables.length} table(s), ${result.totalRows.toLocaleString()} rows → ${displayPath(result.outPath)}`);
2163
4351
  console.log(`\n Attach from DuckDB: \x1B[36mATTACH '${result.outPath}' AS gsc (READ_ONLY); SELECT * FROM gsc.pages LIMIT 10;\x1B[0m`);
2164
4352
  console.log(` Attach in a browser: use DuckDB-WASM registerFileBuffer + \x1B[36mATTACH 'gsc.duckdb' AS gsc (READ_ONLY)\x1B[0m`);
2165
4353
  }
@@ -2181,6 +4369,16 @@ const gcCommand = defineCommand({
2181
4369
  alias: "s",
2182
4370
  description: "Restrict to a single site (default: all sites)"
2183
4371
  },
4372
+ "dry-run": {
4373
+ type: "boolean",
4374
+ default: false,
4375
+ description: "List retired manifest entries past the grace window without deleting"
4376
+ },
4377
+ "json": {
4378
+ type: "boolean",
4379
+ default: false,
4380
+ description: "Output a JSON summary"
4381
+ },
2184
4382
  "quiet": {
2185
4383
  type: "boolean",
2186
4384
  alias: "q",
@@ -2189,15 +4387,52 @@ const gcCommand = defineCommand({
2189
4387
  }
2190
4388
  },
2191
4389
  async run({ args }) {
4390
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2192
4391
  const store = (await createCommandContext({ needsStore: true })).store;
2193
4392
  const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
2194
- const quiet = Boolean(args.quiet);
2195
4393
  const graceMs = Number(args["grace-hours"]) * 36e5;
4394
+ if (args["dry-run"]) {
4395
+ const cutoff = Date.now() - graceMs;
4396
+ const candidates = [];
4397
+ for (const table of allTables()) {
4398
+ const all = await store.engine.listAll({
4399
+ userId: store.userId,
4400
+ siteId,
4401
+ table
4402
+ });
4403
+ for (const e of all) if (e.retiredAt && e.retiredAt < cutoff) candidates.push({
4404
+ table,
4405
+ siteId: e.siteId,
4406
+ partition: e.partition,
4407
+ retiredAt: e.retiredAt,
4408
+ objectKey: e.objectKey
4409
+ });
4410
+ }
4411
+ if (args.json) {
4412
+ console.log(JSON.stringify({
4413
+ graceHours: Number(args["grace-hours"]),
4414
+ candidates
4415
+ }, null, 2));
4416
+ return;
4417
+ }
4418
+ console.log();
4419
+ for (const c of candidates) console.log(` ${c.objectKey} \x1B[90m(retired ${new Date(c.retiredAt).toISOString()})\x1B[0m`);
4420
+ console.log();
4421
+ logger.info(`gc --dry-run: ${candidates.length} retired manifest entry(ies) past ${args["grace-hours"]}h grace; pass without --dry-run to delete`);
4422
+ return;
4423
+ }
2196
4424
  const result = await store.engine.gcOrphans({
2197
4425
  userId: store.userId,
2198
4426
  siteId
2199
4427
  }, graceMs);
2200
- if (!quiet) logger.success(`gc: deleted ${result.deleted} orphan file(s)`);
4428
+ if (args.json) {
4429
+ console.log(JSON.stringify({
4430
+ graceHours: Number(args["grace-hours"]),
4431
+ deleted: result.deleted
4432
+ }, null, 2));
4433
+ return;
4434
+ }
4435
+ logger.success(`gc: deleted ${result.deleted} orphan file(s)`);
2201
4436
  }
2202
4437
  });
2203
4438
  const rollupsCommand = defineCommand({
@@ -2216,6 +4451,11 @@ const rollupsCommand = defineCommand({
2216
4451
  alias: "s",
2217
4452
  description: "Restrict to a single site (default: all sites with local data)"
2218
4453
  },
4454
+ json: {
4455
+ type: "boolean",
4456
+ default: false,
4457
+ description: "Output a JSON summary"
4458
+ },
2219
4459
  quiet: {
2220
4460
  type: "boolean",
2221
4461
  alias: "q",
@@ -2224,9 +4464,9 @@ const rollupsCommand = defineCommand({
2224
4464
  }
2225
4465
  },
2226
4466
  async run({ args }) {
4467
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2227
4468
  const store = (await createCommandContext({ needsStore: true })).store;
2228
4469
  const explicitSiteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
2229
- const quiet = Boolean(args.quiet);
2230
4470
  const allSiteIds = /* @__PURE__ */ new Set();
2231
4471
  if (explicitSiteId) allSiteIds.add(explicitSiteId);
2232
4472
  else for (const table of allTables()) {
@@ -2237,12 +4477,17 @@ const rollupsCommand = defineCommand({
2237
4477
  for (const e of entries) if (e.siteId) allSiteIds.add(e.siteId);
2238
4478
  }
2239
4479
  if (allSiteIds.size === 0) {
2240
- logger.warn("No sites with local data. Run `gscdump sync` first.");
4480
+ if (args.json) console.log(JSON.stringify({
4481
+ sites: [],
4482
+ totalBytes: 0
4483
+ }, null, 2));
4484
+ else logger.warn("No sites with local data. Run `gscdump sync` first.");
2241
4485
  return;
2242
4486
  }
4487
+ const summary = [];
2243
4488
  let totalBytes = 0;
2244
4489
  for (const siteId of allSiteIds) {
2245
- if (!quiet) logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)`);
4490
+ logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)`);
2246
4491
  const results = await rebuildRollups({
2247
4492
  engine: store.engine,
2248
4493
  dataSource: store.dataSource,
@@ -2252,12 +4497,29 @@ const rollupsCommand = defineCommand({
2252
4497
  },
2253
4498
  defs: DEFAULT_ROLLUPS
2254
4499
  });
4500
+ const site = {
4501
+ siteId,
4502
+ rollups: []
4503
+ };
2255
4504
  for (const r of results) {
2256
4505
  totalBytes += r.bytes;
2257
- if (!quiet) console.log(` ${r.id.padEnd(20)} ${(r.bytes / 1024).toFixed(1).padStart(8)} KB ${r.objectKey}`);
4506
+ site.rollups.push({
4507
+ id: r.id,
4508
+ bytes: r.bytes,
4509
+ objectKey: r.objectKey
4510
+ });
4511
+ if (!args.json) console.log(` ${r.id.padEnd(20)} ${(r.bytes / 1024).toFixed(1).padStart(8)} KB ${r.objectKey}`);
2258
4512
  }
4513
+ summary.push(site);
4514
+ }
4515
+ if (args.json) {
4516
+ console.log(JSON.stringify({
4517
+ sites: summary,
4518
+ totalBytes
4519
+ }, null, 2));
4520
+ return;
2259
4521
  }
2260
- if (!quiet) logger.success(`Rebuilt rollups across ${allSiteIds.size} site(s) — total ${(totalBytes / 1024).toFixed(1)} KB`);
4522
+ logger.success(`Rebuilt rollups across ${allSiteIds.size} site(s) — total ${(totalBytes / 1024).toFixed(1)} KB`);
2261
4523
  }
2262
4524
  }) }
2263
4525
  });
@@ -2275,11 +4537,27 @@ const statsCommand = defineCommand({
2275
4537
  site: {
2276
4538
  type: "string",
2277
4539
  description: "Limit to one site URL (sc-domain:example.com, https://example.com/, ...)"
4540
+ },
4541
+ quiet: {
4542
+ type: "boolean",
4543
+ alias: "q",
4544
+ default: false,
4545
+ description: "Suppress info/success output"
2278
4546
  }
2279
4547
  },
2280
4548
  async run({ args }) {
4549
+ setQuiet(Boolean(args.quiet) || Boolean(args.json));
2281
4550
  const store = (await createCommandContext({ needsStore: true })).store;
2282
- const siteId = args.site ? store.siteIdFor(args.site) : void 0;
4551
+ let siteId;
4552
+ if (args.site) {
4553
+ const known = await listKnownSiteIds(store);
4554
+ const candidate = store.siteIdFor(args.site);
4555
+ if (!known.has(candidate)) {
4556
+ logger.error(`No local data for --site=${args.site}. Known site IDs: ${known.size === 0 ? "(none — run `gscdump sync` first)" : Array.from(known).join(", ")}`);
4557
+ process.exit(1);
4558
+ }
4559
+ siteId = candidate;
4560
+ }
2283
4561
  const perTable = await Promise.all(allTables().map(async (table) => {
2284
4562
  const all = await store.engine.listAll({
2285
4563
  userId: store.userId,
@@ -2323,7 +4601,7 @@ const statsCommand = defineCommand({
2323
4601
  return;
2324
4602
  }
2325
4603
  console.log();
2326
- console.log(` \x1B[1m${store.dataDir}\x1B[0m`);
4604
+ console.log(` \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
2327
4605
  console.log(` \x1B[90mDisk: ${disk.files} file(s), ${formatBytes(disk.bytes)}\x1B[0m`);
2328
4606
  console.log();
2329
4607
  const totalRows = perTable.reduce((acc, t) => acc + sumRows(t.live), 0);
@@ -2351,6 +4629,17 @@ const statsCommand = defineCommand({
2351
4629
  console.log();
2352
4630
  }
2353
4631
  });
4632
+ async function listKnownSiteIds(store) {
4633
+ const ids = /* @__PURE__ */ new Set();
4634
+ for (const table of allTables()) {
4635
+ const entries = await store.engine.listLive({
4636
+ userId: store.userId,
4637
+ table
4638
+ });
4639
+ for (const e of entries) if (e.siteId) ids.add(e.siteId);
4640
+ }
4641
+ return ids;
4642
+ }
2354
4643
  function sortWatermarks(ws) {
2355
4644
  return [...ws].sort((a, b) => {
2356
4645
  if (a.table !== b.table) return a.table.localeCompare(b.table);
@@ -2585,6 +4874,16 @@ const syncCommand = defineCommand({
2585
4874
  type: "boolean",
2586
4875
  default: false,
2587
4876
  description: "Run tables sequentially (default: run all tables in parallel)"
4877
+ },
4878
+ "retry-failed": {
4879
+ type: "boolean",
4880
+ default: false,
4881
+ description: "Only re-run dates currently in `failed` state (cheaper than --force)"
4882
+ },
4883
+ "dry-run": {
4884
+ type: "boolean",
4885
+ default: false,
4886
+ description: "Print the planned (table, searchType, date) work and exit without hitting the API"
2588
4887
  }
2589
4888
  },
2590
4889
  async run({ args }) {
@@ -2631,14 +4930,63 @@ const syncCommand = defineCommand({
2631
4930
  else if (args.full) startDate = daysAgo(450);
2632
4931
  else if (args.days) startDate = daysAgo(Number.parseInt(String(args.days), 10) + DEFAULT_PENDING_DAYS - 1);
2633
4932
  else startDate = daysAgo(DEFAULT_PENDING_DAYS + DEFAULT_PENDING_DAYS - 1);
2634
- const dates = getDateRange(startDate, endDate);
4933
+ let dates = getDateRange(startDate, endDate);
2635
4934
  if (dates.length === 0) {
2636
4935
  logger.error(`No dates to sync (start=${startDate}, end=${endDate})`);
2637
4936
  process.exit(1);
2638
4937
  }
2639
4938
  const store = ctx.store;
4939
+ if (args["retry-failed"]) {
4940
+ const failedSet = /* @__PURE__ */ new Set();
4941
+ for (const table of tables) for (const type of types) {
4942
+ const states = await store.engine.getSyncStates({
4943
+ userId: store.userId,
4944
+ siteId,
4945
+ table,
4946
+ searchType: type
4947
+ });
4948
+ for (const s of states) if (s.state === "failed" && s.date >= startDate && s.date <= endDate) failedSet.add(s.date);
4949
+ }
4950
+ dates = dates.filter((d) => failedSet.has(d));
4951
+ if (dates.length === 0) {
4952
+ logger.success("No failed dates in range — nothing to retry.");
4953
+ return;
4954
+ }
4955
+ args.force = true;
4956
+ if (!args.quiet) logger.info(`--retry-failed: ${dates.length} date(s) to retry`);
4957
+ }
4958
+ if (args["dry-run"]) {
4959
+ const plan = [];
4960
+ for (const table of tables) for (const type of types) for (const date of dates) plan.push({
4961
+ table,
4962
+ searchType: type,
4963
+ date
4964
+ });
4965
+ if (args.json) {
4966
+ console.log(JSON.stringify({
4967
+ siteUrl,
4968
+ range: {
4969
+ start: startDate,
4970
+ end: endDate
4971
+ },
4972
+ tables,
4973
+ types,
4974
+ totalCalls: plan.length,
4975
+ plan
4976
+ }, null, 2));
4977
+ return;
4978
+ }
4979
+ console.log();
4980
+ logger.info(`Plan: ${plan.length} API call(s) for ${siteUrl}`);
4981
+ console.log(` Tables: ${tables.join(", ")}`);
4982
+ console.log(` Types: ${types.join(", ")}`);
4983
+ console.log(` Range: ${startDate} → ${endDate} (${dates.length} days)`);
4984
+ console.log();
4985
+ logger.info("Pass without --dry-run to execute.");
4986
+ return;
4987
+ }
2640
4988
  if (!args.quiet) {
2641
- logger.info(`Syncing ${siteUrl} (${tables.join(", ")}) [${types.join(", ")}] → ${store.dataDir}`);
4989
+ logger.info(`Syncing ${siteUrl} (${tables.join(", ")}) [${types.join(", ")}] → ${displayPath(store.dataDir)}`);
2642
4990
  logger.info(`Range: ${startDate} → ${endDate} (${dates.length} days)`);
2643
4991
  }
2644
4992
  const concurrency = args.concurrency ? Math.max(1, Number.parseInt(String(args.concurrency), 10) || DEFAULT_CONCURRENCY) : DEFAULT_CONCURRENCY;
@@ -2763,7 +5111,7 @@ async function printSyncStatus(config, siteFilter, asJson) {
2763
5111
  return;
2764
5112
  }
2765
5113
  console.log();
2766
- console.log(` \x1B[1m${store.dataDir}\x1B[0m`);
5114
+ console.log(` \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
2767
5115
  if (siteFilter) console.log(` \x1B[90mSite: ${siteFilter}\x1B[0m`);
2768
5116
  console.log();
2769
5117
  if (watermarks.length === 0) {
@@ -2794,6 +5142,51 @@ async function printSyncStatus(config, siteFilter, asJson) {
2794
5142
  }
2795
5143
  console.log();
2796
5144
  }
5145
+ function shouldShowSplash() {
5146
+ if (!process.stdout.isTTY) return false;
5147
+ const argv = process.argv;
5148
+ if (argv.includes("mcp")) return false;
5149
+ for (const flag of [
5150
+ "--json",
5151
+ "--quiet",
5152
+ "-q",
5153
+ "--version",
5154
+ "-v",
5155
+ "--help",
5156
+ "-h"
5157
+ ]) if (argv.includes(flag)) return false;
5158
+ return true;
5159
+ }
5160
+ function applyGlobalArgs() {
5161
+ const argv = process.argv;
5162
+ if (argv.includes("--no-color") || process.env.NO_COLOR) setNoColor(true);
5163
+ const profile = pluckArgValue(argv, "--profile") ?? process.env.GSCDUMP_PROFILE;
5164
+ const configDir = pluckArgValue(argv, "--config-dir") ?? process.env.GSCDUMP_CONFIG_DIR;
5165
+ if (configDir) setConfigDir(configDir);
5166
+ else if (profile) setConfigDir(path.join(os.homedir(), ".config", "gscdump", "profiles", profile));
5167
+ if (argv.includes("-v") && !argv.includes("--version")) {
5168
+ const i = argv.indexOf("-v");
5169
+ argv[i] = "--version";
5170
+ }
5171
+ }
5172
+ function pluckArgValue(argv, flag) {
5173
+ for (let i = 0; i < argv.length; i++) {
5174
+ const a = argv[i];
5175
+ if (a === flag && i + 1 < argv.length) {
5176
+ const v = argv[i + 1];
5177
+ argv.splice(i, 2);
5178
+ return v;
5179
+ }
5180
+ if (a.startsWith(`${flag}=`)) {
5181
+ const v = a.slice(flag.length + 1);
5182
+ argv.splice(i, 1);
5183
+ return v;
5184
+ }
5185
+ }
5186
+ return null;
5187
+ }
5188
+ applyGlobalArgs();
5189
+ loadEnvFromCwd();
2797
5190
  runMain(defineCommand({
2798
5191
  meta: {
2799
5192
  name: "gscdump",
@@ -2809,14 +5202,16 @@ runMain(defineCommand({
2809
5202
  sync: syncCommand,
2810
5203
  store: storeCommand,
2811
5204
  inspect: inspectCommand,
5205
+ indexing: indexingCommand,
2812
5206
  entities: entitiesCommand,
2813
5207
  analyze: analyzeCommand,
2814
5208
  auth: authCommand,
2815
5209
  config: configCommand,
5210
+ doctor: doctorCommand,
2816
5211
  mcp: mcpCommand
2817
5212
  },
2818
5213
  setup() {
2819
- if (!process.argv.includes("mcp")) showSplash();
5214
+ if (shouldShowSplash()) showSplash();
2820
5215
  }
2821
5216
  }));
2822
5217
  export {};