@gscdump/cli 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/_chunks/libs/drizzle-orm.mjs +1043 -0
- package/dist/index.d.mts +1 -0
- package/dist/index.mjs +2431 -0
- package/package.json +46 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,2431 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { t as eq } from "./_chunks/libs/drizzle-orm.mjs";
|
|
3
|
+
import "node:module";
|
|
4
|
+
import process from "node:process";
|
|
5
|
+
import { defineCommand, runMain } from "citty";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { batchInspectUrls, batchRequestIndexingForPaths, createGscDb, getIndexingStats, getLastSyncedDate, getSiteByProperty, setupSchema, sitePathDateAnalytics, syncCountries, syncDevices, syncKeywordPaths, syncKeywords, syncPages, syncSites, updateLastSynced } from "@gscdump/db";
|
|
8
|
+
import { createProvider, fetchCannibalizationAnalysis, fetchDecayAnalysis, fetchMoversAnalysis, fetchOpportunityAnalysis, fetchStrikingDistanceAnalysis, fetchZeroClickAnalysis } from "@gscdump/query";
|
|
9
|
+
import dayjs from "dayjs";
|
|
10
|
+
import betterSqlite3 from "db0/connectors/better-sqlite3";
|
|
11
|
+
import fs from "node:fs/promises";
|
|
12
|
+
import { createServer } from "node:http";
|
|
13
|
+
import { cancel, confirm, isCancel, multiselect, select, text } from "@clack/prompts";
|
|
14
|
+
import { OAuth2Client } from "google-auth-library";
|
|
15
|
+
import os from "node:os";
|
|
16
|
+
import { consola } from "consola";
|
|
17
|
+
import { deleteSitemap, fetchSitemap, fetchSitemaps, fetchSites, formatErrorForCli, googleSearchConsole, submitSitemap, userPeriodRange } from "gscdump";
|
|
18
|
+
import { createGscMcpServer } from "@gscdump/mcp/server";
|
|
19
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
20
|
+
|
|
21
|
+
//#region rolldown:runtime
|
|
22
|
+
var __defProp = Object.defineProperty;
|
|
23
|
+
var __exportAll = (all, symbols) => {
|
|
24
|
+
let target = {};
|
|
25
|
+
for (var name in all) {
|
|
26
|
+
__defProp(target, name, {
|
|
27
|
+
get: all[name],
|
|
28
|
+
enumerable: true
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
if (symbols) {
|
|
32
|
+
__defProp(target, Symbol.toStringTag, { value: "Module" });
|
|
33
|
+
}
|
|
34
|
+
return target;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
//#endregion
|
|
38
|
+
//#region src/config.ts
|
|
39
|
+
let configDir = path.join(os.homedir(), ".config", "gscdump");
|
|
40
|
+
function getConfigDir() {
|
|
41
|
+
return configDir;
|
|
42
|
+
}
|
|
43
|
+
const DEFAULT_CLOUD_URL = "https://cloud.gscdump.com";
|
|
44
|
+
async function loadConfig() {
|
|
45
|
+
return fs.readFile(path.join(configDir, "config.json"), "utf-8").then((data) => JSON.parse(data)).catch(() => ({}));
|
|
46
|
+
}
|
|
47
|
+
async function saveConfig(config) {
|
|
48
|
+
await fs.mkdir(configDir, {
|
|
49
|
+
recursive: true,
|
|
50
|
+
mode: 448
|
|
51
|
+
});
|
|
52
|
+
await fs.writeFile(path.join(configDir, "config.json"), JSON.stringify(config, null, 2), { mode: 384 });
|
|
53
|
+
}
|
|
54
|
+
function getConfigPath() {
|
|
55
|
+
return path.join(configDir, "config.json");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
//#endregion
|
|
59
|
+
//#region src/utils.ts
|
|
60
|
+
const VERSION = "1.0.0";
|
|
61
|
+
const logger = consola.withTag("gscdump");
|
|
62
|
+
/**
|
|
63
|
+
* Handles GSC API errors with helpful messages and suggestions.
|
|
64
|
+
* Exits process with code 1.
|
|
65
|
+
*/
|
|
66
|
+
function handleGscError(error) {
|
|
67
|
+
console.error();
|
|
68
|
+
console.error(formatErrorForCli(error));
|
|
69
|
+
console.error();
|
|
70
|
+
process.exit(1);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Creates a .catch() handler for GSC API errors.
|
|
74
|
+
* Use: somePromise.catch(gscErrorHandler)
|
|
75
|
+
*/
|
|
76
|
+
function gscErrorHandler(error) {
|
|
77
|
+
return handleGscError(error);
|
|
78
|
+
}
|
|
79
|
+
const gradientColors = [
|
|
80
|
+
(s) => `\x1B[38;2;52;211;153m${s}\x1B[0m`,
|
|
81
|
+
(s) => `\x1B[38;2;45;212;191m${s}\x1B[0m`,
|
|
82
|
+
(s) => `\x1B[38;2;34;211;238m${s}\x1B[0m`,
|
|
83
|
+
(s) => `\x1B[38;2;56;189;248m${s}\x1B[0m`,
|
|
84
|
+
(s) => `\x1B[38;2;96;165;250m${s}\x1B[0m`
|
|
85
|
+
];
|
|
86
|
+
function applyGradient(text$1) {
|
|
87
|
+
return [...text$1].map((char, i) => {
|
|
88
|
+
const colorIndex = Math.floor(i / text$1.length * gradientColors.length);
|
|
89
|
+
return gradientColors[Math.min(colorIndex, gradientColors.length - 1)](char);
|
|
90
|
+
}).join("");
|
|
91
|
+
}
|
|
92
|
+
function showSplash() {
|
|
93
|
+
console.log();
|
|
94
|
+
console.log(` ${applyGradient("GSC Dump")} v${VERSION}`);
|
|
95
|
+
console.log();
|
|
96
|
+
}
|
|
97
|
+
function progressBar(current, total, label, width = 30) {
|
|
98
|
+
const percent = Math.min(current / total, 1);
|
|
99
|
+
const filled = Math.round(width * percent);
|
|
100
|
+
const empty = width - filled;
|
|
101
|
+
return ` ${`\x1B[36m${"█".repeat(filled)}\x1B[0m\x1B[90m${"░".repeat(empty)}\x1B[0m`} \x1B[90m${current}/${total}\x1B[0m ${label}`;
|
|
102
|
+
}
|
|
103
|
+
function clearLine() {
|
|
104
|
+
process.stdout.write("\r\x1B[K");
|
|
105
|
+
}
|
|
106
|
+
function parsePeriod(periodStr) {
|
|
107
|
+
const match = periodStr.match(/^(\d+)([dmy])$/i);
|
|
108
|
+
if (!match) return null;
|
|
109
|
+
const amount = Number.parseInt(match[1], 10);
|
|
110
|
+
const unit = {
|
|
111
|
+
d: "days",
|
|
112
|
+
m: "months",
|
|
113
|
+
y: "years"
|
|
114
|
+
}[match[2].toLowerCase()];
|
|
115
|
+
if (!unit || amount <= 0) return null;
|
|
116
|
+
if (amount > 450 && unit === "days") return null;
|
|
117
|
+
return {
|
|
118
|
+
amount,
|
|
119
|
+
unit
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
function toCSV(data, columns) {
|
|
123
|
+
return [columns.join(","), ...data.map((row) => columns.map((col) => {
|
|
124
|
+
const val = row[col];
|
|
125
|
+
if (val === null || val === void 0) return "";
|
|
126
|
+
const str = String(val);
|
|
127
|
+
return str.includes(",") || str.includes("\"") || str.includes("\n") ? `"${str.replace(/"/g, "\"\"")}"` : str;
|
|
128
|
+
}).join(","))].join("\n");
|
|
129
|
+
}
|
|
130
|
+
function exportToCSV(output) {
|
|
131
|
+
const sections = [];
|
|
132
|
+
if (output.pages?.data) sections.push(`# Pages\n${toCSV(output.pages.data, [
|
|
133
|
+
"url",
|
|
134
|
+
"clicks",
|
|
135
|
+
"impressions",
|
|
136
|
+
"ctr",
|
|
137
|
+
"position"
|
|
138
|
+
])}`);
|
|
139
|
+
if (output.keywords?.current) sections.push(`# Keywords (Current Period)\n${toCSV(output.keywords.current, [
|
|
140
|
+
"query",
|
|
141
|
+
"clicks",
|
|
142
|
+
"impressions",
|
|
143
|
+
"ctr",
|
|
144
|
+
"position"
|
|
145
|
+
])}`);
|
|
146
|
+
if (output.countries?.current) sections.push(`# Countries (Current Period)\n${toCSV(output.countries.current, [
|
|
147
|
+
"country",
|
|
148
|
+
"clicks",
|
|
149
|
+
"impressions",
|
|
150
|
+
"ctr",
|
|
151
|
+
"position"
|
|
152
|
+
])}`);
|
|
153
|
+
if (output.devices?.current) sections.push(`# Devices (Current Period)\n${toCSV(output.devices.current, [
|
|
154
|
+
"device",
|
|
155
|
+
"clicks",
|
|
156
|
+
"impressions",
|
|
157
|
+
"ctr",
|
|
158
|
+
"position"
|
|
159
|
+
])}`);
|
|
160
|
+
return sections.join("\n\n");
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
//#endregion
|
|
164
|
+
//#region src/auth.ts
|
|
165
|
+
var auth_exports = /* @__PURE__ */ __exportAll({
|
|
166
|
+
authenticate: () => authenticate,
|
|
167
|
+
authenticateCloud: () => authenticateCloud,
|
|
168
|
+
clearCloudTokens: () => clearCloudTokens,
|
|
169
|
+
clearTokens: () => clearTokens,
|
|
170
|
+
getAuth: () => getAuth,
|
|
171
|
+
getAuthCredentials: () => getAuthCredentials,
|
|
172
|
+
loadCloudTokens: () => loadCloudTokens,
|
|
173
|
+
loadTokens: () => loadTokens,
|
|
174
|
+
saveCloudTokens: () => saveCloudTokens,
|
|
175
|
+
saveTokens: () => saveTokens
|
|
176
|
+
});
|
|
177
|
+
function getTokensPath() {
|
|
178
|
+
return path.join(getConfigDir(), "tokens.json");
|
|
179
|
+
}
|
|
180
|
+
async function loadTokens() {
|
|
181
|
+
return fs.readFile(getTokensPath(), "utf-8").then((data) => JSON.parse(data)).catch(() => null);
|
|
182
|
+
}
|
|
183
|
+
async function saveTokens(tokens) {
|
|
184
|
+
await fs.mkdir(getConfigDir(), {
|
|
185
|
+
recursive: true,
|
|
186
|
+
mode: 448
|
|
187
|
+
});
|
|
188
|
+
await fs.writeFile(getTokensPath(), JSON.stringify(tokens, null, 2), { mode: 384 });
|
|
189
|
+
}
|
|
190
|
+
async function clearTokens() {
|
|
191
|
+
await fs.rm(getTokensPath()).catch(() => {});
|
|
192
|
+
logger.success("Logged out, tokens cleared");
|
|
193
|
+
}
|
|
194
|
+
async function getAuthCredentials(interactive) {
|
|
195
|
+
const envClientId = process.env.GOOGLE_CLIENT_ID;
|
|
196
|
+
const envClientSecret = process.env.GOOGLE_CLIENT_SECRET;
|
|
197
|
+
if (envClientId && envClientSecret) {
|
|
198
|
+
logger.success("Using OAuth2 credentials from environment");
|
|
199
|
+
return {
|
|
200
|
+
clientId: envClientId,
|
|
201
|
+
clientSecret: envClientSecret
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
const config = await loadConfig();
|
|
205
|
+
if (config.clientId && config.clientSecret) return {
|
|
206
|
+
clientId: config.clientId,
|
|
207
|
+
clientSecret: config.clientSecret
|
|
208
|
+
};
|
|
209
|
+
if (!interactive) {
|
|
210
|
+
logger.error("GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET required for non-interactive mode");
|
|
211
|
+
process.exit(1);
|
|
212
|
+
}
|
|
213
|
+
console.log();
|
|
214
|
+
console.log(" \x1B[1mOAuth 2.0 Setup Required\x1B[0m");
|
|
215
|
+
console.log(" \x1B[90mThe Google Search Console API requires OAuth 2.0 credentials.\x1B[0m");
|
|
216
|
+
console.log();
|
|
217
|
+
console.log(" \x1B[1mSteps:\x1B[0m");
|
|
218
|
+
console.log(" \x1B[90m1.\x1B[0m Go to \x1B[36mhttps://console.developers.google.com/apis/credentials\x1B[0m");
|
|
219
|
+
console.log(" \x1B[90m2.\x1B[0m Create credentials > OAuth client ID > Desktop application");
|
|
220
|
+
console.log(" \x1B[90m3.\x1B[0m Enable \"Search Console API\" and \"Web Search Indexing API\" for your project");
|
|
221
|
+
console.log(" \x1B[90m4.\x1B[0m Copy the Client ID and Client Secret");
|
|
222
|
+
console.log();
|
|
223
|
+
const clientIdResult = await text({
|
|
224
|
+
message: "Enter your Google OAuth Client ID:",
|
|
225
|
+
placeholder: "your-client-id.googleusercontent.com",
|
|
226
|
+
validate: (v) => v ? void 0 : "Required"
|
|
227
|
+
});
|
|
228
|
+
if (isCancel(clientIdResult)) process.exit(1);
|
|
229
|
+
const clientSecretResult = await text({
|
|
230
|
+
message: "Enter your Google OAuth Client Secret:",
|
|
231
|
+
validate: (v) => v ? void 0 : "Required"
|
|
232
|
+
});
|
|
233
|
+
if (isCancel(clientSecretResult)) process.exit(1);
|
|
234
|
+
console.log();
|
|
235
|
+
logger.info("Tip: Set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET env vars to skip prompts");
|
|
236
|
+
return {
|
|
237
|
+
clientId: clientIdResult,
|
|
238
|
+
clientSecret: clientSecretResult
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
async function getAuthCodeViaLoopback(authUrl) {
|
|
242
|
+
return new Promise((resolve, reject) => {
|
|
243
|
+
let resolvedRedirectUri = "";
|
|
244
|
+
const server = createServer((req, res) => {
|
|
245
|
+
const url = new URL(req.url || "", `http://127.0.0.1`);
|
|
246
|
+
const code = url.searchParams.get("code");
|
|
247
|
+
const error = url.searchParams.get("error");
|
|
248
|
+
if (error) {
|
|
249
|
+
res.writeHead(400, { "Content-Type": "text/html" });
|
|
250
|
+
res.end(`<html><body><h1>Authorization Failed</h1><p>${error}</p><p>You can close this window.</p></body></html>`);
|
|
251
|
+
server.close();
|
|
252
|
+
reject(/* @__PURE__ */ new Error(`OAuth error: ${error}`));
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
if (code) {
|
|
256
|
+
res.writeHead(200, { "Content-Type": "text/html" });
|
|
257
|
+
res.end(`<html><body><h1>Authorization Successful</h1><p>You can close this window and return to the terminal.</p></body></html>`);
|
|
258
|
+
server.close();
|
|
259
|
+
resolve({
|
|
260
|
+
code,
|
|
261
|
+
redirectUri: resolvedRedirectUri
|
|
262
|
+
});
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
res.writeHead(400, { "Content-Type": "text/html" });
|
|
266
|
+
res.end(`<html><body><h1>Missing authorization code</h1></body></html>`);
|
|
267
|
+
});
|
|
268
|
+
server.listen(0, "127.0.0.1", () => {
|
|
269
|
+
const addr = server.address();
|
|
270
|
+
if (!addr || typeof addr === "string") {
|
|
271
|
+
reject(/* @__PURE__ */ new Error("Failed to start local server"));
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
resolvedRedirectUri = `http://127.0.0.1:${addr.port}`;
|
|
275
|
+
const fullAuthUrl = authUrl.replace(/redirect_uri=[^&]+/, `redirect_uri=${encodeURIComponent(resolvedRedirectUri)}`);
|
|
276
|
+
console.log();
|
|
277
|
+
console.log(" \x1B[1mOpening browser for authorization...\x1B[0m");
|
|
278
|
+
console.log(` \x1B[90mIf browser doesn't open, visit:\x1B[0m`);
|
|
279
|
+
console.log(` \x1B[36m${fullAuthUrl}\x1B[0m`);
|
|
280
|
+
console.log();
|
|
281
|
+
import("open").then(({ default: open }) => open(fullAuthUrl)).catch(() => {
|
|
282
|
+
logger.warn("Could not open browser automatically");
|
|
283
|
+
});
|
|
284
|
+
});
|
|
285
|
+
server.on("error", reject);
|
|
286
|
+
setTimeout(() => {
|
|
287
|
+
server.close();
|
|
288
|
+
reject(/* @__PURE__ */ new Error("Authorization timed out"));
|
|
289
|
+
}, 300 * 1e3);
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
async function authenticate(credentials, interactive) {
|
|
293
|
+
const oauth2Client = new OAuth2Client(credentials.clientId, credentials.clientSecret, "http://127.0.0.1");
|
|
294
|
+
const envAccessToken = process.env.GOOGLE_ACCESS_TOKEN;
|
|
295
|
+
const envRefreshToken = process.env.GOOGLE_REFRESH_TOKEN;
|
|
296
|
+
if (envAccessToken || envRefreshToken) {
|
|
297
|
+
oauth2Client.setCredentials({
|
|
298
|
+
access_token: envAccessToken,
|
|
299
|
+
refresh_token: envRefreshToken
|
|
300
|
+
});
|
|
301
|
+
if (envRefreshToken) {
|
|
302
|
+
const { credentials: newTokens } = await oauth2Client.refreshAccessToken().catch(() => ({ credentials: null }));
|
|
303
|
+
if (newTokens) oauth2Client.setCredentials(newTokens);
|
|
304
|
+
}
|
|
305
|
+
return oauth2Client;
|
|
306
|
+
}
|
|
307
|
+
const existingTokens = await loadTokens();
|
|
308
|
+
if (existingTokens) {
|
|
309
|
+
oauth2Client.setCredentials(existingTokens);
|
|
310
|
+
if (existingTokens.expiry_date && existingTokens.expiry_date < Date.now()) {
|
|
311
|
+
const { credentials: newTokens } = await oauth2Client.refreshAccessToken().catch(() => ({ credentials: null }));
|
|
312
|
+
if (newTokens) {
|
|
313
|
+
await saveTokens(newTokens);
|
|
314
|
+
oauth2Client.setCredentials(newTokens);
|
|
315
|
+
logger.success("Token refreshed");
|
|
316
|
+
return oauth2Client;
|
|
317
|
+
}
|
|
318
|
+
} else {
|
|
319
|
+
logger.success("Using saved credentials");
|
|
320
|
+
return oauth2Client;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
if (!interactive) {
|
|
324
|
+
logger.error("No saved tokens. Run interactively first to authenticate.");
|
|
325
|
+
process.exit(1);
|
|
326
|
+
}
|
|
327
|
+
const authUrl = oauth2Client.generateAuthUrl({
|
|
328
|
+
access_type: "offline",
|
|
329
|
+
scope: ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/indexing"],
|
|
330
|
+
prompt: "consent"
|
|
331
|
+
});
|
|
332
|
+
logger.info("Waiting for authorization...");
|
|
333
|
+
const { code, redirectUri } = await getAuthCodeViaLoopback(authUrl);
|
|
334
|
+
const { tokens } = await new OAuth2Client(credentials.clientId, credentials.clientSecret, redirectUri).getToken(code);
|
|
335
|
+
oauth2Client.setCredentials(tokens);
|
|
336
|
+
await saveTokens(tokens);
|
|
337
|
+
logger.success(`Tokens saved to ${getTokensPath()}`);
|
|
338
|
+
return oauth2Client;
|
|
339
|
+
}
|
|
340
|
+
function getCloudTokensPath() {
|
|
341
|
+
return path.join(getConfigDir(), "cloud-tokens.json");
|
|
342
|
+
}
|
|
343
|
+
async function loadCloudTokens() {
|
|
344
|
+
return fs.readFile(getCloudTokensPath(), "utf-8").then((data) => JSON.parse(data)).catch(() => null);
|
|
345
|
+
}
|
|
346
|
+
async function saveCloudTokens(tokens) {
|
|
347
|
+
await fs.mkdir(getConfigDir(), {
|
|
348
|
+
recursive: true,
|
|
349
|
+
mode: 448
|
|
350
|
+
});
|
|
351
|
+
await fs.writeFile(getCloudTokensPath(), JSON.stringify(tokens, null, 2), { mode: 384 });
|
|
352
|
+
}
|
|
353
|
+
async function clearCloudTokens() {
|
|
354
|
+
await fs.rm(getCloudTokensPath()).catch(() => {});
|
|
355
|
+
logger.success("Logged out from cloud");
|
|
356
|
+
}
|
|
357
|
+
async function authenticateCloud(cloudUrl, interactive) {
|
|
358
|
+
const existingTokens = await loadCloudTokens();
|
|
359
|
+
if (existingTokens) {
|
|
360
|
+
const oauth2Client = new OAuth2Client();
|
|
361
|
+
oauth2Client.setCredentials({
|
|
362
|
+
access_token: existingTokens.accessToken,
|
|
363
|
+
refresh_token: existingTokens.refreshToken,
|
|
364
|
+
expiry_date: existingTokens.expiresAt
|
|
365
|
+
});
|
|
366
|
+
logger.success("Using cloud credentials");
|
|
367
|
+
return oauth2Client;
|
|
368
|
+
}
|
|
369
|
+
if (!interactive) {
|
|
370
|
+
logger.error("No cloud tokens. Run gscdump init to authenticate.");
|
|
371
|
+
process.exit(1);
|
|
372
|
+
}
|
|
373
|
+
const initRes = await fetch(`${cloudUrl}/api/cli/auth/init`, { method: "POST" }).then((r) => r.json()).catch((e) => {
|
|
374
|
+
logger.error(`Failed to connect to ${cloudUrl}: ${e.message}`);
|
|
375
|
+
process.exit(1);
|
|
376
|
+
});
|
|
377
|
+
console.log();
|
|
378
|
+
console.log(" \x1B[1mOpen this URL in your browser:\x1B[0m");
|
|
379
|
+
console.log(` \x1B[36m${initRes.authUrl}\x1B[0m`);
|
|
380
|
+
console.log();
|
|
381
|
+
console.log(` \x1B[90mCode: ${initRes.code}\x1B[0m`);
|
|
382
|
+
console.log();
|
|
383
|
+
logger.info("Waiting for authorization...");
|
|
384
|
+
const pollInterval = 2e3;
|
|
385
|
+
const maxAttempts = Math.ceil(initRes.expiresIn * 1e3 / pollInterval);
|
|
386
|
+
for (let i = 0; i < maxAttempts; i++) {
|
|
387
|
+
await new Promise((r) => setTimeout(r, pollInterval));
|
|
388
|
+
const pollRes = await fetch(`${cloudUrl}/api/cli/auth/poll?code=${initRes.code}`).then((r) => r.json()).catch(() => ({ status: "error" }));
|
|
389
|
+
if (pollRes.status === "complete" && pollRes.tokens) {
|
|
390
|
+
await saveCloudTokens(pollRes.tokens);
|
|
391
|
+
logger.success("Authenticated via cloud.gscdump.com");
|
|
392
|
+
const oauth2Client = new OAuth2Client();
|
|
393
|
+
oauth2Client.setCredentials({
|
|
394
|
+
access_token: pollRes.tokens.accessToken,
|
|
395
|
+
refresh_token: pollRes.tokens.refreshToken,
|
|
396
|
+
expiry_date: pollRes.tokens.expiresAt
|
|
397
|
+
});
|
|
398
|
+
return oauth2Client;
|
|
399
|
+
}
|
|
400
|
+
if (pollRes.status === "error") {
|
|
401
|
+
logger.error("Authorization failed");
|
|
402
|
+
process.exit(1);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
logger.error("Authorization timed out");
|
|
406
|
+
process.exit(1);
|
|
407
|
+
}
|
|
408
|
+
async function getAuth(opts = {}) {
|
|
409
|
+
const { interactive = true, config: providedConfig } = opts;
|
|
410
|
+
const config = providedConfig || await loadConfig();
|
|
411
|
+
if (!config.mode) {
|
|
412
|
+
if (!interactive) {
|
|
413
|
+
logger.error("Not configured. Run gscdump init first.");
|
|
414
|
+
process.exit(1);
|
|
415
|
+
}
|
|
416
|
+
logger.warn("GSCDump not configured");
|
|
417
|
+
logger.info("Run: gscdump init");
|
|
418
|
+
process.exit(1);
|
|
419
|
+
}
|
|
420
|
+
if (config.mode === "cloud") return authenticateCloud(config.cloudUrl || DEFAULT_CLOUD_URL, interactive);
|
|
421
|
+
return authenticate(await getAuthCredentials(interactive), interactive);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
//#endregion
|
|
425
|
+
//#region src/commands/analyze.ts
|
|
426
|
+
const ANALYSIS_TYPES = [
|
|
427
|
+
"striking-distance",
|
|
428
|
+
"opportunity",
|
|
429
|
+
"movers",
|
|
430
|
+
"decay",
|
|
431
|
+
"cannibalization",
|
|
432
|
+
"zero-click"
|
|
433
|
+
];
|
|
434
|
+
const analyzeCommand = defineCommand({
|
|
435
|
+
meta: {
|
|
436
|
+
name: "analyze",
|
|
437
|
+
description: "Run SEO analysis on site data"
|
|
438
|
+
},
|
|
439
|
+
args: {
|
|
440
|
+
type: {
|
|
441
|
+
type: "positional",
|
|
442
|
+
required: true,
|
|
443
|
+
description: `Analysis type: ${ANALYSIS_TYPES.join(", ")}`
|
|
444
|
+
},
|
|
445
|
+
site: {
|
|
446
|
+
type: "string",
|
|
447
|
+
alias: "s",
|
|
448
|
+
description: "Site URL (e.g., sc-domain:example.com)"
|
|
449
|
+
},
|
|
450
|
+
period: {
|
|
451
|
+
type: "string",
|
|
452
|
+
alias: "p",
|
|
453
|
+
default: "28d",
|
|
454
|
+
description: "Period to analyze (e.g., 7d, 28d, 3m)"
|
|
455
|
+
},
|
|
456
|
+
limit: {
|
|
457
|
+
type: "string",
|
|
458
|
+
alias: "l",
|
|
459
|
+
default: "20",
|
|
460
|
+
description: "Number of results to show"
|
|
461
|
+
},
|
|
462
|
+
json: {
|
|
463
|
+
type: "boolean",
|
|
464
|
+
default: false,
|
|
465
|
+
description: "Output as JSON"
|
|
466
|
+
},
|
|
467
|
+
db: {
|
|
468
|
+
type: "string",
|
|
469
|
+
alias: "d",
|
|
470
|
+
description: "Path to SQLite database"
|
|
471
|
+
},
|
|
472
|
+
source: {
|
|
473
|
+
type: "string",
|
|
474
|
+
default: "auto",
|
|
475
|
+
description: "Data source: api, db, auto"
|
|
476
|
+
},
|
|
477
|
+
minPosition: {
|
|
478
|
+
type: "string",
|
|
479
|
+
description: "Minimum position (striking-distance)"
|
|
480
|
+
},
|
|
481
|
+
maxPosition: {
|
|
482
|
+
type: "string",
|
|
483
|
+
description: "Maximum position (striking-distance, zero-click)"
|
|
484
|
+
},
|
|
485
|
+
minImpressions: {
|
|
486
|
+
type: "string",
|
|
487
|
+
description: "Minimum impressions"
|
|
488
|
+
},
|
|
489
|
+
minClicks: {
|
|
490
|
+
type: "string",
|
|
491
|
+
description: "Minimum clicks (decay)"
|
|
492
|
+
},
|
|
493
|
+
threshold: {
|
|
494
|
+
type: "string",
|
|
495
|
+
description: "Threshold percentage 0-1 (decay, movers)"
|
|
496
|
+
}
|
|
497
|
+
},
|
|
498
|
+
async run({ args }) {
|
|
499
|
+
const analysisType = args.type;
|
|
500
|
+
if (!ANALYSIS_TYPES.includes(analysisType)) {
|
|
501
|
+
logger.error(`Unknown analysis type: ${analysisType}`);
|
|
502
|
+
logger.info(`Available: ${ANALYSIS_TYPES.join(", ")}`);
|
|
503
|
+
process.exit(1);
|
|
504
|
+
}
|
|
505
|
+
const config = await loadConfig();
|
|
506
|
+
const siteArg = args.site || config.defaultSite;
|
|
507
|
+
if (!siteArg) {
|
|
508
|
+
logger.error("Site required. Use --site or set defaultSite in config.");
|
|
509
|
+
process.exit(1);
|
|
510
|
+
}
|
|
511
|
+
const dbPath = args.db || config.defaultDb;
|
|
512
|
+
const auth = await getAuth({
|
|
513
|
+
interactive: false,
|
|
514
|
+
config
|
|
515
|
+
});
|
|
516
|
+
const period = parsePeriod(args.period);
|
|
517
|
+
if (!period) {
|
|
518
|
+
logger.error(`Invalid period: ${args.period}`);
|
|
519
|
+
process.exit(1);
|
|
520
|
+
}
|
|
521
|
+
const endDate = dayjs().subtract(3, "day");
|
|
522
|
+
const startDate = endDate.subtract(period.amount, period.unit);
|
|
523
|
+
const prevEndDate = startDate.subtract(1, "day");
|
|
524
|
+
const prevStartDate = prevEndDate.subtract(period.amount, period.unit);
|
|
525
|
+
const range = {
|
|
526
|
+
period: {
|
|
527
|
+
start: startDate.format("YYYY-MM-DD"),
|
|
528
|
+
end: endDate.format("YYYY-MM-DD")
|
|
529
|
+
},
|
|
530
|
+
prevPeriod: {
|
|
531
|
+
start: prevStartDate.format("YYYY-MM-DD"),
|
|
532
|
+
end: prevEndDate.format("YYYY-MM-DD")
|
|
533
|
+
}
|
|
534
|
+
};
|
|
535
|
+
const provider = await createProvider({
|
|
536
|
+
auth,
|
|
537
|
+
db: dbPath ? createGscDb(betterSqlite3({ name: path.resolve(dbPath) })).db : null,
|
|
538
|
+
source: args.source,
|
|
539
|
+
siteUrls: [siteArg],
|
|
540
|
+
range
|
|
541
|
+
});
|
|
542
|
+
const limit = Number.parseInt(args.limit, 10) || 20;
|
|
543
|
+
if (!args.json) {
|
|
544
|
+
console.log();
|
|
545
|
+
console.log(` \x1B[1m${siteArg}\x1B[0m`);
|
|
546
|
+
console.log(` \x1B[90mSource: ${provider.source}\x1B[0m`);
|
|
547
|
+
console.log(` \x1B[90mPeriod: ${range.period.start} → ${range.period.end}\x1B[0m`);
|
|
548
|
+
console.log(` \x1B[90mAnalysis: ${analysisType}\x1B[0m`);
|
|
549
|
+
console.log();
|
|
550
|
+
}
|
|
551
|
+
let results;
|
|
552
|
+
switch (analysisType) {
|
|
553
|
+
case "striking-distance":
|
|
554
|
+
results = await fetchStrikingDistanceAnalysis(provider, siteArg, range, {
|
|
555
|
+
minPosition: args.minPosition ? Number(args.minPosition) : void 0,
|
|
556
|
+
maxPosition: args.maxPosition ? Number(args.maxPosition) : void 0,
|
|
557
|
+
minImpressions: args.minImpressions ? Number(args.minImpressions) : void 0
|
|
558
|
+
}).catch(gscErrorHandler);
|
|
559
|
+
break;
|
|
560
|
+
case "opportunity":
|
|
561
|
+
results = await fetchOpportunityAnalysis(provider, siteArg, range, { minImpressions: args.minImpressions ? Number(args.minImpressions) : void 0 }).catch(gscErrorHandler);
|
|
562
|
+
break;
|
|
563
|
+
case "movers":
|
|
564
|
+
results = await fetchMoversAnalysis(provider, siteArg, range, {
|
|
565
|
+
changeThreshold: args.threshold ? Number(args.threshold) : void 0,
|
|
566
|
+
minImpressions: args.minImpressions ? Number(args.minImpressions) : void 0
|
|
567
|
+
}).catch(gscErrorHandler);
|
|
568
|
+
break;
|
|
569
|
+
case "decay":
|
|
570
|
+
results = await fetchDecayAnalysis(provider, siteArg, range, {
|
|
571
|
+
minPreviousClicks: args.minClicks ? Number(args.minClicks) : void 0,
|
|
572
|
+
threshold: args.threshold ? Number(args.threshold) : void 0
|
|
573
|
+
}).catch(gscErrorHandler);
|
|
574
|
+
break;
|
|
575
|
+
case "cannibalization":
|
|
576
|
+
results = await fetchCannibalizationAnalysis(provider, siteArg, range, { minImpressions: args.minImpressions ? Number(args.minImpressions) : void 0 }).catch(gscErrorHandler);
|
|
577
|
+
break;
|
|
578
|
+
case "zero-click":
|
|
579
|
+
results = await fetchZeroClickAnalysis(provider, siteArg, range, {
|
|
580
|
+
minImpressions: args.minImpressions ? Number(args.minImpressions) : void 0,
|
|
581
|
+
maxPosition: args.maxPosition ? Number(args.maxPosition) : void 0
|
|
582
|
+
}).catch(gscErrorHandler);
|
|
583
|
+
break;
|
|
584
|
+
}
|
|
585
|
+
if (args.json) {
|
|
586
|
+
console.log(JSON.stringify({
|
|
587
|
+
site: siteArg,
|
|
588
|
+
source: provider.source,
|
|
589
|
+
analysis: analysisType,
|
|
590
|
+
range,
|
|
591
|
+
results
|
|
592
|
+
}, null, 2));
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
const arr = Array.isArray(results) ? results : [];
|
|
596
|
+
const display = arr.slice(0, limit);
|
|
597
|
+
if (display.length === 0) {
|
|
598
|
+
logger.warn("No results found");
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
console.log(` \x1B[1mResults\x1B[0m \x1B[90m(${arr.length} total, showing ${display.length})\x1B[0m`);
|
|
602
|
+
for (const item of display) formatResultItem(analysisType, item);
|
|
603
|
+
console.log();
|
|
604
|
+
logger.success("Analysis complete");
|
|
605
|
+
}
|
|
606
|
+
});
|
|
607
|
+
function formatResultItem(type, item) {
|
|
608
|
+
const keyword = item.keyword || item.query || "";
|
|
609
|
+
const page = item.page || "";
|
|
610
|
+
switch (type) {
|
|
611
|
+
case "striking-distance":
|
|
612
|
+
console.log(` ${keyword.slice(0, 40).padEnd(40)} pos ${String(item.position || 0).padStart(5)} ${formatNumber$1(item.impressions)} impr ${formatNumber$1(item.potentialClicks)} potential`);
|
|
613
|
+
break;
|
|
614
|
+
case "opportunity":
|
|
615
|
+
console.log(` ${keyword.slice(0, 40).padEnd(40)} score ${String((item.score || 0).toFixed(1)).padStart(6)} ${formatNumber$1(item.impressions)} impr`);
|
|
616
|
+
break;
|
|
617
|
+
case "movers":
|
|
618
|
+
console.log(` ${keyword.slice(0, 40).padEnd(40)} ${formatChange(item.clicksChange)} clicks ${formatChange(item.positionChange)} pos`);
|
|
619
|
+
break;
|
|
620
|
+
case "decay":
|
|
621
|
+
console.log(` ${page.replace(/^https?:\/\/[^/]+/, "").slice(0, 50).padEnd(50)} -${formatNumber$1(item.lostClicks)} clicks (${((item.declinePercent || 0) * 100).toFixed(0)}% decline)`);
|
|
622
|
+
break;
|
|
623
|
+
case "cannibalization":
|
|
624
|
+
console.log(` ${keyword.slice(0, 40).padEnd(40)} ${item.pageCount} pages ${formatNumber$1(item.clicks)} clicks spread ${item.positionSpread}`);
|
|
625
|
+
break;
|
|
626
|
+
case "zero-click":
|
|
627
|
+
console.log(` ${keyword.slice(0, 40).padEnd(40)} pos ${String(item.position || 0).padStart(5)} ${formatNumber$1(item.impressions)} impr ${((item.ctr || 0) * 100).toFixed(2)}% ctr`);
|
|
628
|
+
break;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
function formatNumber$1(val) {
|
|
632
|
+
if (val === void 0 || val === null) return "-";
|
|
633
|
+
return val.toLocaleString();
|
|
634
|
+
}
|
|
635
|
+
function formatChange(val) {
|
|
636
|
+
if (val === void 0 || val === null || !Number.isFinite(val)) return "\x1B[90m-\x1B[0m";
|
|
637
|
+
return `${val > 0 ? "\x1B[32m" : val < 0 ? "\x1B[31m" : "\x1B[90m"}${val > 0 ? "+" : ""}${val.toFixed(1)}%\x1B[0m`;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
//#endregion
|
|
641
|
+
//#region src/commands/auth.ts
|
|
642
|
+
const statusCommand = defineCommand({
|
|
643
|
+
meta: {
|
|
644
|
+
name: "status",
|
|
645
|
+
description: "Show current authentication status"
|
|
646
|
+
},
|
|
647
|
+
async run() {
|
|
648
|
+
const config = await loadConfig();
|
|
649
|
+
console.log();
|
|
650
|
+
console.log(` Mode: ${config.mode ? `\x1B[36m${config.mode}\x1B[0m` : "\x1B[33mnot configured\x1B[0m"}`);
|
|
651
|
+
if (!config.mode) {
|
|
652
|
+
logger.info("Run gscdump init to configure");
|
|
653
|
+
return;
|
|
654
|
+
}
|
|
655
|
+
if (config.mode === "cloud") {
|
|
656
|
+
console.log(` Cloud: \x1B[36m${config.cloudUrl}\x1B[0m`);
|
|
657
|
+
const tokens = await loadCloudTokens();
|
|
658
|
+
if (!tokens) {
|
|
659
|
+
logger.warn("Not authenticated");
|
|
660
|
+
logger.info("Run gscdump init --force to re-authenticate");
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
const hasAccess = !!tokens.accessToken;
|
|
664
|
+
const hasRefresh = !!tokens.refreshToken;
|
|
665
|
+
const expiry = tokens.expiresAt ? new Date(tokens.expiresAt) : null;
|
|
666
|
+
const isExpired = expiry && expiry < /* @__PURE__ */ new Date();
|
|
667
|
+
logger.success("Authenticated");
|
|
668
|
+
console.log();
|
|
669
|
+
console.log(` Access token: ${hasAccess ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
670
|
+
console.log(` Refresh token: ${hasRefresh ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
671
|
+
if (expiry) {
|
|
672
|
+
const status = isExpired ? "\x1B[33mexpired\x1B[0m" : "\x1B[32mvalid\x1B[0m";
|
|
673
|
+
console.log(` Expires: ${expiry.toISOString()} (${status})`);
|
|
674
|
+
}
|
|
675
|
+
} else {
|
|
676
|
+
const tokens = await loadTokens();
|
|
677
|
+
if (!tokens) {
|
|
678
|
+
logger.warn("Not authenticated");
|
|
679
|
+
logger.info("Run gscdump init --force to re-authenticate");
|
|
680
|
+
return;
|
|
681
|
+
}
|
|
682
|
+
const hasAccess = !!tokens.access_token;
|
|
683
|
+
const hasRefresh = !!tokens.refresh_token;
|
|
684
|
+
const expiry = tokens.expiry_date ? new Date(tokens.expiry_date) : null;
|
|
685
|
+
const isExpired = expiry && expiry < /* @__PURE__ */ new Date();
|
|
686
|
+
logger.success("Authenticated");
|
|
687
|
+
console.log();
|
|
688
|
+
console.log(` Access token: ${hasAccess ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
689
|
+
console.log(` Refresh token: ${hasRefresh ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
690
|
+
if (expiry) {
|
|
691
|
+
const status = isExpired ? "\x1B[33mexpired\x1B[0m" : "\x1B[32mvalid\x1B[0m";
|
|
692
|
+
console.log(` Expires: ${expiry.toISOString()} (${status})`);
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
});
|
|
697
|
+
const logoutCommand = defineCommand({
|
|
698
|
+
meta: {
|
|
699
|
+
name: "logout",
|
|
700
|
+
description: "Clear stored OAuth tokens"
|
|
701
|
+
},
|
|
702
|
+
async run() {
|
|
703
|
+
if ((await loadConfig()).mode === "cloud") await clearCloudTokens();
|
|
704
|
+
else await clearTokens();
|
|
705
|
+
}
|
|
706
|
+
});
|
|
707
|
+
const authCommand = defineCommand({
|
|
708
|
+
meta: {
|
|
709
|
+
name: "auth",
|
|
710
|
+
description: "Manage authentication"
|
|
711
|
+
},
|
|
712
|
+
subCommands: {
|
|
713
|
+
status: statusCommand,
|
|
714
|
+
logout: logoutCommand
|
|
715
|
+
}
|
|
716
|
+
});
|
|
717
|
+
|
|
718
|
+
//#endregion
|
|
719
|
+
//#region src/commands/compare.ts
|
|
720
|
+
const SEARCH_TYPES = [
|
|
721
|
+
"web",
|
|
722
|
+
"image",
|
|
723
|
+
"video",
|
|
724
|
+
"news",
|
|
725
|
+
"discover",
|
|
726
|
+
"googleNews"
|
|
727
|
+
];
|
|
728
|
+
function formatPercent(val) {
|
|
729
|
+
if (val === void 0 || val === null || !Number.isFinite(val)) return "\x1B[90m-\x1B[0m";
|
|
730
|
+
return `${val > 0 ? "\x1B[32m" : val < 0 ? "\x1B[31m" : "\x1B[90m"}${val > 0 ? "+" : ""}${val.toFixed(1)}%\x1B[0m`;
|
|
731
|
+
}
|
|
732
|
+
function formatNumber(val) {
|
|
733
|
+
if (val === void 0 || val === null) return "-";
|
|
734
|
+
return val.toLocaleString();
|
|
735
|
+
}
|
|
736
|
+
const compareCommand = defineCommand({
|
|
737
|
+
meta: {
|
|
738
|
+
name: "compare",
|
|
739
|
+
description: "Compare site performance across periods"
|
|
740
|
+
},
|
|
741
|
+
args: {
|
|
742
|
+
site: {
|
|
743
|
+
type: "string",
|
|
744
|
+
alias: "s",
|
|
745
|
+
description: "Site URL (e.g., sc-domain:example.com)"
|
|
746
|
+
},
|
|
747
|
+
period: {
|
|
748
|
+
type: "string",
|
|
749
|
+
alias: "p",
|
|
750
|
+
default: "7d",
|
|
751
|
+
description: "Period to compare (e.g., 7d, 30d, 3m)"
|
|
752
|
+
},
|
|
753
|
+
from: {
|
|
754
|
+
type: "string",
|
|
755
|
+
description: "Custom start date (YYYY-MM-DD)"
|
|
756
|
+
},
|
|
757
|
+
to: {
|
|
758
|
+
type: "string",
|
|
759
|
+
description: "Custom end date (YYYY-MM-DD)"
|
|
760
|
+
},
|
|
761
|
+
vs: {
|
|
762
|
+
type: "string",
|
|
763
|
+
description: "Comparison period start date (YYYY-MM-DD)"
|
|
764
|
+
},
|
|
765
|
+
json: {
|
|
766
|
+
type: "boolean",
|
|
767
|
+
default: false,
|
|
768
|
+
description: "Output as JSON"
|
|
769
|
+
},
|
|
770
|
+
type: {
|
|
771
|
+
type: "string",
|
|
772
|
+
alias: "t",
|
|
773
|
+
default: "summary",
|
|
774
|
+
description: "Output type: summary, pages, keywords"
|
|
775
|
+
},
|
|
776
|
+
limit: {
|
|
777
|
+
type: "string",
|
|
778
|
+
alias: "l",
|
|
779
|
+
default: "10",
|
|
780
|
+
description: "Number of items to show (for pages/keywords)"
|
|
781
|
+
},
|
|
782
|
+
fresh: {
|
|
783
|
+
type: "boolean",
|
|
784
|
+
default: false,
|
|
785
|
+
description: "Include fresh/unfinalized data (last 3 days)"
|
|
786
|
+
},
|
|
787
|
+
searchType: {
|
|
788
|
+
type: "string",
|
|
789
|
+
default: "web",
|
|
790
|
+
description: "Search type: web, image, video, news, discover, googleNews"
|
|
791
|
+
},
|
|
792
|
+
db: {
|
|
793
|
+
type: "string",
|
|
794
|
+
alias: "d",
|
|
795
|
+
description: "Path to SQLite database for local queries"
|
|
796
|
+
},
|
|
797
|
+
source: {
|
|
798
|
+
type: "string",
|
|
799
|
+
default: "auto",
|
|
800
|
+
description: "Data source: api, db, auto"
|
|
801
|
+
},
|
|
802
|
+
quiet: {
|
|
803
|
+
type: "boolean",
|
|
804
|
+
alias: "q",
|
|
805
|
+
default: false,
|
|
806
|
+
description: "Suppress output"
|
|
807
|
+
}
|
|
808
|
+
},
|
|
809
|
+
async run({ args }) {
|
|
810
|
+
const config = await loadConfig();
|
|
811
|
+
const siteArg = args.site || config.defaultSite;
|
|
812
|
+
if (!siteArg) {
|
|
813
|
+
logger.error("Site required. Use --site or set defaultSite in config.");
|
|
814
|
+
process.exit(1);
|
|
815
|
+
}
|
|
816
|
+
const dbPath = args.db || config.defaultDb;
|
|
817
|
+
const periodArg = args.period === "7d" && config.defaultPeriod ? config.defaultPeriod : args.period;
|
|
818
|
+
const auth = await getAuth({
|
|
819
|
+
interactive: false,
|
|
820
|
+
config
|
|
821
|
+
});
|
|
822
|
+
let range;
|
|
823
|
+
if (args.from && args.to) {
|
|
824
|
+
const start = dayjs(args.from);
|
|
825
|
+
const end = dayjs(args.to);
|
|
826
|
+
if (!start.isValid() || !end.isValid()) {
|
|
827
|
+
logger.error("Invalid date format. Use YYYY-MM-DD");
|
|
828
|
+
process.exit(1);
|
|
829
|
+
}
|
|
830
|
+
const periodDays = end.diff(start, "day");
|
|
831
|
+
let prevStart;
|
|
832
|
+
let prevEnd;
|
|
833
|
+
if (args.vs) {
|
|
834
|
+
prevStart = dayjs(args.vs);
|
|
835
|
+
if (!prevStart.isValid()) {
|
|
836
|
+
logger.error("Invalid --vs date format. Use YYYY-MM-DD");
|
|
837
|
+
process.exit(1);
|
|
838
|
+
}
|
|
839
|
+
prevEnd = prevStart.add(periodDays, "day");
|
|
840
|
+
} else {
|
|
841
|
+
prevEnd = start.subtract(1, "day");
|
|
842
|
+
prevStart = prevEnd.subtract(periodDays, "day");
|
|
843
|
+
}
|
|
844
|
+
range = {
|
|
845
|
+
period: {
|
|
846
|
+
start: start.format("YYYY-MM-DD"),
|
|
847
|
+
end: end.format("YYYY-MM-DD")
|
|
848
|
+
},
|
|
849
|
+
prevPeriod: {
|
|
850
|
+
start: prevStart.format("YYYY-MM-DD"),
|
|
851
|
+
end: prevEnd.format("YYYY-MM-DD")
|
|
852
|
+
}
|
|
853
|
+
};
|
|
854
|
+
} else {
|
|
855
|
+
const period = parsePeriod(periodArg);
|
|
856
|
+
if (!period) {
|
|
857
|
+
logger.error(`Invalid period: ${periodArg}`);
|
|
858
|
+
process.exit(1);
|
|
859
|
+
}
|
|
860
|
+
const daysOffset = args.fresh ? 1 : 3;
|
|
861
|
+
const endDate = dayjs().subtract(daysOffset, "day");
|
|
862
|
+
const startDate = endDate.subtract(period.amount, period.unit);
|
|
863
|
+
const prevEndDate = startDate.subtract(1, "day");
|
|
864
|
+
const prevStartDate = prevEndDate.subtract(period.amount, period.unit);
|
|
865
|
+
range = {
|
|
866
|
+
period: {
|
|
867
|
+
start: startDate.format("YYYY-MM-DD"),
|
|
868
|
+
end: endDate.format("YYYY-MM-DD")
|
|
869
|
+
},
|
|
870
|
+
prevPeriod: {
|
|
871
|
+
start: prevStartDate.format("YYYY-MM-DD"),
|
|
872
|
+
end: prevEndDate.format("YYYY-MM-DD")
|
|
873
|
+
}
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
const searchType = SEARCH_TYPES.includes(args.searchType) ? args.searchType : "web";
|
|
877
|
+
const effectiveSource = searchType !== "web" ? "api" : args.source || "auto";
|
|
878
|
+
if (searchType !== "web" && args.source === "db") logger.warn(`Search type '${searchType}' requires API. Ignoring --source db.`);
|
|
879
|
+
const provider = await createProvider({
|
|
880
|
+
auth,
|
|
881
|
+
db: dbPath ? createGscDb(betterSqlite3({ name: path.resolve(dbPath) })).db : null,
|
|
882
|
+
source: effectiveSource,
|
|
883
|
+
siteUrls: [siteArg],
|
|
884
|
+
range
|
|
885
|
+
});
|
|
886
|
+
if (args.json) {
|
|
887
|
+
const [dates, pages, keywords] = await Promise.all([
|
|
888
|
+
provider.getDatesWithComparison(siteArg, range),
|
|
889
|
+
provider.getPagesWithComparison(siteArg, range),
|
|
890
|
+
provider.getKeywordsWithComparison(siteArg, range)
|
|
891
|
+
]).catch(gscErrorHandler);
|
|
892
|
+
console.log(JSON.stringify({
|
|
893
|
+
site: siteArg,
|
|
894
|
+
source: provider.source,
|
|
895
|
+
range,
|
|
896
|
+
dates,
|
|
897
|
+
pages,
|
|
898
|
+
keywords
|
|
899
|
+
}, null, 2));
|
|
900
|
+
return;
|
|
901
|
+
}
|
|
902
|
+
if (args.quiet) return;
|
|
903
|
+
console.log();
|
|
904
|
+
console.log(` \x1B[1m${siteArg}\x1B[0m`);
|
|
905
|
+
console.log(` \x1B[90mSource: ${provider.source}\x1B[0m`);
|
|
906
|
+
console.log(` \x1B[90mCurrent: ${range.period.start} → ${range.period.end}\x1B[0m`);
|
|
907
|
+
console.log(` \x1B[90mPrevious: ${range.prevPeriod?.start} → ${range.prevPeriod?.end}\x1B[0m`);
|
|
908
|
+
console.log();
|
|
909
|
+
if (args.type === "summary" || args.type === "all") {
|
|
910
|
+
const dates = await provider.getDatesWithComparison(siteArg, range).catch(gscErrorHandler);
|
|
911
|
+
console.log(" \x1B[1mSummary\x1B[0m");
|
|
912
|
+
console.log(` ┌─────────────┬──────────────┬──────────────┬─────────────┐`);
|
|
913
|
+
console.log(` │ │ \x1B[1mCurrent\x1B[0m │ \x1B[1mPrevious\x1B[0m │ \x1B[1mChange\x1B[0m │`);
|
|
914
|
+
console.log(` ├─────────────┼──────────────┼──────────────┼─────────────┤`);
|
|
915
|
+
console.log(` │ Clicks │ ${formatNumber(dates.metadata.totals.current.clicks).padStart(12)} │ ${formatNumber(dates.metadata.totals.previous.clicks).padStart(12)} │ ${formatPercent(dates.metadata.totals.clicksPercent).padStart(19)} │`);
|
|
916
|
+
console.log(` │ Impressions │ ${formatNumber(dates.metadata.totals.current.impressions).padStart(12)} │ ${formatNumber(dates.metadata.totals.previous.impressions).padStart(12)} │ ${formatPercent(dates.metadata.totals.impressionsPercent).padStart(19)} │`);
|
|
917
|
+
console.log(` │ CTR │ ${(dates.metadata.totals.current.ctr * 100).toFixed(2).padStart(10)}% │ ${(dates.metadata.totals.previous.ctr * 100).toFixed(2).padStart(10)}% │ ${formatPercent(dates.metadata.totals.ctrPercent).padStart(19)} │`);
|
|
918
|
+
console.log(` │ Position │ ${dates.metadata.totals.current.position.toFixed(1).padStart(12)} │ ${dates.metadata.totals.previous.position.toFixed(1).padStart(12)} │ ${formatPercent(dates.metadata.totals.positionPercent).padStart(19)} │`);
|
|
919
|
+
console.log(` └─────────────┴──────────────┴──────────────┴─────────────┘`);
|
|
920
|
+
console.log();
|
|
921
|
+
}
|
|
922
|
+
const limit = Number.parseInt(args.limit, 10) || 10;
|
|
923
|
+
if (args.type === "pages" || args.type === "all") {
|
|
924
|
+
const pages = await provider.getPagesWithComparison(siteArg, range).catch(gscErrorHandler);
|
|
925
|
+
const topPages = pages.current.slice(0, limit);
|
|
926
|
+
const lostPages = pages.previous.filter((p) => p.lost).slice(0, 5);
|
|
927
|
+
console.log(` \x1B[1mTop Pages\x1B[0m \x1B[90m(${pages.current.length} total)\x1B[0m`);
|
|
928
|
+
for (const p of topPages) {
|
|
929
|
+
const pagePath = p.page.replace(/^https?:\/\/[^/]+/, "") || "/";
|
|
930
|
+
console.log(` ${pagePath.slice(0, 50).padEnd(50)} ${formatNumber(p.clicks).padStart(8)} clicks ${formatPercent(p.clicksPercent)}`);
|
|
931
|
+
}
|
|
932
|
+
if (lostPages.length > 0) {
|
|
933
|
+
console.log();
|
|
934
|
+
console.log(` \x1B[1mLost Pages\x1B[0m \x1B[90m(had traffic in previous period)\x1B[0m`);
|
|
935
|
+
for (const p of lostPages) {
|
|
936
|
+
const pagePath = p.page.replace(/^https?:\/\/[^/]+/, "") || "/";
|
|
937
|
+
console.log(` \x1B[31m${pagePath.slice(0, 50).padEnd(50)}\x1B[0m ${formatNumber(p.prevClicks).padStart(8)} prev clicks`);
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
console.log();
|
|
941
|
+
}
|
|
942
|
+
if (args.type === "keywords" || args.type === "all") {
|
|
943
|
+
const keywords = await provider.getKeywordsWithComparison(siteArg, range).catch(gscErrorHandler);
|
|
944
|
+
const topKeywords = keywords.current.slice(0, limit);
|
|
945
|
+
const lostKeywords = keywords.previous.filter((k) => k.lost).slice(0, 5);
|
|
946
|
+
console.log(` \x1B[1mTop Keywords\x1B[0m \x1B[90m(${keywords.current.length} total)\x1B[0m`);
|
|
947
|
+
for (const k of topKeywords) {
|
|
948
|
+
const posChange = k.prevPosition ? formatPercent(-((k.position || 0) - k.prevPosition) / k.prevPosition * 100) : "";
|
|
949
|
+
console.log(` ${k.keyword.slice(0, 40).padEnd(40)} pos ${(k.position || 0).toFixed(1).padStart(5)} ${posChange} ${formatNumber(k.clicks).padStart(6)} clicks`);
|
|
950
|
+
}
|
|
951
|
+
if (lostKeywords.length > 0) {
|
|
952
|
+
console.log();
|
|
953
|
+
console.log(` \x1B[1mLost Keywords\x1B[0m \x1B[90m(had traffic in previous period)\x1B[0m`);
|
|
954
|
+
for (const k of lostKeywords) console.log(` \x1B[31m${k.keyword.slice(0, 40).padEnd(40)}\x1B[0m pos ${(k.prevPosition || 0).toFixed(1).padStart(5)} ${formatNumber(k.prevClicks).padStart(6)} prev clicks`);
|
|
955
|
+
}
|
|
956
|
+
console.log();
|
|
957
|
+
}
|
|
958
|
+
logger.success("Comparison complete");
|
|
959
|
+
}
|
|
960
|
+
});
|
|
961
|
+
|
|
962
|
+
//#endregion
|
|
963
|
+
//#region src/commands/config.ts
|
|
964
|
+
const showCommand = defineCommand({
|
|
965
|
+
meta: {
|
|
966
|
+
name: "show",
|
|
967
|
+
description: "Show current config"
|
|
968
|
+
},
|
|
969
|
+
async run() {
|
|
970
|
+
const config = await loadConfig();
|
|
971
|
+
const configPath = getConfigPath();
|
|
972
|
+
logger.info(`Config: ${configPath}`);
|
|
973
|
+
console.log();
|
|
974
|
+
if (Object.keys(config).length === 0) {
|
|
975
|
+
logger.warn("No config set");
|
|
976
|
+
return;
|
|
977
|
+
}
|
|
978
|
+
console.log(JSON.stringify(config, null, 2));
|
|
979
|
+
}
|
|
980
|
+
});
|
|
981
|
+
const setCommand = defineCommand({
|
|
982
|
+
meta: {
|
|
983
|
+
name: "set",
|
|
984
|
+
description: "Set a config value"
|
|
985
|
+
},
|
|
986
|
+
args: {
|
|
987
|
+
key: {
|
|
988
|
+
type: "positional",
|
|
989
|
+
description: "Config key (defaultSite, defaultPeriod, defaultFormat, defaultDb)",
|
|
990
|
+
required: true
|
|
991
|
+
},
|
|
992
|
+
value: {
|
|
993
|
+
type: "positional",
|
|
994
|
+
description: "Value to set",
|
|
995
|
+
required: true
|
|
996
|
+
}
|
|
997
|
+
},
|
|
998
|
+
async run({ args }) {
|
|
999
|
+
const validKeys = [
|
|
1000
|
+
"defaultSite",
|
|
1001
|
+
"defaultPeriod",
|
|
1002
|
+
"defaultFormat",
|
|
1003
|
+
"defaultDb"
|
|
1004
|
+
];
|
|
1005
|
+
if (!validKeys.includes(args.key)) {
|
|
1006
|
+
logger.error(`Invalid key: ${args.key}`);
|
|
1007
|
+
logger.info(`Valid keys: ${validKeys.join(", ")}`);
|
|
1008
|
+
process.exit(1);
|
|
1009
|
+
}
|
|
1010
|
+
const config = await loadConfig();
|
|
1011
|
+
config[args.key] = args.value;
|
|
1012
|
+
await saveConfig(config);
|
|
1013
|
+
logger.success(`Set ${args.key} = ${args.value}`);
|
|
1014
|
+
}
|
|
1015
|
+
});
|
|
1016
|
+
const unsetCommand = defineCommand({
|
|
1017
|
+
meta: {
|
|
1018
|
+
name: "unset",
|
|
1019
|
+
description: "Remove a config value"
|
|
1020
|
+
},
|
|
1021
|
+
args: { key: {
|
|
1022
|
+
type: "positional",
|
|
1023
|
+
description: "Config key to remove",
|
|
1024
|
+
required: true
|
|
1025
|
+
} },
|
|
1026
|
+
async run({ args }) {
|
|
1027
|
+
const config = await loadConfig();
|
|
1028
|
+
delete config[args.key];
|
|
1029
|
+
await saveConfig(config);
|
|
1030
|
+
logger.success(`Removed ${args.key}`);
|
|
1031
|
+
}
|
|
1032
|
+
});
|
|
1033
|
+
const pathCommand = defineCommand({
|
|
1034
|
+
meta: {
|
|
1035
|
+
name: "path",
|
|
1036
|
+
description: "Show config file path"
|
|
1037
|
+
},
|
|
1038
|
+
run() {
|
|
1039
|
+
console.log(getConfigPath());
|
|
1040
|
+
}
|
|
1041
|
+
});
|
|
1042
|
+
const configCommand = defineCommand({
|
|
1043
|
+
meta: {
|
|
1044
|
+
name: "config",
|
|
1045
|
+
description: "Manage configuration"
|
|
1046
|
+
},
|
|
1047
|
+
subCommands: {
|
|
1048
|
+
show: showCommand,
|
|
1049
|
+
set: setCommand,
|
|
1050
|
+
unset: unsetCommand,
|
|
1051
|
+
path: pathCommand
|
|
1052
|
+
}
|
|
1053
|
+
});
|
|
1054
|
+
|
|
1055
|
+
//#endregion
|
|
1056
|
+
//#region src/commands/dump.ts
|
|
1057
|
+
const DUMP_DATA_TYPES = [
|
|
1058
|
+
"pages",
|
|
1059
|
+
"keywords",
|
|
1060
|
+
"countries",
|
|
1061
|
+
"devices"
|
|
1062
|
+
];
|
|
1063
|
+
async function getSites(client) {
|
|
1064
|
+
return (await fetchSites(client)).filter((site) => site.siteUrl && site.permissionLevel !== "siteUnverifiedUser").map((site) => site.siteUrl);
|
|
1065
|
+
}
|
|
1066
|
+
async function runDump(provider, selectedSites, dataTypes, period, format, outputFile, options = {}) {
|
|
1067
|
+
const daysOffset = options.fresh ? 1 : 3;
|
|
1068
|
+
const endDate = dayjs().subtract(daysOffset, "days").format("YYYY-MM-DD");
|
|
1069
|
+
const startDate = dayjs().subtract(period.amount, period.unit).subtract(daysOffset, "days").format("YYYY-MM-DD");
|
|
1070
|
+
const range = {
|
|
1071
|
+
period: {
|
|
1072
|
+
start: startDate,
|
|
1073
|
+
end: endDate
|
|
1074
|
+
},
|
|
1075
|
+
prevPeriod: {
|
|
1076
|
+
start: dayjs(startDate).subtract(period.amount, period.unit).format("YYYY-MM-DD"),
|
|
1077
|
+
end: dayjs(endDate).subtract(period.amount, period.unit).format("YYYY-MM-DD")
|
|
1078
|
+
}
|
|
1079
|
+
};
|
|
1080
|
+
for (const siteUrl of selectedSites) {
|
|
1081
|
+
const siteName = siteUrl.replace(/https?:\/\//, "").replace(/\/$/, "");
|
|
1082
|
+
const output = {
|
|
1083
|
+
siteUrl,
|
|
1084
|
+
source: provider.source,
|
|
1085
|
+
dateRange: {
|
|
1086
|
+
startDate,
|
|
1087
|
+
endDate
|
|
1088
|
+
},
|
|
1089
|
+
period: {
|
|
1090
|
+
amount: period.amount,
|
|
1091
|
+
unit: period.unit
|
|
1092
|
+
},
|
|
1093
|
+
dataTypes,
|
|
1094
|
+
fresh: options.fresh || false,
|
|
1095
|
+
searchType: options.searchType || "web"
|
|
1096
|
+
};
|
|
1097
|
+
const totalSteps = dataTypes.length;
|
|
1098
|
+
let currentStep = 0;
|
|
1099
|
+
for (const dataType of dataTypes) {
|
|
1100
|
+
currentStep++;
|
|
1101
|
+
clearLine();
|
|
1102
|
+
process.stdout.write(progressBar(currentStep, totalSteps, `${dataType} (${siteName})`));
|
|
1103
|
+
if (dataType === "pages") {
|
|
1104
|
+
const pages = await provider.getPages(siteUrl, range);
|
|
1105
|
+
output.pages = {
|
|
1106
|
+
total: pages.length,
|
|
1107
|
+
data: pages
|
|
1108
|
+
};
|
|
1109
|
+
} else if (dataType === "keywords") {
|
|
1110
|
+
const keywordsData = await provider.getKeywordsWithComparison(siteUrl, range);
|
|
1111
|
+
output.keywords = {
|
|
1112
|
+
total: keywordsData.current.length,
|
|
1113
|
+
current: keywordsData.current,
|
|
1114
|
+
previous: keywordsData.previous,
|
|
1115
|
+
metadata: keywordsData.metadata
|
|
1116
|
+
};
|
|
1117
|
+
} else if (dataType === "countries") {
|
|
1118
|
+
const countriesData = await provider.getCountriesWithComparison(siteUrl, range);
|
|
1119
|
+
output.countries = {
|
|
1120
|
+
current: countriesData.current,
|
|
1121
|
+
previous: countriesData.previous,
|
|
1122
|
+
metadata: countriesData.metadata
|
|
1123
|
+
};
|
|
1124
|
+
} else if (dataType === "devices") {
|
|
1125
|
+
const devicesData = await provider.getDevicesWithComparison(siteUrl, range);
|
|
1126
|
+
output.devices = {
|
|
1127
|
+
current: devicesData.current,
|
|
1128
|
+
previous: devicesData.previous,
|
|
1129
|
+
metadata: devicesData.metadata
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
clearLine();
|
|
1134
|
+
const ext = format === "csv" ? "csv" : "json";
|
|
1135
|
+
const filename = outputFile || `gsc-${siteName.replace(/\//g, "_")}-${dataTypes.join("-")}-${period.amount}${period.unit[0]}-${endDate}.${ext}`;
|
|
1136
|
+
const content = format === "csv" ? exportToCSV(output) : JSON.stringify(output, null, 2);
|
|
1137
|
+
await fs.writeFile(filename, content);
|
|
1138
|
+
const totalItems = (output.pages?.total || 0) + (output.keywords?.total || 0) + (output.countries?.current?.length || 0) + (output.devices?.current?.length || 0);
|
|
1139
|
+
logger.success(`Saved ${totalItems} items to ${filename} (source: ${provider.source})`);
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
async function interactiveMode(auth, dbPath, source) {
|
|
1143
|
+
process.stdout.write(" Fetching sites...");
|
|
1144
|
+
const sites = await getSites(googleSearchConsole(auth));
|
|
1145
|
+
clearLine();
|
|
1146
|
+
logger.success(`Found ${sites.length} sites`);
|
|
1147
|
+
if (sites.length === 0) {
|
|
1148
|
+
cancel("No sites found in your Google Search Console account.");
|
|
1149
|
+
return;
|
|
1150
|
+
}
|
|
1151
|
+
const selectedSites = await multiselect({
|
|
1152
|
+
message: "Select sites to dump data from:",
|
|
1153
|
+
options: sites.map((site) => ({
|
|
1154
|
+
value: site,
|
|
1155
|
+
label: site
|
|
1156
|
+
}))
|
|
1157
|
+
});
|
|
1158
|
+
if (isCancel(selectedSites) || selectedSites.length === 0) {
|
|
1159
|
+
cancel("Operation cancelled.");
|
|
1160
|
+
return;
|
|
1161
|
+
}
|
|
1162
|
+
const dataTypes = await multiselect({
|
|
1163
|
+
message: "What data would you like to dump?",
|
|
1164
|
+
options: [
|
|
1165
|
+
{
|
|
1166
|
+
value: "pages",
|
|
1167
|
+
label: "Pages (URLs and performance data)",
|
|
1168
|
+
hint: "recommended"
|
|
1169
|
+
},
|
|
1170
|
+
{
|
|
1171
|
+
value: "keywords",
|
|
1172
|
+
label: "Keywords (search queries and rankings)"
|
|
1173
|
+
},
|
|
1174
|
+
{
|
|
1175
|
+
value: "countries",
|
|
1176
|
+
label: "Countries (geographic performance)"
|
|
1177
|
+
},
|
|
1178
|
+
{
|
|
1179
|
+
value: "devices",
|
|
1180
|
+
label: "Devices (desktop, mobile, tablet)"
|
|
1181
|
+
}
|
|
1182
|
+
],
|
|
1183
|
+
required: true
|
|
1184
|
+
});
|
|
1185
|
+
if (isCancel(dataTypes) || dataTypes.length === 0) {
|
|
1186
|
+
cancel("Operation cancelled.");
|
|
1187
|
+
return;
|
|
1188
|
+
}
|
|
1189
|
+
const periodInput = await text({
|
|
1190
|
+
message: "Time period (e.g., 90d, 6m, 1y):",
|
|
1191
|
+
placeholder: "180d",
|
|
1192
|
+
defaultValue: "180d",
|
|
1193
|
+
validate: (value) => {
|
|
1194
|
+
if (!value) return void 0;
|
|
1195
|
+
return parsePeriod(value) ? void 0 : "Invalid format. Use: 90d, 6m, 1y";
|
|
1196
|
+
}
|
|
1197
|
+
});
|
|
1198
|
+
if (isCancel(periodInput)) {
|
|
1199
|
+
cancel("Operation cancelled.");
|
|
1200
|
+
return;
|
|
1201
|
+
}
|
|
1202
|
+
const formatResult = await multiselect({
|
|
1203
|
+
message: "Output format:",
|
|
1204
|
+
options: [{
|
|
1205
|
+
value: "json",
|
|
1206
|
+
label: "JSON",
|
|
1207
|
+
hint: "structured data"
|
|
1208
|
+
}, {
|
|
1209
|
+
value: "csv",
|
|
1210
|
+
label: "CSV",
|
|
1211
|
+
hint: "spreadsheet compatible"
|
|
1212
|
+
}],
|
|
1213
|
+
required: true
|
|
1214
|
+
});
|
|
1215
|
+
if (isCancel(formatResult)) {
|
|
1216
|
+
cancel("Operation cancelled.");
|
|
1217
|
+
return;
|
|
1218
|
+
}
|
|
1219
|
+
let finalSource = source;
|
|
1220
|
+
if (dbPath && source === "auto") {
|
|
1221
|
+
const sourceResult = await select({
|
|
1222
|
+
message: "Data source:",
|
|
1223
|
+
options: [
|
|
1224
|
+
{
|
|
1225
|
+
value: "auto",
|
|
1226
|
+
label: "Auto",
|
|
1227
|
+
hint: "use DB if data exists, else API"
|
|
1228
|
+
},
|
|
1229
|
+
{
|
|
1230
|
+
value: "api",
|
|
1231
|
+
label: "API",
|
|
1232
|
+
hint: "always fetch from Google"
|
|
1233
|
+
},
|
|
1234
|
+
{
|
|
1235
|
+
value: "db",
|
|
1236
|
+
label: "Database",
|
|
1237
|
+
hint: "use synced data"
|
|
1238
|
+
}
|
|
1239
|
+
]
|
|
1240
|
+
});
|
|
1241
|
+
if (isCancel(sourceResult)) {
|
|
1242
|
+
cancel("Operation cancelled.");
|
|
1243
|
+
return;
|
|
1244
|
+
}
|
|
1245
|
+
finalSource = sourceResult;
|
|
1246
|
+
}
|
|
1247
|
+
const format = formatResult[0];
|
|
1248
|
+
const period = parsePeriod(periodInput || "180d");
|
|
1249
|
+
const ready = await confirm({ message: `Dump ${dataTypes.join(", ")} for ${selectedSites.length} site(s), ${period.amount}${period.unit[0]}, as ${format.toUpperCase()}?` });
|
|
1250
|
+
if (isCancel(ready) || !ready) {
|
|
1251
|
+
cancel("Operation cancelled.");
|
|
1252
|
+
return;
|
|
1253
|
+
}
|
|
1254
|
+
console.log();
|
|
1255
|
+
const endDate = dayjs().subtract(3, "days").format("YYYY-MM-DD");
|
|
1256
|
+
const startDate = dayjs().subtract(period.amount, period.unit).subtract(3, "days").format("YYYY-MM-DD");
|
|
1257
|
+
const range = {
|
|
1258
|
+
period: {
|
|
1259
|
+
start: startDate,
|
|
1260
|
+
end: endDate
|
|
1261
|
+
},
|
|
1262
|
+
prevPeriod: {
|
|
1263
|
+
start: dayjs(startDate).subtract(period.amount, period.unit).format("YYYY-MM-DD"),
|
|
1264
|
+
end: dayjs(endDate).subtract(period.amount, period.unit).format("YYYY-MM-DD")
|
|
1265
|
+
}
|
|
1266
|
+
};
|
|
1267
|
+
let db = null;
|
|
1268
|
+
if (dbPath) {
|
|
1269
|
+
if (await fs.access(dbPath).then(() => true).catch(() => false)) db = createGscDb(betterSqlite3({ name: path.resolve(dbPath) })).db;
|
|
1270
|
+
}
|
|
1271
|
+
const provider = await createProvider({
|
|
1272
|
+
auth,
|
|
1273
|
+
db,
|
|
1274
|
+
source: finalSource,
|
|
1275
|
+
siteUrls: selectedSites,
|
|
1276
|
+
range
|
|
1277
|
+
});
|
|
1278
|
+
logger.info(`Using ${provider.source.toUpperCase()} as data source`);
|
|
1279
|
+
await runDump(provider, selectedSites, dataTypes, period, format, null);
|
|
1280
|
+
}
|
|
1281
|
+
async function nonInteractiveMode(auth, site, data, periodStr, format, output, dbPath, source, options = {}) {
|
|
1282
|
+
if (site.length === 0) {
|
|
1283
|
+
logger.error("--site required in non-interactive mode");
|
|
1284
|
+
process.exit(1);
|
|
1285
|
+
}
|
|
1286
|
+
if (data.length === 0) {
|
|
1287
|
+
logger.error("--data required in non-interactive mode");
|
|
1288
|
+
process.exit(1);
|
|
1289
|
+
}
|
|
1290
|
+
const invalidData = data.filter((d) => !DUMP_DATA_TYPES.includes(d));
|
|
1291
|
+
if (invalidData.length > 0) {
|
|
1292
|
+
logger.error(`Invalid data types: ${invalidData.join(", ")}`);
|
|
1293
|
+
logger.info("Valid types: pages, keywords, countries, devices");
|
|
1294
|
+
process.exit(1);
|
|
1295
|
+
}
|
|
1296
|
+
const period = parsePeriod(periodStr);
|
|
1297
|
+
if (!period) {
|
|
1298
|
+
logger.error(`Invalid period: ${periodStr}`);
|
|
1299
|
+
logger.info("Examples: 90d, 6m, 1y");
|
|
1300
|
+
process.exit(1);
|
|
1301
|
+
}
|
|
1302
|
+
process.stdout.write(" Validating sites...");
|
|
1303
|
+
const availableSites = await getSites(googleSearchConsole(auth));
|
|
1304
|
+
clearLine();
|
|
1305
|
+
const normalizedSites = [];
|
|
1306
|
+
for (const s of site) {
|
|
1307
|
+
const match = availableSites.find((avail) => avail === s || avail === `https://${s}` || avail === `http://${s}` || avail === `sc-domain:${s}` || avail.replace(/https?:\/\//, "").replace(/\/$/, "") === s.replace(/\/$/, ""));
|
|
1308
|
+
if (!match) {
|
|
1309
|
+
logger.error(`Site not found: ${s}`);
|
|
1310
|
+
logger.info("Available sites:");
|
|
1311
|
+
availableSites.slice(0, 5).forEach((a) => console.log(` - ${a}`));
|
|
1312
|
+
if (availableSites.length > 5) console.log(` ... and ${availableSites.length - 5} more`);
|
|
1313
|
+
process.exit(1);
|
|
1314
|
+
}
|
|
1315
|
+
normalizedSites.push(match);
|
|
1316
|
+
}
|
|
1317
|
+
const daysOffset = options.fresh ? 1 : 3;
|
|
1318
|
+
const endDate = dayjs().subtract(daysOffset, "days").format("YYYY-MM-DD");
|
|
1319
|
+
const startDate = dayjs().subtract(period.amount, period.unit).subtract(daysOffset, "days").format("YYYY-MM-DD");
|
|
1320
|
+
const range = {
|
|
1321
|
+
period: {
|
|
1322
|
+
start: startDate,
|
|
1323
|
+
end: endDate
|
|
1324
|
+
},
|
|
1325
|
+
prevPeriod: {
|
|
1326
|
+
start: dayjs(startDate).subtract(period.amount, period.unit).format("YYYY-MM-DD"),
|
|
1327
|
+
end: dayjs(endDate).subtract(period.amount, period.unit).format("YYYY-MM-DD")
|
|
1328
|
+
}
|
|
1329
|
+
};
|
|
1330
|
+
let db = null;
|
|
1331
|
+
if (dbPath) {
|
|
1332
|
+
if (await fs.access(dbPath).then(() => true).catch(() => false)) db = createGscDb(betterSqlite3({ name: path.resolve(dbPath) })).db;
|
|
1333
|
+
else if (source === "db") {
|
|
1334
|
+
logger.error(`Database not found: ${dbPath}`);
|
|
1335
|
+
logger.info("Run 'gscdump sync' first to create the database.");
|
|
1336
|
+
process.exit(1);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
const provider = await createProvider({
|
|
1340
|
+
auth,
|
|
1341
|
+
db,
|
|
1342
|
+
source,
|
|
1343
|
+
siteUrls: normalizedSites,
|
|
1344
|
+
range
|
|
1345
|
+
}).catch(gscErrorHandler);
|
|
1346
|
+
const extras = [];
|
|
1347
|
+
if (options.fresh) extras.push("fresh");
|
|
1348
|
+
if (options.searchType && options.searchType !== "web") extras.push(options.searchType);
|
|
1349
|
+
const extrasStr = extras.length ? ` [${extras.join(", ")}]` : "";
|
|
1350
|
+
logger.info(`${normalizedSites.length} site(s), ${data.join("+")} data, ${period.amount}${period.unit[0]}, ${format}, source: ${provider.source}${extrasStr}`);
|
|
1351
|
+
console.log();
|
|
1352
|
+
await runDump(provider, normalizedSites, data, period, format, output, options);
|
|
1353
|
+
}
|
|
1354
|
+
const dumpCommand = defineCommand({
|
|
1355
|
+
meta: {
|
|
1356
|
+
name: "dump",
|
|
1357
|
+
description: "Export GSC data to JSON or CSV files"
|
|
1358
|
+
},
|
|
1359
|
+
args: {
|
|
1360
|
+
site: {
|
|
1361
|
+
type: "string",
|
|
1362
|
+
alias: "s",
|
|
1363
|
+
description: "Site URL (comma-separated for multiple)"
|
|
1364
|
+
},
|
|
1365
|
+
data: {
|
|
1366
|
+
type: "string",
|
|
1367
|
+
alias: "d",
|
|
1368
|
+
description: "Data types: pages,keywords,countries,devices"
|
|
1369
|
+
},
|
|
1370
|
+
period: {
|
|
1371
|
+
type: "string",
|
|
1372
|
+
alias: "p",
|
|
1373
|
+
default: "180d",
|
|
1374
|
+
description: "Time period: 90d, 6m, 1y"
|
|
1375
|
+
},
|
|
1376
|
+
format: {
|
|
1377
|
+
type: "string",
|
|
1378
|
+
alias: "f",
|
|
1379
|
+
default: "json",
|
|
1380
|
+
description: "Output format: json, csv"
|
|
1381
|
+
},
|
|
1382
|
+
output: {
|
|
1383
|
+
type: "string",
|
|
1384
|
+
alias: "o",
|
|
1385
|
+
description: "Output filename"
|
|
1386
|
+
},
|
|
1387
|
+
source: {
|
|
1388
|
+
type: "string",
|
|
1389
|
+
default: "auto",
|
|
1390
|
+
description: "Data source: auto, api, db"
|
|
1391
|
+
},
|
|
1392
|
+
db: {
|
|
1393
|
+
type: "string",
|
|
1394
|
+
description: "SQLite database path (for db/auto source)"
|
|
1395
|
+
},
|
|
1396
|
+
fresh: {
|
|
1397
|
+
type: "boolean",
|
|
1398
|
+
default: false,
|
|
1399
|
+
description: "Include fresh/unfinalized data (last 3 days)"
|
|
1400
|
+
},
|
|
1401
|
+
type: {
|
|
1402
|
+
type: "string",
|
|
1403
|
+
alias: "t",
|
|
1404
|
+
default: "web",
|
|
1405
|
+
description: "Search type: web, image, video, news, discover, googleNews"
|
|
1406
|
+
}
|
|
1407
|
+
},
|
|
1408
|
+
async run({ args }) {
|
|
1409
|
+
const config = await loadConfig();
|
|
1410
|
+
const siteArg = args.site || config.defaultSite;
|
|
1411
|
+
const sites = siteArg ? siteArg.split(",") : [];
|
|
1412
|
+
const data = args.data ? args.data.split(",") : [];
|
|
1413
|
+
const periodArg = args.period === "180d" && config.defaultPeriod ? config.defaultPeriod : args.period;
|
|
1414
|
+
const formatArg = args.format === "json" && config.defaultFormat ? config.defaultFormat : args.format;
|
|
1415
|
+
const dbPath = args.db || config.defaultDb || null;
|
|
1416
|
+
const source = [
|
|
1417
|
+
"auto",
|
|
1418
|
+
"api",
|
|
1419
|
+
"db"
|
|
1420
|
+
].includes(args.source) ? args.source : "auto";
|
|
1421
|
+
const isInteractive = sites.length === 0 && data.length === 0;
|
|
1422
|
+
const { getAuth: getAuth$1 } = await Promise.resolve().then(() => auth_exports);
|
|
1423
|
+
const auth = await getAuth$1({
|
|
1424
|
+
interactive: isInteractive,
|
|
1425
|
+
config
|
|
1426
|
+
});
|
|
1427
|
+
const searchType = [
|
|
1428
|
+
"web",
|
|
1429
|
+
"image",
|
|
1430
|
+
"video",
|
|
1431
|
+
"news",
|
|
1432
|
+
"discover",
|
|
1433
|
+
"googleNews"
|
|
1434
|
+
].includes(args.type) ? args.type : "web";
|
|
1435
|
+
const options = {
|
|
1436
|
+
fresh: args.fresh,
|
|
1437
|
+
searchType
|
|
1438
|
+
};
|
|
1439
|
+
if (isInteractive) await interactiveMode(auth, dbPath, source);
|
|
1440
|
+
else await nonInteractiveMode(auth, sites, data, periodArg, formatArg === "csv" ? "csv" : "json", args.output || null, dbPath, source, options);
|
|
1441
|
+
logger.success("Done!");
|
|
1442
|
+
}
|
|
1443
|
+
});
|
|
1444
|
+
|
|
1445
|
+
//#endregion
|
|
1446
|
+
//#region src/commands/indexing.ts
|
|
1447
|
+
const inspectCommand = defineCommand({
|
|
1448
|
+
meta: {
|
|
1449
|
+
name: "inspect",
|
|
1450
|
+
description: "Inspect URLs to check their indexing status (shorthand for `gscdump index inspect`)"
|
|
1451
|
+
},
|
|
1452
|
+
args: {
|
|
1453
|
+
db: {
|
|
1454
|
+
type: "string",
|
|
1455
|
+
alias: "d",
|
|
1456
|
+
default: "./gscdump.db",
|
|
1457
|
+
description: "SQLite database path"
|
|
1458
|
+
},
|
|
1459
|
+
site: {
|
|
1460
|
+
type: "string",
|
|
1461
|
+
alias: "s",
|
|
1462
|
+
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1463
|
+
},
|
|
1464
|
+
limit: {
|
|
1465
|
+
type: "string",
|
|
1466
|
+
alias: "l",
|
|
1467
|
+
default: "100",
|
|
1468
|
+
description: "Max URLs to inspect"
|
|
1469
|
+
},
|
|
1470
|
+
delay: {
|
|
1471
|
+
type: "string",
|
|
1472
|
+
default: "200",
|
|
1473
|
+
description: "Delay between requests (ms)"
|
|
1474
|
+
},
|
|
1475
|
+
quiet: {
|
|
1476
|
+
type: "boolean",
|
|
1477
|
+
alias: "q",
|
|
1478
|
+
default: false,
|
|
1479
|
+
description: "Suppress progress output"
|
|
1480
|
+
},
|
|
1481
|
+
json: {
|
|
1482
|
+
type: "boolean",
|
|
1483
|
+
default: false,
|
|
1484
|
+
description: "Output as JSON"
|
|
1485
|
+
}
|
|
1486
|
+
},
|
|
1487
|
+
async run({ args }) {
|
|
1488
|
+
const config = await loadConfig();
|
|
1489
|
+
const dbPath = String(args.db === "./gscdump.db" && config.defaultDb ? config.defaultDb : args.db);
|
|
1490
|
+
const siteArg = String(args.site || config.defaultSite || "");
|
|
1491
|
+
if (!siteArg) {
|
|
1492
|
+
logger.error("Site required. Use --site or set defaultSite in config.");
|
|
1493
|
+
process.exit(1);
|
|
1494
|
+
}
|
|
1495
|
+
const { getAuth: getAuth$1 } = await Promise.resolve().then(() => auth_exports);
|
|
1496
|
+
const client = googleSearchConsole(await getAuth$1({
|
|
1497
|
+
interactive: false,
|
|
1498
|
+
config
|
|
1499
|
+
}));
|
|
1500
|
+
const { db, db0 } = createGscDb(betterSqlite3({ name: path.resolve(dbPath) }));
|
|
1501
|
+
await setupSchema(db0);
|
|
1502
|
+
await syncSites(db, client).catch(gscErrorHandler);
|
|
1503
|
+
const siteRecord = await getSiteByProperty(db, siteArg);
|
|
1504
|
+
if (!siteRecord) {
|
|
1505
|
+
logger.error(`Site not found: ${siteArg}`);
|
|
1506
|
+
process.exit(1);
|
|
1507
|
+
}
|
|
1508
|
+
const siteId = siteRecord.site_id || siteRecord.siteId;
|
|
1509
|
+
const limit = Number.parseInt(String(args.limit), 10);
|
|
1510
|
+
const delayMs = Number.parseInt(String(args.delay), 10);
|
|
1511
|
+
const paths = await db.selectDistinct({ path: sitePathDateAnalytics.path }).from(sitePathDateAnalytics).where(eq(sitePathDateAnalytics.siteId, siteId)).limit(limit);
|
|
1512
|
+
if (paths.length === 0) {
|
|
1513
|
+
logger.warn("No URLs found. Run `gscdump sync` first.");
|
|
1514
|
+
return;
|
|
1515
|
+
}
|
|
1516
|
+
if (!args.quiet && !args.json) logger.info(`Inspecting ${paths.length} URLs...`);
|
|
1517
|
+
const results = [];
|
|
1518
|
+
const stats = await batchInspectUrls(db, client, siteId, siteArg, paths.map((p) => p.path), {
|
|
1519
|
+
delayMs,
|
|
1520
|
+
onProgress: (result, i, total) => {
|
|
1521
|
+
results.push(result);
|
|
1522
|
+
if (!args.quiet && !args.json) {
|
|
1523
|
+
clearLine();
|
|
1524
|
+
process.stdout.write(progressBar(i + 1, total, result.path.slice(0, 40)));
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
}).catch(gscErrorHandler);
|
|
1528
|
+
if (!args.quiet && !args.json) clearLine();
|
|
1529
|
+
if (args.json) console.log(JSON.stringify({
|
|
1530
|
+
stats,
|
|
1531
|
+
results
|
|
1532
|
+
}, null, 2));
|
|
1533
|
+
else {
|
|
1534
|
+
console.log();
|
|
1535
|
+
logger.success(`Inspected ${paths.length} URLs`);
|
|
1536
|
+
console.log(` Indexed: ${stats.indexed}`);
|
|
1537
|
+
console.log(` Not Indexed: ${stats.notIndexed}`);
|
|
1538
|
+
console.log(` Errors: ${stats.errors}`);
|
|
1539
|
+
console.log();
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
});
|
|
1543
|
+
const indexingCommand = defineCommand({
|
|
1544
|
+
meta: {
|
|
1545
|
+
name: "index",
|
|
1546
|
+
description: "Manage URL indexing via Google Indexing API"
|
|
1547
|
+
},
|
|
1548
|
+
subCommands: {
|
|
1549
|
+
status: defineCommand({
|
|
1550
|
+
meta: {
|
|
1551
|
+
name: "status",
|
|
1552
|
+
description: "Show indexing status for URLs"
|
|
1553
|
+
},
|
|
1554
|
+
args: {
|
|
1555
|
+
db: {
|
|
1556
|
+
type: "string",
|
|
1557
|
+
alias: "d",
|
|
1558
|
+
default: "./gscdump.db",
|
|
1559
|
+
description: "SQLite database path"
|
|
1560
|
+
},
|
|
1561
|
+
site: {
|
|
1562
|
+
type: "string",
|
|
1563
|
+
alias: "s",
|
|
1564
|
+
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1565
|
+
},
|
|
1566
|
+
json: {
|
|
1567
|
+
type: "boolean",
|
|
1568
|
+
default: false,
|
|
1569
|
+
description: "Output as JSON"
|
|
1570
|
+
}
|
|
1571
|
+
},
|
|
1572
|
+
async run({ args }) {
|
|
1573
|
+
const config = await loadConfig();
|
|
1574
|
+
const dbPath = String(args.db === "./gscdump.db" && config.defaultDb ? config.defaultDb : args.db);
|
|
1575
|
+
const siteArg = String(args.site || config.defaultSite || "");
|
|
1576
|
+
if (!siteArg) {
|
|
1577
|
+
logger.error("Site required. Use --site or set defaultSite in config.");
|
|
1578
|
+
process.exit(1);
|
|
1579
|
+
}
|
|
1580
|
+
const { db, db0 } = createGscDb(betterSqlite3({ name: path.resolve(dbPath) }));
|
|
1581
|
+
await setupSchema(db0);
|
|
1582
|
+
const siteRecord = await getSiteByProperty(db, siteArg);
|
|
1583
|
+
if (!siteRecord) {
|
|
1584
|
+
logger.error(`Site not found: ${siteArg}`);
|
|
1585
|
+
process.exit(1);
|
|
1586
|
+
}
|
|
1587
|
+
const stats = await getIndexingStats(db, siteRecord.site_id || siteRecord.siteId);
|
|
1588
|
+
if (args.json) console.log(JSON.stringify(stats, null, 2));
|
|
1589
|
+
else {
|
|
1590
|
+
console.log();
|
|
1591
|
+
logger.info(`Indexing Status for ${siteArg}`);
|
|
1592
|
+
console.log();
|
|
1593
|
+
console.log(` Total URLs: ${stats.total}`);
|
|
1594
|
+
console.log(` Indexed: ${stats.indexed} (${stats.total ? Math.round(stats.indexed / stats.total * 100) : 0}%)`);
|
|
1595
|
+
console.log(` Not Indexed: ${stats.notIndexed}`);
|
|
1596
|
+
console.log(` Unknown: ${stats.unknown}`);
|
|
1597
|
+
console.log(` Requested: ${stats.requested}`);
|
|
1598
|
+
console.log();
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
}),
|
|
1602
|
+
inspect: defineCommand({
|
|
1603
|
+
meta: {
|
|
1604
|
+
name: "inspect",
|
|
1605
|
+
description: "Inspect URLs to check their indexing status"
|
|
1606
|
+
},
|
|
1607
|
+
args: {
|
|
1608
|
+
db: {
|
|
1609
|
+
type: "string",
|
|
1610
|
+
alias: "d",
|
|
1611
|
+
default: "./gscdump.db",
|
|
1612
|
+
description: "SQLite database path"
|
|
1613
|
+
},
|
|
1614
|
+
site: {
|
|
1615
|
+
type: "string",
|
|
1616
|
+
alias: "s",
|
|
1617
|
+
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1618
|
+
},
|
|
1619
|
+
limit: {
|
|
1620
|
+
type: "string",
|
|
1621
|
+
alias: "l",
|
|
1622
|
+
default: "100",
|
|
1623
|
+
description: "Max URLs to inspect"
|
|
1624
|
+
},
|
|
1625
|
+
delay: {
|
|
1626
|
+
type: "string",
|
|
1627
|
+
default: "200",
|
|
1628
|
+
description: "Delay between requests (ms)"
|
|
1629
|
+
},
|
|
1630
|
+
quiet: {
|
|
1631
|
+
type: "boolean",
|
|
1632
|
+
alias: "q",
|
|
1633
|
+
default: false,
|
|
1634
|
+
description: "Suppress progress output"
|
|
1635
|
+
},
|
|
1636
|
+
json: {
|
|
1637
|
+
type: "boolean",
|
|
1638
|
+
default: false,
|
|
1639
|
+
description: "Output as JSON"
|
|
1640
|
+
}
|
|
1641
|
+
},
|
|
1642
|
+
async run({ args }) {
|
|
1643
|
+
const config = await loadConfig();
|
|
1644
|
+
const dbPath = String(args.db === "./gscdump.db" && config.defaultDb ? config.defaultDb : args.db);
|
|
1645
|
+
const siteArg = String(args.site || config.defaultSite || "");
|
|
1646
|
+
if (!siteArg) {
|
|
1647
|
+
logger.error("Site required. Use --site or set defaultSite in config.");
|
|
1648
|
+
process.exit(1);
|
|
1649
|
+
}
|
|
1650
|
+
const { getAuth: getAuth$1 } = await Promise.resolve().then(() => auth_exports);
|
|
1651
|
+
const client = googleSearchConsole(await getAuth$1({
|
|
1652
|
+
interactive: false,
|
|
1653
|
+
config
|
|
1654
|
+
}));
|
|
1655
|
+
const { db, db0 } = createGscDb(betterSqlite3({ name: path.resolve(dbPath) }));
|
|
1656
|
+
await setupSchema(db0);
|
|
1657
|
+
await syncSites(db, client).catch(gscErrorHandler);
|
|
1658
|
+
const siteRecord = await getSiteByProperty(db, siteArg);
|
|
1659
|
+
if (!siteRecord) {
|
|
1660
|
+
logger.error(`Site not found: ${siteArg}`);
|
|
1661
|
+
process.exit(1);
|
|
1662
|
+
}
|
|
1663
|
+
const siteId = siteRecord.site_id || siteRecord.siteId;
|
|
1664
|
+
const limit = Number.parseInt(String(args.limit), 10);
|
|
1665
|
+
const delayMs = Number.parseInt(String(args.delay), 10);
|
|
1666
|
+
const paths = await db.selectDistinct({ path: sitePathDateAnalytics.path }).from(sitePathDateAnalytics).where(eq(sitePathDateAnalytics.siteId, siteId)).limit(limit);
|
|
1667
|
+
if (paths.length === 0) {
|
|
1668
|
+
logger.warn("No URLs found. Run `gscdump sync` first.");
|
|
1669
|
+
return;
|
|
1670
|
+
}
|
|
1671
|
+
if (!args.quiet && !args.json) logger.info(`Inspecting ${paths.length} URLs...`);
|
|
1672
|
+
const results = [];
|
|
1673
|
+
const stats = await batchInspectUrls(db, client, siteId, siteArg, paths.map((p) => p.path), {
|
|
1674
|
+
delayMs,
|
|
1675
|
+
onProgress: (result, i, total) => {
|
|
1676
|
+
results.push(result);
|
|
1677
|
+
if (!args.quiet && !args.json) {
|
|
1678
|
+
clearLine();
|
|
1679
|
+
process.stdout.write(progressBar(i + 1, total, result.path.slice(0, 40)));
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
}).catch(gscErrorHandler);
|
|
1683
|
+
if (!args.quiet && !args.json) clearLine();
|
|
1684
|
+
if (args.json) console.log(JSON.stringify({
|
|
1685
|
+
stats,
|
|
1686
|
+
results
|
|
1687
|
+
}, null, 2));
|
|
1688
|
+
else {
|
|
1689
|
+
console.log();
|
|
1690
|
+
logger.success(`Inspected ${paths.length} URLs`);
|
|
1691
|
+
console.log(` Indexed: ${stats.indexed}`);
|
|
1692
|
+
console.log(` Not Indexed: ${stats.notIndexed}`);
|
|
1693
|
+
console.log(` Errors: ${stats.errors}`);
|
|
1694
|
+
console.log();
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
}),
|
|
1698
|
+
request: defineCommand({
|
|
1699
|
+
meta: {
|
|
1700
|
+
name: "request",
|
|
1701
|
+
description: "Request indexing for URLs via Google Indexing API"
|
|
1702
|
+
},
|
|
1703
|
+
args: {
|
|
1704
|
+
"db": {
|
|
1705
|
+
type: "string",
|
|
1706
|
+
alias: "d",
|
|
1707
|
+
default: "./gscdump.db",
|
|
1708
|
+
description: "SQLite database path"
|
|
1709
|
+
},
|
|
1710
|
+
"site": {
|
|
1711
|
+
type: "string",
|
|
1712
|
+
alias: "s",
|
|
1713
|
+
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1714
|
+
},
|
|
1715
|
+
"limit": {
|
|
1716
|
+
type: "string",
|
|
1717
|
+
alias: "l",
|
|
1718
|
+
default: "200",
|
|
1719
|
+
description: "Max URLs to request (API quota: 200/day)"
|
|
1720
|
+
},
|
|
1721
|
+
"delay": {
|
|
1722
|
+
type: "string",
|
|
1723
|
+
default: "100",
|
|
1724
|
+
description: "Delay between requests (ms)"
|
|
1725
|
+
},
|
|
1726
|
+
"type": {
|
|
1727
|
+
type: "string",
|
|
1728
|
+
alias: "t",
|
|
1729
|
+
default: "URL_UPDATED",
|
|
1730
|
+
description: "Notification type: URL_UPDATED or URL_DELETED"
|
|
1731
|
+
},
|
|
1732
|
+
"not-indexed": {
|
|
1733
|
+
type: "boolean",
|
|
1734
|
+
default: true,
|
|
1735
|
+
description: "Only request for non-indexed URLs"
|
|
1736
|
+
},
|
|
1737
|
+
"quiet": {
|
|
1738
|
+
type: "boolean",
|
|
1739
|
+
alias: "q",
|
|
1740
|
+
default: false,
|
|
1741
|
+
description: "Suppress progress output"
|
|
1742
|
+
},
|
|
1743
|
+
"json": {
|
|
1744
|
+
type: "boolean",
|
|
1745
|
+
default: false,
|
|
1746
|
+
description: "Output as JSON"
|
|
1747
|
+
}
|
|
1748
|
+
},
|
|
1749
|
+
async run({ args }) {
|
|
1750
|
+
const config = await loadConfig();
|
|
1751
|
+
const dbPath = String(args.db === "./gscdump.db" && config.defaultDb ? config.defaultDb : args.db);
|
|
1752
|
+
const siteArg = String(args.site || config.defaultSite || "");
|
|
1753
|
+
if (!siteArg) {
|
|
1754
|
+
logger.error("Site required. Use --site or set defaultSite in config.");
|
|
1755
|
+
process.exit(1);
|
|
1756
|
+
}
|
|
1757
|
+
const { getAuth: getAuth$1 } = await Promise.resolve().then(() => auth_exports);
|
|
1758
|
+
const client = googleSearchConsole(await getAuth$1({
|
|
1759
|
+
interactive: false,
|
|
1760
|
+
config
|
|
1761
|
+
}));
|
|
1762
|
+
const { db, db0 } = createGscDb(betterSqlite3({ name: path.resolve(dbPath) }));
|
|
1763
|
+
await setupSchema(db0);
|
|
1764
|
+
await syncSites(db, client).catch(gscErrorHandler);
|
|
1765
|
+
const siteRecord = await getSiteByProperty(db, siteArg);
|
|
1766
|
+
if (!siteRecord) {
|
|
1767
|
+
logger.error(`Site not found: ${siteArg}`);
|
|
1768
|
+
process.exit(1);
|
|
1769
|
+
}
|
|
1770
|
+
const siteId = siteRecord.site_id || siteRecord.siteId;
|
|
1771
|
+
const limit = Number.parseInt(String(args.limit), 10);
|
|
1772
|
+
const delayMs = Number.parseInt(String(args.delay), 10);
|
|
1773
|
+
const type = String(args.type);
|
|
1774
|
+
const paths = await db.selectDistinct({ path: sitePathDateAnalytics.path }).from(sitePathDateAnalytics).where(eq(sitePathDateAnalytics.siteId, siteId)).limit(limit);
|
|
1775
|
+
if (paths.length === 0) {
|
|
1776
|
+
logger.warn("No URLs found. Run `gscdump sync` first.");
|
|
1777
|
+
return;
|
|
1778
|
+
}
|
|
1779
|
+
if (!args.quiet && !args.json) {
|
|
1780
|
+
logger.info(`Requesting indexing for ${paths.length} URLs...`);
|
|
1781
|
+
logger.warn("Note: Indexing API quota is typically 200 requests/day");
|
|
1782
|
+
}
|
|
1783
|
+
const results = [];
|
|
1784
|
+
const stats = await batchRequestIndexingForPaths(db, client, siteId, siteArg, paths.map((p) => p.path), {
|
|
1785
|
+
type,
|
|
1786
|
+
delayMs,
|
|
1787
|
+
onProgress: (result, i, total) => {
|
|
1788
|
+
results.push(result);
|
|
1789
|
+
if (!args.quiet && !args.json) {
|
|
1790
|
+
clearLine();
|
|
1791
|
+
const status = result.error ? "ERR" : "OK";
|
|
1792
|
+
process.stdout.write(progressBar(i + 1, total, `[${status}] ${result.url.slice(-40)}`));
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
}).catch(gscErrorHandler);
|
|
1796
|
+
if (!args.quiet && !args.json) clearLine();
|
|
1797
|
+
if (args.json) console.log(JSON.stringify({
|
|
1798
|
+
stats,
|
|
1799
|
+
results
|
|
1800
|
+
}, null, 2));
|
|
1801
|
+
else {
|
|
1802
|
+
console.log();
|
|
1803
|
+
logger.success(`Requested indexing for ${paths.length} URLs`);
|
|
1804
|
+
console.log(` Success: ${stats.success}`);
|
|
1805
|
+
console.log(` Errors: ${stats.errors}`);
|
|
1806
|
+
console.log();
|
|
1807
|
+
if (stats.errors > 0) {
|
|
1808
|
+
const errorResults = results.filter((r) => r.error);
|
|
1809
|
+
logger.warn("Errors:");
|
|
1810
|
+
errorResults.slice(0, 5).forEach((r) => {
|
|
1811
|
+
console.log(` ${r.url}: ${r.error}`);
|
|
1812
|
+
});
|
|
1813
|
+
if (errorResults.length > 5) console.log(` ... and ${errorResults.length - 5} more`);
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
})
|
|
1818
|
+
}
|
|
1819
|
+
});
|
|
1820
|
+
|
|
1821
|
+
//#endregion
|
|
1822
|
+
//#region src/commands/init.ts
|
|
1823
|
+
async function loadEnvFile() {
|
|
1824
|
+
const envPath = path.join(process.cwd(), ".env");
|
|
1825
|
+
const content = await fs.readFile(envPath, "utf-8").catch(() => null);
|
|
1826
|
+
if (!content) return null;
|
|
1827
|
+
const env = {};
|
|
1828
|
+
for (const line of content.split("\n")) {
|
|
1829
|
+
const trimmed = line.trim();
|
|
1830
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
1831
|
+
const match = trimmed.match(/^([^=]+)=(.*)$/);
|
|
1832
|
+
if (match) {
|
|
1833
|
+
const key = match[1].trim();
|
|
1834
|
+
let value = match[2].trim();
|
|
1835
|
+
if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
|
|
1836
|
+
env[key] = value;
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
return env;
|
|
1840
|
+
}
|
|
1841
|
+
const initCommand = defineCommand({
|
|
1842
|
+
meta: {
|
|
1843
|
+
name: "init",
|
|
1844
|
+
description: "Set up GSCDump (choose cloud or local mode)"
|
|
1845
|
+
},
|
|
1846
|
+
args: { force: {
|
|
1847
|
+
type: "boolean",
|
|
1848
|
+
alias: "f",
|
|
1849
|
+
description: "Force re-initialization"
|
|
1850
|
+
} },
|
|
1851
|
+
async run({ args }) {
|
|
1852
|
+
const config = await loadConfig();
|
|
1853
|
+
if (config.mode && !args.force) {
|
|
1854
|
+
logger.info(`Already configured in ${config.mode} mode`);
|
|
1855
|
+
logger.info("Run with --force to reconfigure");
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
const envFile = await loadEnvFile();
|
|
1859
|
+
if (envFile?.GOOGLE_CLIENT_ID && envFile?.GOOGLE_CLIENT_SECRET && envFile?.GOOGLE_REFRESH_TOKEN) {
|
|
1860
|
+
logger.info("Found .env file with Google credentials");
|
|
1861
|
+
process.env.GOOGLE_CLIENT_ID = envFile.GOOGLE_CLIENT_ID;
|
|
1862
|
+
process.env.GOOGLE_CLIENT_SECRET = envFile.GOOGLE_CLIENT_SECRET;
|
|
1863
|
+
process.env.GOOGLE_REFRESH_TOKEN = envFile.GOOGLE_REFRESH_TOKEN;
|
|
1864
|
+
if (envFile.GOOGLE_ACCESS_TOKEN) process.env.GOOGLE_ACCESS_TOKEN = envFile.GOOGLE_ACCESS_TOKEN;
|
|
1865
|
+
await saveConfig({
|
|
1866
|
+
...config,
|
|
1867
|
+
mode: "local",
|
|
1868
|
+
clientId: envFile.GOOGLE_CLIENT_ID,
|
|
1869
|
+
clientSecret: envFile.GOOGLE_CLIENT_SECRET
|
|
1870
|
+
});
|
|
1871
|
+
const creds = (await authenticate({
|
|
1872
|
+
clientId: envFile.GOOGLE_CLIENT_ID,
|
|
1873
|
+
clientSecret: envFile.GOOGLE_CLIENT_SECRET
|
|
1874
|
+
}, false)).credentials;
|
|
1875
|
+
if (creds.access_token) await saveTokens({
|
|
1876
|
+
access_token: creds.access_token,
|
|
1877
|
+
refresh_token: creds.refresh_token || envFile.GOOGLE_REFRESH_TOKEN,
|
|
1878
|
+
expiry_date: creds.expiry_date
|
|
1879
|
+
});
|
|
1880
|
+
console.log();
|
|
1881
|
+
logger.success("Setup complete using .env credentials! Run gscdump to get started.");
|
|
1882
|
+
return;
|
|
1883
|
+
}
|
|
1884
|
+
console.log();
|
|
1885
|
+
console.log(" \x1B[1mWelcome to GSCDump!\x1B[0m");
|
|
1886
|
+
console.log(" \x1B[90mGoogle Search Console data extraction CLI\x1B[0m");
|
|
1887
|
+
console.log();
|
|
1888
|
+
const mode = await select({
|
|
1889
|
+
message: "Choose your setup mode:",
|
|
1890
|
+
options: [{
|
|
1891
|
+
value: "cloud",
|
|
1892
|
+
label: "Cloud (Recommended)",
|
|
1893
|
+
hint: "Easy setup via cloud.gscdump.com - no API keys needed"
|
|
1894
|
+
}, {
|
|
1895
|
+
value: "local",
|
|
1896
|
+
label: "Local",
|
|
1897
|
+
hint: "Use your own Google OAuth credentials"
|
|
1898
|
+
}]
|
|
1899
|
+
});
|
|
1900
|
+
if (isCancel(mode)) process.exit(1);
|
|
1901
|
+
if (mode === "cloud") {
|
|
1902
|
+
const cloudUrl = config.cloudUrl || DEFAULT_CLOUD_URL;
|
|
1903
|
+
await saveConfig({
|
|
1904
|
+
...config,
|
|
1905
|
+
mode: "cloud",
|
|
1906
|
+
cloudUrl
|
|
1907
|
+
});
|
|
1908
|
+
await authenticateCloud(cloudUrl, true);
|
|
1909
|
+
} else {
|
|
1910
|
+
await saveConfig({
|
|
1911
|
+
...config,
|
|
1912
|
+
mode: "local"
|
|
1913
|
+
});
|
|
1914
|
+
await authenticate(await getAuthCredentials(true), true);
|
|
1915
|
+
}
|
|
1916
|
+
console.log();
|
|
1917
|
+
logger.success("Setup complete! Run gscdump to get started.");
|
|
1918
|
+
}
|
|
1919
|
+
});
|
|
1920
|
+
|
|
1921
|
+
//#endregion
|
|
1922
|
+
//#region src/commands/mcp.ts
|
|
1923
|
+
async function checkAuth() {
|
|
1924
|
+
if ((process.env.GOOGLE_ACCESS_TOKEN || process.env.GOOGLE_REFRESH_TOKEN) && process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET) return { ok: true };
|
|
1925
|
+
const config = await loadConfig();
|
|
1926
|
+
if (!config.mode) return {
|
|
1927
|
+
ok: false,
|
|
1928
|
+
error: `GSCDump not configured.
|
|
1929
|
+
|
|
1930
|
+
Run this command to set up authentication:
|
|
1931
|
+
|
|
1932
|
+
npx @gscdump/cli init
|
|
1933
|
+
|
|
1934
|
+
Or provide env vars: GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, GOOGLE_ACCESS_TOKEN
|
|
1935
|
+
|
|
1936
|
+
Then restart your MCP client.`
|
|
1937
|
+
};
|
|
1938
|
+
if (config.mode === "cloud") {
|
|
1939
|
+
if (!await loadCloudTokens()) return {
|
|
1940
|
+
ok: false,
|
|
1941
|
+
error: `Cloud authentication expired or missing.
|
|
1942
|
+
|
|
1943
|
+
Run this command to re-authenticate:
|
|
1944
|
+
|
|
1945
|
+
npx @gscdump/cli init
|
|
1946
|
+
|
|
1947
|
+
Then restart your MCP client.`
|
|
1948
|
+
};
|
|
1949
|
+
} else if (!await loadTokens()) return {
|
|
1950
|
+
ok: false,
|
|
1951
|
+
error: `Local authentication missing.
|
|
1952
|
+
|
|
1953
|
+
Run this command to authenticate:
|
|
1954
|
+
|
|
1955
|
+
npx @gscdump/cli auth
|
|
1956
|
+
|
|
1957
|
+
Then restart your MCP client.`
|
|
1958
|
+
};
|
|
1959
|
+
return { ok: true };
|
|
1960
|
+
}
|
|
1961
|
+
const mcpCommand = defineCommand({
|
|
1962
|
+
meta: {
|
|
1963
|
+
name: "mcp",
|
|
1964
|
+
description: "Start MCP server for AI assistants"
|
|
1965
|
+
},
|
|
1966
|
+
async run() {
|
|
1967
|
+
const authCheck = await checkAuth();
|
|
1968
|
+
if (!authCheck.ok) {
|
|
1969
|
+
process.stderr.write(`\n${authCheck.error}\n\n`);
|
|
1970
|
+
process.exit(1);
|
|
1971
|
+
}
|
|
1972
|
+
const server = createGscMcpServer({
|
|
1973
|
+
name: "gscdump",
|
|
1974
|
+
version: VERSION,
|
|
1975
|
+
getAuth: () => getAuth({ interactive: false })
|
|
1976
|
+
});
|
|
1977
|
+
const transport = new StdioServerTransport();
|
|
1978
|
+
await server.connect(transport);
|
|
1979
|
+
}
|
|
1980
|
+
});
|
|
1981
|
+
|
|
1982
|
+
//#endregion
|
|
1983
|
+
//#region src/commands/sitemaps.ts
|
|
1984
|
+
const listCommand = defineCommand({
|
|
1985
|
+
meta: {
|
|
1986
|
+
name: "list",
|
|
1987
|
+
description: "List sitemaps for a site"
|
|
1988
|
+
},
|
|
1989
|
+
args: {
|
|
1990
|
+
site: {
|
|
1991
|
+
type: "string",
|
|
1992
|
+
alias: "s",
|
|
1993
|
+
required: true,
|
|
1994
|
+
description: "Site URL (e.g., sc-domain:example.com or https://example.com/)"
|
|
1995
|
+
},
|
|
1996
|
+
json: {
|
|
1997
|
+
type: "boolean",
|
|
1998
|
+
default: false,
|
|
1999
|
+
description: "Output as JSON"
|
|
2000
|
+
}
|
|
2001
|
+
},
|
|
2002
|
+
async run({ args }) {
|
|
2003
|
+
const sitemaps = await fetchSitemaps(googleSearchConsole(await getAuth({ interactive: false })), args.site).catch(gscErrorHandler);
|
|
2004
|
+
if (args.json) {
|
|
2005
|
+
console.log(JSON.stringify(sitemaps, null, 2));
|
|
2006
|
+
return;
|
|
2007
|
+
}
|
|
2008
|
+
if (sitemaps.length === 0) {
|
|
2009
|
+
logger.warn("No sitemaps found");
|
|
2010
|
+
return;
|
|
2011
|
+
}
|
|
2012
|
+
logger.success(`Found ${sitemaps.length} sitemaps:`);
|
|
2013
|
+
console.log();
|
|
2014
|
+
for (const sm of sitemaps) {
|
|
2015
|
+
const pending = sm.isPending ? " \x1B[33m(pending)\x1B[0m" : "";
|
|
2016
|
+
const errors = sm.errors ? ` \x1B[31m${sm.errors} errors\x1B[0m` : "";
|
|
2017
|
+
const warnings = sm.warnings ? ` \x1B[33m${sm.warnings} warnings\x1B[0m` : "";
|
|
2018
|
+
console.log(` ${sm.path}${pending}${errors}${warnings}`);
|
|
2019
|
+
}
|
|
2020
|
+
}
|
|
2021
|
+
});
|
|
2022
|
+
const getCommand = defineCommand({
|
|
2023
|
+
meta: {
|
|
2024
|
+
name: "get",
|
|
2025
|
+
description: "Get details for a specific sitemap"
|
|
2026
|
+
},
|
|
2027
|
+
args: {
|
|
2028
|
+
site: {
|
|
2029
|
+
type: "string",
|
|
2030
|
+
alias: "s",
|
|
2031
|
+
required: true,
|
|
2032
|
+
description: "Site URL"
|
|
2033
|
+
},
|
|
2034
|
+
url: {
|
|
2035
|
+
type: "positional",
|
|
2036
|
+
required: true,
|
|
2037
|
+
description: "Sitemap URL"
|
|
2038
|
+
},
|
|
2039
|
+
json: {
|
|
2040
|
+
type: "boolean",
|
|
2041
|
+
default: false,
|
|
2042
|
+
description: "Output as JSON"
|
|
2043
|
+
}
|
|
2044
|
+
},
|
|
2045
|
+
async run({ args }) {
|
|
2046
|
+
const sitemap = await fetchSitemap(googleSearchConsole(await getAuth({ interactive: false })), args.site, args.url).catch(gscErrorHandler);
|
|
2047
|
+
if (args.json) {
|
|
2048
|
+
console.log(JSON.stringify(sitemap, null, 2));
|
|
2049
|
+
return;
|
|
2050
|
+
}
|
|
2051
|
+
console.log();
|
|
2052
|
+
console.log(` \x1B[1mPath:\x1B[0m ${sitemap.path}`);
|
|
2053
|
+
console.log(` \x1B[1mType:\x1B[0m ${sitemap.type || "sitemap"}`);
|
|
2054
|
+
console.log(` \x1B[1mLast Submitted:\x1B[0m ${sitemap.lastSubmitted || "N/A"}`);
|
|
2055
|
+
console.log(` \x1B[1mLast Downloaded:\x1B[0m ${sitemap.lastDownloaded || "N/A"}`);
|
|
2056
|
+
console.log(` \x1B[1mPending:\x1B[0m ${sitemap.isPending ? "Yes" : "No"}`);
|
|
2057
|
+
console.log(` \x1B[1mErrors:\x1B[0m ${sitemap.errors || 0}`);
|
|
2058
|
+
console.log(` \x1B[1mWarnings:\x1B[0m ${sitemap.warnings || 0}`);
|
|
2059
|
+
if (sitemap.contents?.length) {
|
|
2060
|
+
console.log();
|
|
2061
|
+
console.log(" \x1B[1mContents:\x1B[0m");
|
|
2062
|
+
for (const c of sitemap.contents) console.log(` ${c.type}: ${c.submitted} submitted, ${c.indexed} indexed`);
|
|
2063
|
+
}
|
|
2064
|
+
}
|
|
2065
|
+
});
|
|
2066
|
+
const submitCommand = defineCommand({
|
|
2067
|
+
meta: {
|
|
2068
|
+
name: "submit",
|
|
2069
|
+
description: "Submit a sitemap to GSC"
|
|
2070
|
+
},
|
|
2071
|
+
args: {
|
|
2072
|
+
site: {
|
|
2073
|
+
type: "string",
|
|
2074
|
+
alias: "s",
|
|
2075
|
+
required: true,
|
|
2076
|
+
description: "Site URL"
|
|
2077
|
+
},
|
|
2078
|
+
url: {
|
|
2079
|
+
type: "positional",
|
|
2080
|
+
required: true,
|
|
2081
|
+
description: "Sitemap URL to submit"
|
|
2082
|
+
}
|
|
2083
|
+
},
|
|
2084
|
+
async run({ args }) {
|
|
2085
|
+
await submitSitemap(googleSearchConsole(await getAuth({ interactive: false })), args.site, args.url).catch(gscErrorHandler);
|
|
2086
|
+
logger.success(`Submitted sitemap: ${args.url}`);
|
|
2087
|
+
}
|
|
2088
|
+
});
|
|
2089
|
+
const deleteCommand = defineCommand({
|
|
2090
|
+
meta: {
|
|
2091
|
+
name: "delete",
|
|
2092
|
+
description: "Delete a sitemap from GSC"
|
|
2093
|
+
},
|
|
2094
|
+
args: {
|
|
2095
|
+
site: {
|
|
2096
|
+
type: "string",
|
|
2097
|
+
alias: "s",
|
|
2098
|
+
required: true,
|
|
2099
|
+
description: "Site URL"
|
|
2100
|
+
},
|
|
2101
|
+
url: {
|
|
2102
|
+
type: "positional",
|
|
2103
|
+
required: true,
|
|
2104
|
+
description: "Sitemap URL to delete"
|
|
2105
|
+
}
|
|
2106
|
+
},
|
|
2107
|
+
async run({ args }) {
|
|
2108
|
+
await deleteSitemap(googleSearchConsole(await getAuth({ interactive: false })), args.site, args.url).catch(gscErrorHandler);
|
|
2109
|
+
logger.success(`Deleted sitemap: ${args.url}`);
|
|
2110
|
+
}
|
|
2111
|
+
});
|
|
2112
|
+
const sitemapsCommand = defineCommand({
|
|
2113
|
+
meta: {
|
|
2114
|
+
name: "sitemaps",
|
|
2115
|
+
description: "Manage sitemaps"
|
|
2116
|
+
},
|
|
2117
|
+
subCommands: {
|
|
2118
|
+
list: listCommand,
|
|
2119
|
+
get: getCommand,
|
|
2120
|
+
submit: submitCommand,
|
|
2121
|
+
delete: deleteCommand
|
|
2122
|
+
}
|
|
2123
|
+
});
|
|
2124
|
+
|
|
2125
|
+
//#endregion
|
|
2126
|
+
//#region src/commands/sites.ts
|
|
2127
|
+
const sitesCommand = defineCommand({
|
|
2128
|
+
meta: {
|
|
2129
|
+
name: "sites",
|
|
2130
|
+
description: "List available GSC sites"
|
|
2131
|
+
},
|
|
2132
|
+
args: { json: {
|
|
2133
|
+
type: "boolean",
|
|
2134
|
+
default: false,
|
|
2135
|
+
description: "Output as JSON for scripting"
|
|
2136
|
+
} },
|
|
2137
|
+
async run({ args }) {
|
|
2138
|
+
const sites = (await fetchSites(googleSearchConsole(await getAuth({ interactive: false }))).catch(gscErrorHandler)).filter((site) => site.siteUrl && site.permissionLevel !== "siteUnverifiedUser").map((site) => ({
|
|
2139
|
+
url: site.siteUrl,
|
|
2140
|
+
permission: site.permissionLevel || "unknown"
|
|
2141
|
+
}));
|
|
2142
|
+
if (args.json) {
|
|
2143
|
+
console.log(JSON.stringify(sites, null, 2));
|
|
2144
|
+
return;
|
|
2145
|
+
}
|
|
2146
|
+
if (sites.length === 0) {
|
|
2147
|
+
logger.warn("No verified sites found");
|
|
2148
|
+
return;
|
|
2149
|
+
}
|
|
2150
|
+
logger.success(`Found ${sites.length} sites:`);
|
|
2151
|
+
console.log();
|
|
2152
|
+
for (const site of sites) {
|
|
2153
|
+
const perm = site.permission === "siteOwner" ? "\x1B[32m" : "\x1B[90m";
|
|
2154
|
+
console.log(` ${site.url} ${perm}(${site.permission})\x1B[0m`);
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
});
|
|
2158
|
+
|
|
2159
|
+
//#endregion
|
|
2160
|
+
//#region src/commands/sync.ts
|
|
2161
|
+
async function runSync(client, dbPath, siteArg, period, granular, options) {
|
|
2162
|
+
const resolvedPath = path.resolve(dbPath);
|
|
2163
|
+
const report = {
|
|
2164
|
+
database: resolvedPath,
|
|
2165
|
+
period: {
|
|
2166
|
+
start: "",
|
|
2167
|
+
end: ""
|
|
2168
|
+
},
|
|
2169
|
+
sites: [],
|
|
2170
|
+
totalRows: 0
|
|
2171
|
+
};
|
|
2172
|
+
if (!options.quiet && !options.json) logger.info(`Database: ${resolvedPath}`);
|
|
2173
|
+
const { db, db0 } = createGscDb(betterSqlite3({ name: resolvedPath }));
|
|
2174
|
+
await setupSchema(db0);
|
|
2175
|
+
if (!options.quiet && !options.json) logger.start("Syncing sites...");
|
|
2176
|
+
const syncedSites = await syncSites(db, client);
|
|
2177
|
+
if (!options.quiet && !options.json) logger.success(`Synced ${syncedSites.length} sites`);
|
|
2178
|
+
let sitesToSync = [];
|
|
2179
|
+
if (siteArg) {
|
|
2180
|
+
const siteRecord = await getSiteByProperty(db, siteArg);
|
|
2181
|
+
if (!siteRecord) {
|
|
2182
|
+
if (options.json) console.log(JSON.stringify({ error: `Site not found: ${siteArg}` }));
|
|
2183
|
+
else {
|
|
2184
|
+
logger.error(`Site not found: ${siteArg}`);
|
|
2185
|
+
logger.info("Available sites:");
|
|
2186
|
+
syncedSites.slice(0, 5).forEach((s) => console.log(` - ${s.property}`));
|
|
2187
|
+
}
|
|
2188
|
+
process.exit(1);
|
|
2189
|
+
}
|
|
2190
|
+
sitesToSync = [{
|
|
2191
|
+
siteId: siteRecord.site_id || siteRecord.siteId,
|
|
2192
|
+
siteUrl: siteRecord.property
|
|
2193
|
+
}];
|
|
2194
|
+
} else sitesToSync = syncedSites.map((s) => ({
|
|
2195
|
+
siteId: s.siteId,
|
|
2196
|
+
siteUrl: s.property
|
|
2197
|
+
}));
|
|
2198
|
+
const periodRange = userPeriodRange(period);
|
|
2199
|
+
const daysOffset = options.fresh ? 1 : 3;
|
|
2200
|
+
const adjustedEndDate = dayjs(periodRange.period.endDate).subtract(daysOffset, "day").format("YYYY-MM-DD");
|
|
2201
|
+
const baseStartDate = dayjs(periodRange.period.startDate).subtract(daysOffset, "day").format("YYYY-MM-DD");
|
|
2202
|
+
const adjustedPrevEndDate = dayjs(periodRange.prevPeriod.endDate).subtract(daysOffset, "day").format("YYYY-MM-DD");
|
|
2203
|
+
const adjustedPrevStartDate = dayjs(periodRange.prevPeriod.startDate).subtract(daysOffset, "day").format("YYYY-MM-DD");
|
|
2204
|
+
const getRangeForSite = async (siteId) => {
|
|
2205
|
+
let startDate = baseStartDate;
|
|
2206
|
+
if (options.since) startDate = options.since;
|
|
2207
|
+
else if (options.incremental) {
|
|
2208
|
+
const lastSynced = await getLastSyncedDate(db, siteId);
|
|
2209
|
+
if (lastSynced) {
|
|
2210
|
+
const incrementalStart = dayjs(lastSynced).add(1, "day").format("YYYY-MM-DD");
|
|
2211
|
+
startDate = incrementalStart > baseStartDate ? incrementalStart : baseStartDate;
|
|
2212
|
+
}
|
|
2213
|
+
}
|
|
2214
|
+
if (startDate > adjustedEndDate) return {
|
|
2215
|
+
range: {
|
|
2216
|
+
period: {
|
|
2217
|
+
start: startDate,
|
|
2218
|
+
end: adjustedEndDate
|
|
2219
|
+
},
|
|
2220
|
+
prevPeriod: {
|
|
2221
|
+
start: adjustedPrevStartDate,
|
|
2222
|
+
end: adjustedPrevEndDate
|
|
2223
|
+
}
|
|
2224
|
+
},
|
|
2225
|
+
startDate,
|
|
2226
|
+
skipped: true
|
|
2227
|
+
};
|
|
2228
|
+
return {
|
|
2229
|
+
range: {
|
|
2230
|
+
period: {
|
|
2231
|
+
start: startDate,
|
|
2232
|
+
end: adjustedEndDate
|
|
2233
|
+
},
|
|
2234
|
+
prevPeriod: {
|
|
2235
|
+
start: adjustedPrevStartDate,
|
|
2236
|
+
end: adjustedPrevEndDate
|
|
2237
|
+
}
|
|
2238
|
+
},
|
|
2239
|
+
startDate,
|
|
2240
|
+
skipped: false
|
|
2241
|
+
};
|
|
2242
|
+
};
|
|
2243
|
+
report.period = {
|
|
2244
|
+
start: options.since || baseStartDate,
|
|
2245
|
+
end: adjustedEndDate
|
|
2246
|
+
};
|
|
2247
|
+
if (!options.quiet && !options.json) {
|
|
2248
|
+
const modeNote = options.incremental ? " (incremental)" : options.since ? ` (since ${options.since})` : options.fresh ? " (fresh)" : "";
|
|
2249
|
+
logger.info(`Period: ${options.since || baseStartDate} to ${adjustedEndDate}${modeNote}`);
|
|
2250
|
+
console.log();
|
|
2251
|
+
}
|
|
2252
|
+
const dataTypes = granular ? [
|
|
2253
|
+
"pages",
|
|
2254
|
+
"keywords",
|
|
2255
|
+
"keyword-paths",
|
|
2256
|
+
"countries",
|
|
2257
|
+
"devices"
|
|
2258
|
+
] : [
|
|
2259
|
+
"pages",
|
|
2260
|
+
"keywords",
|
|
2261
|
+
"countries",
|
|
2262
|
+
"devices"
|
|
2263
|
+
];
|
|
2264
|
+
for (const { siteId, siteUrl } of sitesToSync) {
|
|
2265
|
+
const siteName = siteUrl.replace(/^(sc-domain:|https?:\/\/)/, "");
|
|
2266
|
+
const { range, startDate, skipped } = await getRangeForSite(siteId);
|
|
2267
|
+
if (skipped) {
|
|
2268
|
+
if (!options.quiet && !options.json) logger.info(`${siteName} is up to date (last synced: ${startDate})`);
|
|
2269
|
+
continue;
|
|
2270
|
+
}
|
|
2271
|
+
const siteReport = {
|
|
2272
|
+
site: siteUrl,
|
|
2273
|
+
rows: {
|
|
2274
|
+
pages: 0,
|
|
2275
|
+
keywords: 0,
|
|
2276
|
+
countries: 0,
|
|
2277
|
+
devices: 0,
|
|
2278
|
+
total: 0
|
|
2279
|
+
}
|
|
2280
|
+
};
|
|
2281
|
+
if (!options.quiet && !options.json && (options.incremental || options.since)) logger.info(`${siteName}: syncing ${startDate} to ${adjustedEndDate}`);
|
|
2282
|
+
for (let i = 0; i < dataTypes.length; i++) {
|
|
2283
|
+
const dataType = dataTypes[i];
|
|
2284
|
+
if (!options.quiet && !options.json) {
|
|
2285
|
+
clearLine();
|
|
2286
|
+
process.stdout.write(progressBar(i + 1, dataTypes.length, `${dataType} (${siteName})`));
|
|
2287
|
+
}
|
|
2288
|
+
let rows = [];
|
|
2289
|
+
if (dataType === "pages") {
|
|
2290
|
+
rows = await syncPages(db, client, siteId, siteUrl, range);
|
|
2291
|
+
siteReport.rows.pages = rows.length;
|
|
2292
|
+
} else if (dataType === "keywords") {
|
|
2293
|
+
rows = await syncKeywords(db, client, siteId, siteUrl, range);
|
|
2294
|
+
siteReport.rows.keywords = rows.length;
|
|
2295
|
+
} else if (dataType === "keyword-paths") {
|
|
2296
|
+
rows = await syncKeywordPaths(db, client, siteId, siteUrl, range);
|
|
2297
|
+
siteReport.rows.keywordPaths = rows.length;
|
|
2298
|
+
} else if (dataType === "countries") {
|
|
2299
|
+
rows = await syncCountries(db, client, siteId, siteUrl, range);
|
|
2300
|
+
siteReport.rows.countries = rows.length;
|
|
2301
|
+
} else if (dataType === "devices") {
|
|
2302
|
+
rows = await syncDevices(db, client, siteId, siteUrl, range);
|
|
2303
|
+
siteReport.rows.devices = rows.length;
|
|
2304
|
+
}
|
|
2305
|
+
siteReport.rows.total += rows.length;
|
|
2306
|
+
}
|
|
2307
|
+
if (!options.quiet && !options.json) clearLine();
|
|
2308
|
+
await updateLastSynced(db, siteId);
|
|
2309
|
+
report.sites.push(siteReport);
|
|
2310
|
+
report.totalRows += siteReport.rows.total;
|
|
2311
|
+
if (!options.quiet && !options.json) logger.success(`Synced ${siteName} (${siteReport.rows.total.toLocaleString()} rows)`);
|
|
2312
|
+
}
|
|
2313
|
+
if (!options.quiet && !options.json) {
|
|
2314
|
+
console.log();
|
|
2315
|
+
logger.success(`Database saved to ${resolvedPath}`);
|
|
2316
|
+
}
|
|
2317
|
+
return report;
|
|
2318
|
+
}
|
|
2319
|
+
const syncCommand = defineCommand({
|
|
2320
|
+
meta: {
|
|
2321
|
+
name: "sync",
|
|
2322
|
+
description: "Sync GSC data to SQLite database"
|
|
2323
|
+
},
|
|
2324
|
+
args: {
|
|
2325
|
+
db: {
|
|
2326
|
+
type: "string",
|
|
2327
|
+
alias: "d",
|
|
2328
|
+
default: "./gscdump.db",
|
|
2329
|
+
description: "SQLite database path"
|
|
2330
|
+
},
|
|
2331
|
+
site: {
|
|
2332
|
+
type: "string",
|
|
2333
|
+
alias: "s",
|
|
2334
|
+
description: "Site URL (e.g., sc-domain:example.com)"
|
|
2335
|
+
},
|
|
2336
|
+
period: {
|
|
2337
|
+
type: "string",
|
|
2338
|
+
alias: "p",
|
|
2339
|
+
default: "90d",
|
|
2340
|
+
description: "Time period: 90d, 6m, 1y, max (all GSC history ~16mo)"
|
|
2341
|
+
},
|
|
2342
|
+
granular: {
|
|
2343
|
+
type: "boolean",
|
|
2344
|
+
alias: "g",
|
|
2345
|
+
default: false,
|
|
2346
|
+
description: "Include keyword-per-page data (large dataset)"
|
|
2347
|
+
},
|
|
2348
|
+
quiet: {
|
|
2349
|
+
type: "boolean",
|
|
2350
|
+
alias: "q",
|
|
2351
|
+
default: false,
|
|
2352
|
+
description: "Suppress output"
|
|
2353
|
+
},
|
|
2354
|
+
json: {
|
|
2355
|
+
type: "boolean",
|
|
2356
|
+
default: false,
|
|
2357
|
+
description: "Output sync report as JSON"
|
|
2358
|
+
},
|
|
2359
|
+
fresh: {
|
|
2360
|
+
type: "boolean",
|
|
2361
|
+
default: false,
|
|
2362
|
+
description: "Include fresh/unfinalized data (last 3 days)"
|
|
2363
|
+
},
|
|
2364
|
+
incremental: {
|
|
2365
|
+
type: "boolean",
|
|
2366
|
+
alias: "i",
|
|
2367
|
+
default: false,
|
|
2368
|
+
description: "Only fetch data since last sync"
|
|
2369
|
+
},
|
|
2370
|
+
since: {
|
|
2371
|
+
type: "string",
|
|
2372
|
+
description: "Fetch data from specific date (YYYY-MM-DD)"
|
|
2373
|
+
}
|
|
2374
|
+
},
|
|
2375
|
+
async run({ args }) {
|
|
2376
|
+
const config = await loadConfig();
|
|
2377
|
+
const dbPath = args.db === "./gscdump.db" && config.defaultDb ? config.defaultDb : args.db;
|
|
2378
|
+
const siteArg = args.site || config.defaultSite || null;
|
|
2379
|
+
const periodArg = args.period === "90d" && config.defaultPeriod ? config.defaultPeriod : args.period;
|
|
2380
|
+
const { getAuth: getAuth$1 } = await Promise.resolve().then(() => auth_exports);
|
|
2381
|
+
const report = await runSync(googleSearchConsole(await getAuth$1({
|
|
2382
|
+
interactive: false,
|
|
2383
|
+
config
|
|
2384
|
+
})), dbPath, siteArg, periodArg, args.granular, {
|
|
2385
|
+
quiet: args.quiet,
|
|
2386
|
+
json: args.json,
|
|
2387
|
+
fresh: args.fresh,
|
|
2388
|
+
incremental: args.incremental,
|
|
2389
|
+
since: args.since
|
|
2390
|
+
}).catch(gscErrorHandler);
|
|
2391
|
+
if (args.json) console.log(JSON.stringify(report, null, 2));
|
|
2392
|
+
else if (!args.quiet) logger.success("Done!");
|
|
2393
|
+
}
|
|
2394
|
+
});
|
|
2395
|
+
|
|
2396
|
+
//#endregion
|
|
2397
|
+
//#region src/index.ts
|
|
2398
|
+
runMain(defineCommand({
|
|
2399
|
+
meta: {
|
|
2400
|
+
name: "gscdump",
|
|
2401
|
+
version: VERSION,
|
|
2402
|
+
description: "Google Search Console Data Extractor"
|
|
2403
|
+
},
|
|
2404
|
+
subCommands: {
|
|
2405
|
+
init: initCommand,
|
|
2406
|
+
dump: dumpCommand,
|
|
2407
|
+
sync: syncCommand,
|
|
2408
|
+
compare: compareCommand,
|
|
2409
|
+
analyze: analyzeCommand,
|
|
2410
|
+
sites: sitesCommand,
|
|
2411
|
+
sitemaps: sitemapsCommand,
|
|
2412
|
+
index: indexingCommand,
|
|
2413
|
+
inspect: inspectCommand,
|
|
2414
|
+
auth: authCommand,
|
|
2415
|
+
config: configCommand,
|
|
2416
|
+
mcp: mcpCommand
|
|
2417
|
+
},
|
|
2418
|
+
setup() {
|
|
2419
|
+
if (!process.argv.includes("mcp")) showSplash();
|
|
2420
|
+
},
|
|
2421
|
+
async run({ args }) {
|
|
2422
|
+
if (!args._.length) await dumpCommand.run({
|
|
2423
|
+
args,
|
|
2424
|
+
rawArgs: [],
|
|
2425
|
+
cmd: dumpCommand
|
|
2426
|
+
});
|
|
2427
|
+
}
|
|
2428
|
+
}));
|
|
2429
|
+
|
|
2430
|
+
//#endregion
|
|
2431
|
+
export { getAuth as a, loadTokens as c, clearTokens as i, saveCloudTokens as l, authenticateCloud as n, getAuthCredentials as o, clearCloudTokens as r, loadCloudTokens as s, authenticate as t, saveTokens as u };
|