@gscdump/cli 3.7.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -30
- package/bin/gscdump.mjs +21 -1
- package/dist/analysis-local.mjs +172 -53
- package/dist/auth-state.mjs +21 -6
- package/dist/auth.mjs +27 -18
- package/dist/bing-data.mjs +6 -2
- package/dist/bing-hosted.mjs +1 -1
- package/dist/cli-args.mjs +8 -0
- package/dist/cli.d.mts +5 -1
- package/dist/cli.mjs +41 -10
- package/dist/cloud-google.mjs +8 -1
- package/dist/command-meta.mjs +4 -4
- package/dist/command-registry.mjs +86 -10
- package/dist/commands/analyze.mjs +84 -43
- package/dist/commands/auth.mjs +46 -37
- package/dist/commands/bing.mjs +3 -3
- package/dist/commands/compact.mjs +12 -8
- package/dist/commands/config.mjs +14 -14
- package/dist/commands/doctor.mjs +26 -26
- package/dist/commands/dump.mjs +203 -105
- package/dist/commands/entities.mjs +10 -113
- package/dist/commands/gc.mjs +4 -3
- package/dist/commands/indexing-urls.mjs +185 -0
- package/dist/commands/indexing.mjs +12 -8
- package/dist/commands/init.mjs +77 -18
- package/dist/commands/inspect.mjs +97 -110
- package/dist/commands/mcp.mjs +37 -57
- package/dist/commands/papercut.mjs +1 -1
- package/dist/commands/profile.mjs +9 -17
- package/dist/commands/query.mjs +219 -210
- package/dist/commands/report.mjs +68 -85
- package/dist/commands/rollups.mjs +9 -6
- package/dist/commands/sitemaps.mjs +51 -104
- package/dist/commands/sites.mjs +40 -24
- package/dist/commands/stats.mjs +14 -17
- package/dist/commands/store-purge.mjs +9 -5
- package/dist/commands/store.mjs +0 -12
- package/dist/commands/sync.mjs +774 -296
- package/dist/config.mjs +7 -4
- package/dist/context.mjs +51 -21
- package/dist/coverage.mjs +251 -0
- package/dist/dump-bing.mjs +71 -0
- package/dist/dump-writers.mjs +246 -0
- package/dist/error-handler.mjs +108 -38
- package/dist/filters.mjs +125 -0
- package/dist/hosted-site.mjs +91 -0
- package/dist/inspect-urls.mjs +76 -0
- package/dist/inspection-record.mjs +93 -0
- package/dist/local-entities.mjs +658 -0
- package/dist/mcp/errors.mjs +71 -18
- package/dist/mcp/handlers/diagnostics.mjs +15 -12
- package/dist/mcp/handlers/reports.mjs +33 -42
- package/dist/mcp/server/index.mjs +80 -38
- package/dist/mcp/types.mjs +7 -7
- package/dist/package.mjs +1 -1
- package/dist/quota-ledger.mjs +370 -0
- package/dist/render/analysis.mjs +8 -1
- package/dist/render/report.mjs +8 -1
- package/dist/request-pacer.mjs +29 -0
- package/dist/route.mjs +295 -0
- package/dist/sitemap.mjs +4 -0
- package/dist/sql-views.mjs +117 -0
- package/dist/store-sites.mjs +134 -0
- package/dist/sync-plan.mjs +129 -0
- package/dist/sync-run.mjs +96 -0
- package/dist/table-sources.mjs +57 -0
- package/dist/token-info.mjs +54 -0
- package/dist/utils.mjs +28 -19
- package/dist/window.mjs +159 -0
- package/package.json +17 -14
- package/skills/gscdump/SKILL.md +167 -37
- package/dist/commands/export.mjs +0 -76
- package/dist/native-duckdb.mjs +0 -46
package/dist/commands/query.mjs
CHANGED
|
@@ -1,21 +1,25 @@
|
|
|
1
1
|
import { queryCommandMeta } from "../command-meta.mjs";
|
|
2
|
+
import { useCliRuntime } from "../runtime.mjs";
|
|
2
3
|
import { loadConfig } from "../config.mjs";
|
|
3
4
|
import { terminalOutputOptions } from "../render/terminal.mjs";
|
|
4
5
|
import { ALL_SEARCH_TYPES, logger, parseSearchType, toCSV } from "../utils.mjs";
|
|
5
|
-
import { allTables
|
|
6
|
-
import { createCommandContext } from "../context.mjs";
|
|
7
|
-
import {
|
|
6
|
+
import { allTables } from "../local-store.mjs";
|
|
7
|
+
import { createCommandContext, formatSiteResolution } from "../context.mjs";
|
|
8
|
+
import { decideRoute, describeStop, liveNote, readRouteState, readSiteStates, resolveReadSite, stopAtRoute } from "../route.mjs";
|
|
9
|
+
import { DEFAULT_WINDOW, checkWindowFlags, newestDoneDate, parseWindowFlags } from "../window.mjs";
|
|
8
10
|
import { renderTable } from "../render/layout.mjs";
|
|
9
11
|
import { asRecord, columnsFor } from "../render/analysis.mjs";
|
|
12
|
+
import { FILTER_DIMS, filterDimensions, parseFilterArgs, toLiveFilter, toLocalFilter } from "../filters.mjs";
|
|
10
13
|
import { renderQuery } from "../render/query.mjs";
|
|
14
|
+
import { openSqlViews, referencedTables } from "../sql-views.mjs";
|
|
11
15
|
import process from "node:process";
|
|
12
16
|
import { defineCommand } from "citty";
|
|
13
17
|
import fs from "node:fs/promises";
|
|
14
|
-
import { and, between,
|
|
15
|
-
import {
|
|
18
|
+
import { and, between, country, date, device, gsc, hour, page, query, searchAppearance } from "gscdump/query";
|
|
19
|
+
import { getLatestGscDate } from "gscdump/dates";
|
|
16
20
|
import { cancel, isCancel, multiselect, text } from "@clack/prompts";
|
|
17
|
-
import { daysAgoUtc } from "gscdump/dates";
|
|
18
21
|
import { collectSpans } from "@gscdump/engine/profile";
|
|
22
|
+
import { inferDataset, isDatasetResolvable } from "gscdump/query/plan";
|
|
19
23
|
const DIMENSIONS = [
|
|
20
24
|
"page",
|
|
21
25
|
"query",
|
|
@@ -34,20 +38,6 @@ const DIM_COLUMNS = {
|
|
|
34
38
|
device,
|
|
35
39
|
searchAppearance
|
|
36
40
|
};
|
|
37
|
-
const FILTER_DIMS = [
|
|
38
|
-
"query",
|
|
39
|
-
"page",
|
|
40
|
-
"country",
|
|
41
|
-
"device",
|
|
42
|
-
"searchAppearance"
|
|
43
|
-
];
|
|
44
|
-
const FILTER_COL = {
|
|
45
|
-
query,
|
|
46
|
-
page,
|
|
47
|
-
country,
|
|
48
|
-
device,
|
|
49
|
-
searchAppearance
|
|
50
|
-
};
|
|
51
41
|
const DATA_STATES = [
|
|
52
42
|
"all",
|
|
53
43
|
"final",
|
|
@@ -59,27 +49,6 @@ const AGGREGATION_TYPES = [
|
|
|
59
49
|
"byProperty"
|
|
60
50
|
];
|
|
61
51
|
const POSITIVE_INTEGER_RE = /^\d+$/;
|
|
62
|
-
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
|
63
|
-
function buildFilterFromArg(col, raw) {
|
|
64
|
-
if (raw.startsWith("!~")) return makeLeaf(col, "notContains", raw.slice(2));
|
|
65
|
-
if (raw.startsWith("!re:")) return notRegex(col, raw.slice(4));
|
|
66
|
-
if (raw.startsWith("!")) return makeLeaf(col, "notEquals", raw.slice(1));
|
|
67
|
-
if (raw.startsWith("~")) return contains(col, raw.slice(1));
|
|
68
|
-
if (raw.startsWith("re:")) return regex(col, raw.slice(3));
|
|
69
|
-
if (raw.startsWith("contains:")) return contains(col, raw.slice(9));
|
|
70
|
-
if (raw.startsWith("eq:")) return eq(col, raw.slice(3));
|
|
71
|
-
return eq(col, raw);
|
|
72
|
-
}
|
|
73
|
-
function makeLeaf(col, operator, value) {
|
|
74
|
-
return {
|
|
75
|
-
_constraints: {},
|
|
76
|
-
_filters: [{
|
|
77
|
-
dimension: col.dimension,
|
|
78
|
-
operator,
|
|
79
|
-
expression: value
|
|
80
|
-
}]
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
52
|
async function runLiveQuery(client, siteUrl, opts) {
|
|
84
53
|
const allRows = [];
|
|
85
54
|
const totalLimit = Math.max(0, opts.rowLimit);
|
|
@@ -137,7 +106,7 @@ const queryCommand = defineCommand({
|
|
|
137
106
|
"site": {
|
|
138
107
|
type: "string",
|
|
139
108
|
alias: "s",
|
|
140
|
-
description: "Site
|
|
109
|
+
description: "Site, for example example.com"
|
|
141
110
|
},
|
|
142
111
|
"dimensions": {
|
|
143
112
|
type: "string",
|
|
@@ -169,11 +138,12 @@ const queryCommand = defineCommand({
|
|
|
169
138
|
},
|
|
170
139
|
"sql": {
|
|
171
140
|
type: "string",
|
|
172
|
-
description: "
|
|
141
|
+
description: "DuckDB SQL over one view per Store table, with site and search_type columns. Example: SELECT search_type, SUM(clicks) AS clicks, gsc_position(sum_position, impressions) AS position FROM pages GROUP BY search_type. Run --schema to list views"
|
|
173
142
|
},
|
|
174
|
-
"
|
|
175
|
-
type: "
|
|
176
|
-
|
|
143
|
+
"schema": {
|
|
144
|
+
type: "boolean",
|
|
145
|
+
default: false,
|
|
146
|
+
description: "List the --sql views, their columns, and the date range of each view, then exit"
|
|
177
147
|
},
|
|
178
148
|
"live": {
|
|
179
149
|
type: "boolean",
|
|
@@ -214,7 +184,7 @@ const queryCommand = defineCommand({
|
|
|
214
184
|
},
|
|
215
185
|
"type": {
|
|
216
186
|
type: "string",
|
|
217
|
-
description: `Search type
|
|
187
|
+
description: `Search type. One of: ${ALL_SEARCH_TYPES.join(",")} (default: web; --sql and --schema default to every search type)`
|
|
218
188
|
},
|
|
219
189
|
"data-state": {
|
|
220
190
|
type: "string",
|
|
@@ -242,20 +212,28 @@ const queryCommand = defineCommand({
|
|
|
242
212
|
logger.error("Invalid --format. Use table, json, or csv.");
|
|
243
213
|
process.exit(1);
|
|
244
214
|
}
|
|
245
|
-
if (args.sql) {
|
|
246
|
-
await
|
|
247
|
-
|
|
215
|
+
if (args.sql || args.schema) {
|
|
216
|
+
await runSqlMode({
|
|
217
|
+
mode: args.schema ? { kind: "schema" } : {
|
|
218
|
+
kind: "sql",
|
|
219
|
+
sql: String(args.sql)
|
|
220
|
+
},
|
|
248
221
|
site: args.site ? String(args.site) : void 0,
|
|
249
|
-
table: args.table ? String(args.table) : "pages",
|
|
250
222
|
output: args.output ? String(args.output) : void 0,
|
|
251
223
|
format,
|
|
252
224
|
quiet: Boolean(args.quiet),
|
|
225
|
+
forceLive: Boolean(args.live),
|
|
253
226
|
searchType: parseSearchType(args.type, "--type")
|
|
254
227
|
});
|
|
255
228
|
return;
|
|
256
229
|
}
|
|
257
230
|
const dimNames = await resolveDimensions(args);
|
|
258
|
-
const
|
|
231
|
+
const windowFlags = await promptRange(args);
|
|
232
|
+
const invalidWindow = checkWindowFlags(windowFlags);
|
|
233
|
+
if (invalidWindow) {
|
|
234
|
+
logger.error(invalidWindow.message);
|
|
235
|
+
process.exit(1);
|
|
236
|
+
}
|
|
259
237
|
await promptFilters(args);
|
|
260
238
|
const limitArg = String(args.limit ?? ctxConfig.defaultLimit ?? 1e3);
|
|
261
239
|
const rowLimit = Number(limitArg);
|
|
@@ -263,10 +241,12 @@ const queryCommand = defineCommand({
|
|
|
263
241
|
logger.error("Invalid --limit. Use a positive safe integer, such as --limit 1000.");
|
|
264
242
|
process.exit(1);
|
|
265
243
|
}
|
|
266
|
-
const
|
|
244
|
+
const filters = parseFilterArgs(args);
|
|
267
245
|
const searchType = parseSearchType(args.type ?? ctxConfig.defaultSearchType, "--type");
|
|
246
|
+
const localSearchType = searchType ?? "web";
|
|
268
247
|
const dataState = args["data-state"] ? String(args["data-state"]) : ctxConfig.defaultDataState;
|
|
269
248
|
const aggregationType = args["aggregation-type"] ? String(args["aggregation-type"]) : void 0;
|
|
249
|
+
const filterDims = filterDimensions(filters);
|
|
270
250
|
if (dataState && !DATA_STATES.includes(dataState)) {
|
|
271
251
|
logger.error(`Invalid --data-state: ${dataState}. Allowed: ${DATA_STATES.join(", ")}`);
|
|
272
252
|
process.exit(1);
|
|
@@ -275,15 +255,54 @@ const queryCommand = defineCommand({
|
|
|
275
255
|
logger.error(`Invalid --aggregation-type: ${aggregationType}. Allowed: ${AGGREGATION_TYPES.join(", ")}`);
|
|
276
256
|
process.exit(1);
|
|
277
257
|
}
|
|
278
|
-
const
|
|
258
|
+
const forceLive = Boolean(args.live);
|
|
259
|
+
const resolvable = isDatasetResolvable(dimNames, filterDims);
|
|
260
|
+
if (!resolvable && !forceLive) throw new Error(`No Store table holds ${[.../* @__PURE__ */ new Set([...dimNames, ...filterDims])].join(" with ")}. Remove a dimension or filter, or pass --live.`);
|
|
261
|
+
const storeCtx = await createCommandContext({ needsStore: true });
|
|
262
|
+
const store = storeCtx.store;
|
|
263
|
+
let liveCtx;
|
|
264
|
+
const connect = () => liveCtx ??= createCommandContext({
|
|
279
265
|
needsAuth: true,
|
|
280
|
-
needsStore: !args.live,
|
|
281
266
|
interactive: Boolean(args.interactive)
|
|
282
267
|
});
|
|
283
|
-
const
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
268
|
+
const { site, siteHint, auth } = await resolveReadSite(storeCtx, args.site ? String(args.site) : void 0, {
|
|
269
|
+
forceLive,
|
|
270
|
+
connect
|
|
271
|
+
});
|
|
272
|
+
const table = resolvable ? inferDataset(dimNames, filterDims) : void 0;
|
|
273
|
+
const states = site ? await readSiteStates(store, site) : [];
|
|
274
|
+
const { start: startDate, end: endDate } = windowOrExit(windowFlags, forceLive || !table ? getLatestGscDate() : newestDoneDate(states, [table]) ?? getLatestGscDate());
|
|
275
|
+
const req = {
|
|
276
|
+
site,
|
|
277
|
+
siteHint,
|
|
278
|
+
label: "query",
|
|
279
|
+
localCapable: resolvable,
|
|
280
|
+
liveCapable: true,
|
|
281
|
+
forceLive,
|
|
282
|
+
argv: useCliRuntime().rawArgs
|
|
283
|
+
};
|
|
284
|
+
const routeState = await readRouteState({
|
|
285
|
+
store,
|
|
286
|
+
site,
|
|
287
|
+
needs: table ? [{
|
|
288
|
+
kind: "window",
|
|
289
|
+
period: "current",
|
|
290
|
+
table,
|
|
291
|
+
searchType: localSearchType,
|
|
292
|
+
window: {
|
|
293
|
+
start: startDate,
|
|
294
|
+
end: endDate
|
|
295
|
+
}
|
|
296
|
+
}] : [],
|
|
297
|
+
states,
|
|
298
|
+
auth
|
|
299
|
+
});
|
|
300
|
+
const route = decideRoute(req, routeState);
|
|
301
|
+
const json = format === "json";
|
|
302
|
+
if (route.kind === "live") {
|
|
303
|
+
const live = await connect();
|
|
304
|
+
const siteUrl = site ?? await live.resolveSite(void 0);
|
|
305
|
+
const dimensionFilter = toLiveFilter(filters, siteUrl);
|
|
287
306
|
if (args.explain) {
|
|
288
307
|
const body = {
|
|
289
308
|
startDate,
|
|
@@ -297,12 +316,14 @@ const queryCommand = defineCommand({
|
|
|
297
316
|
if (dimensionFilter) body.dimensionFilterGroups = filterToGroups(dimensionFilter);
|
|
298
317
|
console.log(JSON.stringify({
|
|
299
318
|
siteUrl,
|
|
319
|
+
source: "live",
|
|
300
320
|
body
|
|
301
321
|
}, null, 2));
|
|
302
322
|
return;
|
|
303
323
|
}
|
|
324
|
+
if (route.reason === "no-store-data") logger.warn(liveNote(siteUrl));
|
|
304
325
|
if (!args.quiet) logger.debug(`Querying ${siteUrl} via live GSC API...`);
|
|
305
|
-
const result = await runLiveQuery(
|
|
326
|
+
const result = await runLiveQuery(live.client, siteUrl, {
|
|
306
327
|
startDate,
|
|
307
328
|
endDate,
|
|
308
329
|
dimensions: dimNames,
|
|
@@ -311,7 +332,7 @@ const queryCommand = defineCommand({
|
|
|
311
332
|
dataState,
|
|
312
333
|
aggregationType,
|
|
313
334
|
dimensionFilter
|
|
314
|
-
})
|
|
335
|
+
});
|
|
315
336
|
await writeOutput({
|
|
316
337
|
output: {
|
|
317
338
|
siteUrl,
|
|
@@ -321,7 +342,8 @@ const queryCommand = defineCommand({
|
|
|
321
342
|
end: endDate
|
|
322
343
|
},
|
|
323
344
|
total: result.rows.length,
|
|
324
|
-
data: result.rows
|
|
345
|
+
data: result.rows,
|
|
346
|
+
meta: { source: "live" }
|
|
325
347
|
},
|
|
326
348
|
format,
|
|
327
349
|
path: args.output ? String(args.output) : void 0,
|
|
@@ -329,26 +351,28 @@ const queryCommand = defineCommand({
|
|
|
329
351
|
});
|
|
330
352
|
return;
|
|
331
353
|
}
|
|
332
|
-
|
|
333
|
-
if (!args.quiet) logger.debug(`Querying ${siteUrl} from local Parquet store...`);
|
|
334
|
-
const state = buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter);
|
|
335
|
-
const store = ctx.store;
|
|
336
|
-
const table = inferTable(dimNames);
|
|
354
|
+
const state = buildLocalState(dimNames, startDate, endDate, rowLimit, toLocalFilter(filters));
|
|
337
355
|
if (args.explain) {
|
|
356
|
+
const stop = route.kind === "local" ? void 0 : describeStop(route, req, routeState.auth);
|
|
338
357
|
console.log(JSON.stringify({
|
|
339
|
-
siteUrl,
|
|
358
|
+
siteUrl: site ?? null,
|
|
359
|
+
source: "local",
|
|
340
360
|
table,
|
|
341
|
-
state
|
|
361
|
+
state,
|
|
362
|
+
...stop ? { stop } : {}
|
|
342
363
|
}, null, 2));
|
|
343
364
|
return;
|
|
344
365
|
}
|
|
345
|
-
|
|
366
|
+
if (route.kind === "syncing" || route.kind === "prompt") stopAtRoute(route, req, routeState.auth, { json });
|
|
367
|
+
if (dataState || aggregationType) logger.warn("--data-state / --aggregation-type are ignored without --live");
|
|
368
|
+
const siteUrl = site;
|
|
369
|
+
if (!args.quiet) logger.debug(`Querying ${siteUrl} from local Parquet store...`);
|
|
346
370
|
const probe = Boolean(args.profile) ? collectSpans() : void 0;
|
|
347
371
|
const result = await store.engine.query({
|
|
348
372
|
userId: store.userId,
|
|
349
373
|
siteId: store.siteIdFor(siteUrl),
|
|
350
374
|
table,
|
|
351
|
-
|
|
375
|
+
searchType: localSearchType,
|
|
352
376
|
...probe ? { profiler: probe.profiler } : {}
|
|
353
377
|
}, state).catch((e) => {
|
|
354
378
|
logger.error(`Query failed: ${e.message}`);
|
|
@@ -364,7 +388,8 @@ const queryCommand = defineCommand({
|
|
|
364
388
|
end: endDate
|
|
365
389
|
},
|
|
366
390
|
total: result.rows.length,
|
|
367
|
-
data: result.rows
|
|
391
|
+
data: result.rows,
|
|
392
|
+
meta: { source: "local" }
|
|
368
393
|
},
|
|
369
394
|
format,
|
|
370
395
|
path: args.output ? String(args.output) : void 0,
|
|
@@ -398,51 +423,46 @@ async function resolveDimensions(args) {
|
|
|
398
423
|
}
|
|
399
424
|
return ["page", "query"];
|
|
400
425
|
}
|
|
401
|
-
async function
|
|
402
|
-
let
|
|
403
|
-
let
|
|
426
|
+
async function promptRange(args) {
|
|
427
|
+
let start = args.start != null ? String(args.start) : void 0;
|
|
428
|
+
let end = args.end != null ? String(args.end) : void 0;
|
|
404
429
|
if (args.interactive) {
|
|
405
|
-
if (
|
|
430
|
+
if (start === void 0) {
|
|
406
431
|
const startInput = await text({
|
|
407
|
-
message: "Start date (YYYY-MM-DD)",
|
|
408
|
-
placeholder:
|
|
432
|
+
message: "Start date (YYYY-MM-DD, blank for the last 28 synced days)",
|
|
433
|
+
placeholder: ""
|
|
409
434
|
});
|
|
410
435
|
if (isCancel(startInput)) {
|
|
411
436
|
cancel("Cancelled");
|
|
412
437
|
process.exit(0);
|
|
413
438
|
}
|
|
414
|
-
|
|
439
|
+
start = String(startInput) || void 0;
|
|
415
440
|
}
|
|
416
|
-
if (
|
|
441
|
+
if (end === void 0) {
|
|
417
442
|
const endInput = await text({
|
|
418
|
-
message: "End date (YYYY-MM-DD)",
|
|
419
|
-
placeholder:
|
|
443
|
+
message: "End date (YYYY-MM-DD, blank for the newest synced day)",
|
|
444
|
+
placeholder: ""
|
|
420
445
|
});
|
|
421
446
|
if (isCancel(endInput)) {
|
|
422
447
|
cancel("Cancelled");
|
|
423
448
|
process.exit(0);
|
|
424
449
|
}
|
|
425
|
-
|
|
450
|
+
end = String(endInput) || void 0;
|
|
426
451
|
}
|
|
427
452
|
}
|
|
428
|
-
startDate ??= daysAgoUtc(31);
|
|
429
|
-
endDate ??= daysAgoUtc(3);
|
|
430
|
-
for (const [flag, value] of [["--start", startDate], ["--end", endDate]]) {
|
|
431
|
-
const date = /* @__PURE__ */ new Date(`${value}T00:00:00Z`);
|
|
432
|
-
if (!ISO_DATE_RE.test(value) || !Number.isFinite(date.getTime()) || date.toISOString().slice(0, 10) !== value) {
|
|
433
|
-
logger.error(`Invalid ${flag}. Use a calendar date in YYYY-MM-DD format.`);
|
|
434
|
-
process.exit(1);
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
if (startDate > endDate) {
|
|
438
|
-
logger.error("Invalid date range. --start must be on or before --end.");
|
|
439
|
-
process.exit(1);
|
|
440
|
-
}
|
|
441
453
|
return {
|
|
442
|
-
|
|
443
|
-
|
|
454
|
+
start,
|
|
455
|
+
end
|
|
444
456
|
};
|
|
445
457
|
}
|
|
458
|
+
function windowOrExit(flags, anchor) {
|
|
459
|
+
const window = parseWindowFlags(flags, DEFAULT_WINDOW, anchor);
|
|
460
|
+
if (!window.ok) {
|
|
461
|
+
logger.error(window.error.message);
|
|
462
|
+
process.exit(1);
|
|
463
|
+
}
|
|
464
|
+
return window.value;
|
|
465
|
+
}
|
|
446
466
|
async function promptFilters(args) {
|
|
447
467
|
if (!args.interactive) return;
|
|
448
468
|
for (const dim of FILTER_DIMS) {
|
|
@@ -480,126 +500,118 @@ async function promptFilters(args) {
|
|
|
480
500
|
if (ds && String(ds).length > 0) args["data-state"] = String(ds);
|
|
481
501
|
}
|
|
482
502
|
}
|
|
483
|
-
async function resolveLocalSite(store, hint) {
|
|
484
|
-
const siteIds = new Set((await store.engine.getWatermarks({ userId: store.userId })).map((w) => w.siteId).filter((siteId) => siteId !== void 0));
|
|
485
|
-
const exact = store.siteIdFor(hint);
|
|
486
|
-
if (siteIds.has(exact)) return hint;
|
|
487
|
-
const lowered = exact.toLowerCase();
|
|
488
|
-
const matches = [...siteIds].filter((siteId) => siteId.toLowerCase() === lowered);
|
|
489
|
-
return matches.length === 1 ? decodeSiteId(matches[0]) : void 0;
|
|
490
|
-
}
|
|
491
503
|
function buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter) {
|
|
492
504
|
const dims = dimNames.map((d) => DIM_COLUMNS[d]).filter((c) => Boolean(c));
|
|
493
505
|
const dateFilter = between(date, startDate, endDate);
|
|
494
506
|
const filter = dimensionFilter ? and(dateFilter, dimensionFilter) : dateFilter;
|
|
495
507
|
return gsc.select(...dims).where(filter).limit(rowLimit).getState();
|
|
496
508
|
}
|
|
497
|
-
function
|
|
498
|
-
const
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
}
|
|
508
|
-
async function assertRangeCovered(store, siteUrl, table, startDate, endDate, json, searchType) {
|
|
509
|
-
const watermarks = await store.engine.getWatermarks({
|
|
510
|
-
userId: store.userId,
|
|
511
|
-
siteId: store.siteIdFor(siteUrl),
|
|
512
|
-
table,
|
|
513
|
-
...searchType !== void 0 ? { searchType } : {}
|
|
514
|
-
});
|
|
515
|
-
const wm = watermarks[0];
|
|
516
|
-
const states = await store.engine.getSyncStates({
|
|
517
|
-
userId: store.userId,
|
|
518
|
-
siteId: store.siteIdFor(siteUrl),
|
|
519
|
-
table,
|
|
520
|
-
searchType: searchType ?? "web",
|
|
521
|
-
state: "done"
|
|
509
|
+
async function runSqlMode(opts) {
|
|
510
|
+
const ctx = await createCommandContext({ needsStore: true });
|
|
511
|
+
const store = ctx.store;
|
|
512
|
+
const found = opts.site ? await ctx.matchSite(opts.site, { scope: "store" }) : void 0;
|
|
513
|
+
if (found && found.kind !== "resolved") throw new Error(formatSiteResolution(found, "store"));
|
|
514
|
+
const site = found?.kind === "resolved" ? found.siteUrl : void 0;
|
|
515
|
+
const siteIds = site ? [store.siteIdFor(site)] : void 0;
|
|
516
|
+
const views = await openSqlViews(store, {
|
|
517
|
+
...siteIds ? { siteIds } : {},
|
|
518
|
+
...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
|
|
522
519
|
});
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
520
|
+
if (opts.mode.kind === "sql") {
|
|
521
|
+
const tables = referencedTables(opts.mode.sql, allTables());
|
|
522
|
+
if (tables.length > 0) {
|
|
523
|
+
const req = {
|
|
524
|
+
site,
|
|
525
|
+
siteHint: opts.site,
|
|
526
|
+
label: "query --sql",
|
|
527
|
+
localCapable: true,
|
|
528
|
+
liveCapable: false,
|
|
529
|
+
forceLive: opts.forceLive
|
|
530
|
+
};
|
|
531
|
+
const routeState = await readRouteState({
|
|
532
|
+
store,
|
|
533
|
+
site,
|
|
534
|
+
needs: [],
|
|
535
|
+
states: []
|
|
536
|
+
});
|
|
537
|
+
const stored = tables.some((table) => !views.emptyTables.includes(table));
|
|
538
|
+
const route = decideRoute(req, {
|
|
539
|
+
...routeState,
|
|
540
|
+
coverage: [{
|
|
541
|
+
kind: "any",
|
|
542
|
+
tables,
|
|
543
|
+
stored
|
|
544
|
+
}]
|
|
545
|
+
});
|
|
546
|
+
if (route.kind === "syncing" || route.kind === "prompt") {
|
|
547
|
+
views.close();
|
|
548
|
+
stopAtRoute(route, req, routeState.auth, { json: opts.format === "json" });
|
|
549
|
+
}
|
|
550
|
+
}
|
|
528
551
|
}
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
"
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
];
|
|
544
|
-
const nextCommand = `gscdump ${nextArgs.map((value) => /^[\w:./=-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`).join(" ")}`;
|
|
545
|
-
const message = !wm ? `No data synced for ${siteUrl} / ${table}.` : `Store coverage is incomplete for ${missingDates.length} requested dates.`;
|
|
546
|
-
if (json) {
|
|
547
|
-
const available = await store.engine.getWatermarks({
|
|
548
|
-
userId: store.userId,
|
|
549
|
-
siteId: store.siteIdFor(siteUrl)
|
|
550
|
-
});
|
|
551
|
-
console.log(JSON.stringify({ error: {
|
|
552
|
-
code: "STORE_RANGE_NOT_COVERED",
|
|
553
|
-
message,
|
|
554
|
-
siteUrl,
|
|
555
|
-
table,
|
|
556
|
-
range: {
|
|
557
|
-
start: startDate,
|
|
558
|
-
end: endDate
|
|
559
|
-
},
|
|
560
|
-
missingDates,
|
|
561
|
-
watermarks: watermarks.filter((w) => w.table === table),
|
|
562
|
-
availableTables: available.map((w) => ({
|
|
563
|
-
...w,
|
|
564
|
-
dimensions: tableDimensions(w.table)
|
|
565
|
-
})),
|
|
566
|
-
nextArgs,
|
|
567
|
-
nextCommand
|
|
568
|
-
} }, null, 2));
|
|
552
|
+
try {
|
|
553
|
+
const result = opts.mode.kind === "schema" ? await describeViews(views) : await runSql(views, opts.mode.sql);
|
|
554
|
+
const rows = result.rows;
|
|
555
|
+
const payload = opts.format === "table" && opts.mode.kind === "schema" ? renderSchema(rows) : opts.format === "table" ? renderTable(rows, columnsFor(rows), terminalOutputOptions(Boolean(opts.output && opts.output !== "-"))).join("\n") : opts.format === "csv" ? toCSV(rows.map((row) => Object.fromEntries(Object.entries(row).map(([key, value]) => [key, value !== null && typeof value === "object" ? JSON.stringify(value) : value]))), result.columns) : JSON.stringify({
|
|
556
|
+
total: rows.length,
|
|
557
|
+
data: rows,
|
|
558
|
+
...result.warnings.length > 0 ? { warnings: result.warnings } : {}
|
|
559
|
+
}, null, 2);
|
|
560
|
+
if (opts.output && opts.output !== "-") {
|
|
561
|
+
await fs.writeFile(opts.output, payload);
|
|
562
|
+
if (!opts.quiet) logger.info(`Written to ${opts.output}`);
|
|
563
|
+
} else console.log(payload);
|
|
564
|
+
} finally {
|
|
565
|
+
views.close();
|
|
569
566
|
}
|
|
570
|
-
logger.error(`${message} Run ${nextCommand}, or pass --live.`);
|
|
571
|
-
process.exit(1);
|
|
572
567
|
}
|
|
573
|
-
async function
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
568
|
+
async function runSql(views, sql) {
|
|
569
|
+
const warnings = referencedTables(sql, views.emptyTables).map((table) => `No synced data for table ${table}. Run gscdump sync --tables ${table} to fill it.`);
|
|
570
|
+
for (const warning of warnings) logger.warn(warning);
|
|
571
|
+
return {
|
|
572
|
+
...await views.run(sql).catch((e) => {
|
|
573
|
+
logger.error(`SQL failed: ${e.message}`);
|
|
574
|
+
process.exit(1);
|
|
575
|
+
}),
|
|
576
|
+
warnings
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
function renderSchema(rows) {
|
|
580
|
+
return rows.map((row) => {
|
|
581
|
+
const summary = row.rows === 0 ? "no synced data" : `${Number(row.rows).toLocaleString()} rows, ${row.start} to ${row.end}, ${row.search_types}, ${row.sites}`;
|
|
582
|
+
return `${row.view} ${summary}\n ${row.columns}`;
|
|
583
|
+
}).join("\n");
|
|
584
|
+
}
|
|
585
|
+
async function describeViews(views) {
|
|
586
|
+
const rows = [];
|
|
587
|
+
for (const view of views.views) {
|
|
588
|
+
const range = view.sources.length === 0 ? {
|
|
589
|
+
start: null,
|
|
590
|
+
end: null
|
|
591
|
+
} : (await views.run(`SELECT min(date) AS start, max(date) AS end FROM ${view.table}`)).rows[0];
|
|
592
|
+
rows.push({
|
|
593
|
+
view: view.table,
|
|
594
|
+
columns: view.columns.map((column) => `${column.name} ${column.type}`).join(", "),
|
|
595
|
+
sites: [...new Set(view.sources.map((source) => source.site))].join(", "),
|
|
596
|
+
search_types: [...new Set(view.sources.map((source) => source.searchType))].join(", "),
|
|
597
|
+
rows: view.sources.reduce((sum, source) => sum + source.rows, 0),
|
|
598
|
+
start: range.start ?? null,
|
|
599
|
+
end: range.end ?? null
|
|
600
|
+
});
|
|
577
601
|
}
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
}
|
|
591
|
-
logger.error(`SQL failed: ${e.message}`);
|
|
592
|
-
process.exit(1);
|
|
593
|
-
});
|
|
594
|
-
const payload = opts.format === "table" ? renderTable(rows, columnsFor(rows), terminalOutputOptions(Boolean(opts.output && opts.output !== "-"))).join("\n") : opts.format === "csv" ? toCSV(rows, Object.keys(rows[0] ?? {})) : JSON.stringify({
|
|
595
|
-
sql,
|
|
596
|
-
total: rows.length,
|
|
597
|
-
data: rows
|
|
598
|
-
}, (_key, value) => typeof value === "bigint" ? value.toString() : value, 2);
|
|
599
|
-
if (opts.output && opts.output !== "-") {
|
|
600
|
-
await fs.writeFile(opts.output, payload);
|
|
601
|
-
if (!opts.quiet) logger.info(`Written to ${opts.output}`);
|
|
602
|
-
} else console.log(payload);
|
|
602
|
+
return {
|
|
603
|
+
columns: [
|
|
604
|
+
"view",
|
|
605
|
+
"columns",
|
|
606
|
+
"sites",
|
|
607
|
+
"search_types",
|
|
608
|
+
"rows",
|
|
609
|
+
"start",
|
|
610
|
+
"end"
|
|
611
|
+
],
|
|
612
|
+
rows,
|
|
613
|
+
warnings: []
|
|
614
|
+
};
|
|
603
615
|
}
|
|
604
616
|
function logProfile(spans) {
|
|
605
617
|
if (spans.length === 0) {
|
|
@@ -632,7 +644,4 @@ async function writeOutput(opts) {
|
|
|
632
644
|
if (!opts.quiet) logger.info(`Written to ${opts.path}`);
|
|
633
645
|
} else console.log(content);
|
|
634
646
|
}
|
|
635
|
-
function isKnownTable(name) {
|
|
636
|
-
return allTables().includes(name);
|
|
637
|
-
}
|
|
638
647
|
export { queryCommand };
|