@gscdump/cli 3.8.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +113 -30
  2. package/bin/gscdump.mjs +21 -1
  3. package/dist/analysis-local.mjs +172 -53
  4. package/dist/auth-state.mjs +21 -6
  5. package/dist/auth.mjs +27 -18
  6. package/dist/bing-data.mjs +6 -2
  7. package/dist/bing-hosted.mjs +1 -1
  8. package/dist/cli-args.mjs +8 -0
  9. package/dist/cli.d.mts +5 -1
  10. package/dist/cli.mjs +41 -10
  11. package/dist/cloud-google.mjs +8 -1
  12. package/dist/command-meta.mjs +4 -4
  13. package/dist/command-registry.mjs +86 -10
  14. package/dist/commands/analyze.mjs +84 -43
  15. package/dist/commands/auth.mjs +46 -37
  16. package/dist/commands/bing.mjs +3 -3
  17. package/dist/commands/compact.mjs +12 -8
  18. package/dist/commands/config.mjs +14 -14
  19. package/dist/commands/doctor.mjs +26 -26
  20. package/dist/commands/dump.mjs +203 -105
  21. package/dist/commands/entities.mjs +10 -113
  22. package/dist/commands/gc.mjs +4 -3
  23. package/dist/commands/indexing-urls.mjs +185 -0
  24. package/dist/commands/indexing.mjs +12 -8
  25. package/dist/commands/init.mjs +77 -18
  26. package/dist/commands/inspect.mjs +97 -110
  27. package/dist/commands/mcp.mjs +37 -57
  28. package/dist/commands/papercut.mjs +1 -1
  29. package/dist/commands/profile.mjs +9 -17
  30. package/dist/commands/query.mjs +219 -210
  31. package/dist/commands/report.mjs +68 -85
  32. package/dist/commands/rollups.mjs +9 -6
  33. package/dist/commands/sitemaps.mjs +51 -104
  34. package/dist/commands/sites.mjs +40 -24
  35. package/dist/commands/stats.mjs +14 -17
  36. package/dist/commands/store-purge.mjs +9 -5
  37. package/dist/commands/store.mjs +0 -12
  38. package/dist/commands/sync.mjs +774 -296
  39. package/dist/config.mjs +7 -4
  40. package/dist/context.mjs +51 -21
  41. package/dist/coverage.mjs +251 -0
  42. package/dist/dump-bing.mjs +71 -0
  43. package/dist/dump-writers.mjs +246 -0
  44. package/dist/error-handler.mjs +108 -38
  45. package/dist/filters.mjs +125 -0
  46. package/dist/hosted-site.mjs +91 -0
  47. package/dist/inspect-urls.mjs +76 -0
  48. package/dist/inspection-record.mjs +93 -0
  49. package/dist/local-entities.mjs +658 -0
  50. package/dist/mcp/errors.mjs +71 -18
  51. package/dist/mcp/handlers/diagnostics.mjs +15 -12
  52. package/dist/mcp/handlers/reports.mjs +33 -42
  53. package/dist/mcp/server/index.mjs +80 -38
  54. package/dist/mcp/types.mjs +7 -7
  55. package/dist/package.mjs +1 -1
  56. package/dist/quota-ledger.mjs +370 -0
  57. package/dist/render/analysis.mjs +8 -1
  58. package/dist/render/report.mjs +8 -1
  59. package/dist/request-pacer.mjs +29 -0
  60. package/dist/route.mjs +295 -0
  61. package/dist/sitemap.mjs +4 -0
  62. package/dist/sql-views.mjs +117 -0
  63. package/dist/store-sites.mjs +134 -0
  64. package/dist/sync-plan.mjs +129 -0
  65. package/dist/sync-run.mjs +96 -0
  66. package/dist/table-sources.mjs +57 -0
  67. package/dist/token-info.mjs +54 -0
  68. package/dist/utils.mjs +28 -19
  69. package/dist/window.mjs +159 -0
  70. package/package.json +17 -14
  71. package/skills/gscdump/SKILL.md +167 -37
  72. package/dist/commands/export.mjs +0 -76
  73. package/dist/native-duckdb.mjs +0 -46
@@ -1,21 +1,25 @@
1
1
  import { queryCommandMeta } from "../command-meta.mjs";
2
+ import { useCliRuntime } from "../runtime.mjs";
2
3
  import { loadConfig } from "../config.mjs";
3
4
  import { terminalOutputOptions } from "../render/terminal.mjs";
4
5
  import { ALL_SEARCH_TYPES, logger, parseSearchType, toCSV } from "../utils.mjs";
5
- import { allTables, inferTable, tableDimensions } from "../local-store.mjs";
6
- import { createCommandContext } from "../context.mjs";
7
- import { gscErrorHandler } from "../error-handler.mjs";
6
+ import { allTables } from "../local-store.mjs";
7
+ import { createCommandContext, formatSiteResolution } from "../context.mjs";
8
+ import { decideRoute, describeStop, liveNote, readRouteState, readSiteStates, resolveReadSite, stopAtRoute } from "../route.mjs";
9
+ import { DEFAULT_WINDOW, checkWindowFlags, newestDoneDate, parseWindowFlags } from "../window.mjs";
8
10
  import { renderTable } from "../render/layout.mjs";
9
11
  import { asRecord, columnsFor } from "../render/analysis.mjs";
12
+ import { FILTER_DIMS, filterDimensions, parseFilterArgs, toLiveFilter, toLocalFilter } from "../filters.mjs";
10
13
  import { renderQuery } from "../render/query.mjs";
14
+ import { openSqlViews, referencedTables } from "../sql-views.mjs";
11
15
  import process from "node:process";
12
16
  import { defineCommand } from "citty";
13
17
  import fs from "node:fs/promises";
14
- import { and, between, contains, country, date, device, eq, gsc, hour, notRegex, page, query, regex, searchAppearance } from "gscdump/query";
15
- import { decodeSiteId } from "gscdump/tenant";
18
+ import { and, between, country, date, device, gsc, hour, page, query, searchAppearance } from "gscdump/query";
19
+ import { getLatestGscDate } from "gscdump/dates";
16
20
  import { cancel, isCancel, multiselect, text } from "@clack/prompts";
17
- import { daysAgoUtc } from "gscdump/dates";
18
21
  import { collectSpans } from "@gscdump/engine/profile";
22
+ import { inferDataset, isDatasetResolvable } from "gscdump/query/plan";
19
23
  const DIMENSIONS = [
20
24
  "page",
21
25
  "query",
@@ -34,20 +38,6 @@ const DIM_COLUMNS = {
34
38
  device,
35
39
  searchAppearance
36
40
  };
37
- const FILTER_DIMS = [
38
- "query",
39
- "page",
40
- "country",
41
- "device",
42
- "searchAppearance"
43
- ];
44
- const FILTER_COL = {
45
- query,
46
- page,
47
- country,
48
- device,
49
- searchAppearance
50
- };
51
41
  const DATA_STATES = [
52
42
  "all",
53
43
  "final",
@@ -59,27 +49,6 @@ const AGGREGATION_TYPES = [
59
49
  "byProperty"
60
50
  ];
61
51
  const POSITIVE_INTEGER_RE = /^\d+$/;
62
- const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
63
- function buildFilterFromArg(col, raw) {
64
- if (raw.startsWith("!~")) return makeLeaf(col, "notContains", raw.slice(2));
65
- if (raw.startsWith("!re:")) return notRegex(col, raw.slice(4));
66
- if (raw.startsWith("!")) return makeLeaf(col, "notEquals", raw.slice(1));
67
- if (raw.startsWith("~")) return contains(col, raw.slice(1));
68
- if (raw.startsWith("re:")) return regex(col, raw.slice(3));
69
- if (raw.startsWith("contains:")) return contains(col, raw.slice(9));
70
- if (raw.startsWith("eq:")) return eq(col, raw.slice(3));
71
- return eq(col, raw);
72
- }
73
- function makeLeaf(col, operator, value) {
74
- return {
75
- _constraints: {},
76
- _filters: [{
77
- dimension: col.dimension,
78
- operator,
79
- expression: value
80
- }]
81
- };
82
- }
83
52
  async function runLiveQuery(client, siteUrl, opts) {
84
53
  const allRows = [];
85
54
  const totalLimit = Math.max(0, opts.rowLimit);
@@ -137,7 +106,7 @@ const queryCommand = defineCommand({
137
106
  "site": {
138
107
  type: "string",
139
108
  alias: "s",
140
- description: "Site URL (e.g., sc-domain:example.com)"
109
+ description: "Site, for example example.com"
141
110
  },
142
111
  "dimensions": {
143
112
  type: "string",
@@ -169,11 +138,12 @@ const queryCommand = defineCommand({
169
138
  },
170
139
  "sql": {
171
140
  type: "string",
172
- description: "Raw DuckDB SQL using {{FILES}} as the file list placeholder (bypasses builder)"
141
+ description: "DuckDB SQL over one view per Store table, with site and search_type columns. Example: SELECT search_type, SUM(clicks) AS clicks, gsc_position(sum_position, impressions) AS position FROM pages GROUP BY search_type. Run --schema to list views"
173
142
  },
174
- "table": {
175
- type: "string",
176
- description: "Analytics table for --sql (default: pages)"
143
+ "schema": {
144
+ type: "boolean",
145
+ default: false,
146
+ description: "List the --sql views, their columns, and the date range of each view, then exit"
177
147
  },
178
148
  "live": {
179
149
  type: "boolean",
@@ -214,7 +184,7 @@ const queryCommand = defineCommand({
214
184
  },
215
185
  "type": {
216
186
  type: "string",
217
- description: `Search type (live mode only). One of: ${ALL_SEARCH_TYPES.join(",")}`
187
+ description: `Search type. One of: ${ALL_SEARCH_TYPES.join(",")} (default: web; --sql and --schema default to every search type)`
218
188
  },
219
189
  "data-state": {
220
190
  type: "string",
@@ -242,20 +212,28 @@ const queryCommand = defineCommand({
242
212
  logger.error("Invalid --format. Use table, json, or csv.");
243
213
  process.exit(1);
244
214
  }
245
- if (args.sql) {
246
- await runRawSqlMode({
247
- sql: String(args.sql),
215
+ if (args.sql || args.schema) {
216
+ await runSqlMode({
217
+ mode: args.schema ? { kind: "schema" } : {
218
+ kind: "sql",
219
+ sql: String(args.sql)
220
+ },
248
221
  site: args.site ? String(args.site) : void 0,
249
- table: args.table ? String(args.table) : "pages",
250
222
  output: args.output ? String(args.output) : void 0,
251
223
  format,
252
224
  quiet: Boolean(args.quiet),
225
+ forceLive: Boolean(args.live),
253
226
  searchType: parseSearchType(args.type, "--type")
254
227
  });
255
228
  return;
256
229
  }
257
230
  const dimNames = await resolveDimensions(args);
258
- const { startDate, endDate } = await resolveRange(args);
231
+ const windowFlags = await promptRange(args);
232
+ const invalidWindow = checkWindowFlags(windowFlags);
233
+ if (invalidWindow) {
234
+ logger.error(invalidWindow.message);
235
+ process.exit(1);
236
+ }
259
237
  await promptFilters(args);
260
238
  const limitArg = String(args.limit ?? ctxConfig.defaultLimit ?? 1e3);
261
239
  const rowLimit = Number(limitArg);
@@ -263,10 +241,12 @@ const queryCommand = defineCommand({
263
241
  logger.error("Invalid --limit. Use a positive safe integer, such as --limit 1000.");
264
242
  process.exit(1);
265
243
  }
266
- const dimensionFilter = buildDimensionFilter(args);
244
+ const filters = parseFilterArgs(args);
267
245
  const searchType = parseSearchType(args.type ?? ctxConfig.defaultSearchType, "--type");
246
+ const localSearchType = searchType ?? "web";
268
247
  const dataState = args["data-state"] ? String(args["data-state"]) : ctxConfig.defaultDataState;
269
248
  const aggregationType = args["aggregation-type"] ? String(args["aggregation-type"]) : void 0;
249
+ const filterDims = filterDimensions(filters);
270
250
  if (dataState && !DATA_STATES.includes(dataState)) {
271
251
  logger.error(`Invalid --data-state: ${dataState}. Allowed: ${DATA_STATES.join(", ")}`);
272
252
  process.exit(1);
@@ -275,15 +255,54 @@ const queryCommand = defineCommand({
275
255
  logger.error(`Invalid --aggregation-type: ${aggregationType}. Allowed: ${AGGREGATION_TYPES.join(", ")}`);
276
256
  process.exit(1);
277
257
  }
278
- const ctx = await createCommandContext({
258
+ const forceLive = Boolean(args.live);
259
+ const resolvable = isDatasetResolvable(dimNames, filterDims);
260
+ if (!resolvable && !forceLive) throw new Error(`No Store table holds ${[.../* @__PURE__ */ new Set([...dimNames, ...filterDims])].join(" with ")}. Remove a dimension or filter, or pass --live.`);
261
+ const storeCtx = await createCommandContext({ needsStore: true });
262
+ const store = storeCtx.store;
263
+ let liveCtx;
264
+ const connect = () => liveCtx ??= createCommandContext({
279
265
  needsAuth: true,
280
- needsStore: !args.live,
281
266
  interactive: Boolean(args.interactive)
282
267
  });
283
- const hint = args.site ? String(args.site) : ctx.config.defaultSite;
284
- const exactHint = !args.live && hint && /^(?:sc-domain:\S+|https?:\/\/\S+)$/.test(hint) ? hint : void 0;
285
- const siteUrl = exactHint ? await resolveLocalSite(ctx.store, exactHint) ?? await ctx.resolveSite(hint) : await ctx.resolveSite(hint);
286
- if (args.live) {
268
+ const { site, siteHint, auth } = await resolveReadSite(storeCtx, args.site ? String(args.site) : void 0, {
269
+ forceLive,
270
+ connect
271
+ });
272
+ const table = resolvable ? inferDataset(dimNames, filterDims) : void 0;
273
+ const states = site ? await readSiteStates(store, site) : [];
274
+ const { start: startDate, end: endDate } = windowOrExit(windowFlags, forceLive || !table ? getLatestGscDate() : newestDoneDate(states, [table]) ?? getLatestGscDate());
275
+ const req = {
276
+ site,
277
+ siteHint,
278
+ label: "query",
279
+ localCapable: resolvable,
280
+ liveCapable: true,
281
+ forceLive,
282
+ argv: useCliRuntime().rawArgs
283
+ };
284
+ const routeState = await readRouteState({
285
+ store,
286
+ site,
287
+ needs: table ? [{
288
+ kind: "window",
289
+ period: "current",
290
+ table,
291
+ searchType: localSearchType,
292
+ window: {
293
+ start: startDate,
294
+ end: endDate
295
+ }
296
+ }] : [],
297
+ states,
298
+ auth
299
+ });
300
+ const route = decideRoute(req, routeState);
301
+ const json = format === "json";
302
+ if (route.kind === "live") {
303
+ const live = await connect();
304
+ const siteUrl = site ?? await live.resolveSite(void 0);
305
+ const dimensionFilter = toLiveFilter(filters, siteUrl);
287
306
  if (args.explain) {
288
307
  const body = {
289
308
  startDate,
@@ -297,12 +316,14 @@ const queryCommand = defineCommand({
297
316
  if (dimensionFilter) body.dimensionFilterGroups = filterToGroups(dimensionFilter);
298
317
  console.log(JSON.stringify({
299
318
  siteUrl,
319
+ source: "live",
300
320
  body
301
321
  }, null, 2));
302
322
  return;
303
323
  }
324
+ if (route.reason === "no-store-data") logger.warn(liveNote(siteUrl));
304
325
  if (!args.quiet) logger.debug(`Querying ${siteUrl} via live GSC API...`);
305
- const result = await runLiveQuery(ctx.client, siteUrl, {
326
+ const result = await runLiveQuery(live.client, siteUrl, {
306
327
  startDate,
307
328
  endDate,
308
329
  dimensions: dimNames,
@@ -311,7 +332,7 @@ const queryCommand = defineCommand({
311
332
  dataState,
312
333
  aggregationType,
313
334
  dimensionFilter
314
- }).catch(gscErrorHandler);
335
+ });
315
336
  await writeOutput({
316
337
  output: {
317
338
  siteUrl,
@@ -321,7 +342,8 @@ const queryCommand = defineCommand({
321
342
  end: endDate
322
343
  },
323
344
  total: result.rows.length,
324
- data: result.rows
345
+ data: result.rows,
346
+ meta: { source: "live" }
325
347
  },
326
348
  format,
327
349
  path: args.output ? String(args.output) : void 0,
@@ -329,26 +351,28 @@ const queryCommand = defineCommand({
329
351
  });
330
352
  return;
331
353
  }
332
- if (dataState || aggregationType) logger.warn("--data-state / --aggregation-type are ignored without --live");
333
- if (!args.quiet) logger.debug(`Querying ${siteUrl} from local Parquet store...`);
334
- const state = buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter);
335
- const store = ctx.store;
336
- const table = inferTable(dimNames);
354
+ const state = buildLocalState(dimNames, startDate, endDate, rowLimit, toLocalFilter(filters));
337
355
  if (args.explain) {
356
+ const stop = route.kind === "local" ? void 0 : describeStop(route, req, routeState.auth);
338
357
  console.log(JSON.stringify({
339
- siteUrl,
358
+ siteUrl: site ?? null,
359
+ source: "local",
340
360
  table,
341
- state
361
+ state,
362
+ ...stop ? { stop } : {}
342
363
  }, null, 2));
343
364
  return;
344
365
  }
345
- await assertRangeCovered(store, siteUrl, table, startDate, endDate, format === "json", searchType);
366
+ if (route.kind === "syncing" || route.kind === "prompt") stopAtRoute(route, req, routeState.auth, { json });
367
+ if (dataState || aggregationType) logger.warn("--data-state / --aggregation-type are ignored without --live");
368
+ const siteUrl = site;
369
+ if (!args.quiet) logger.debug(`Querying ${siteUrl} from local Parquet store...`);
346
370
  const probe = Boolean(args.profile) ? collectSpans() : void 0;
347
371
  const result = await store.engine.query({
348
372
  userId: store.userId,
349
373
  siteId: store.siteIdFor(siteUrl),
350
374
  table,
351
- ...searchType !== void 0 ? { searchType } : {},
375
+ searchType: localSearchType,
352
376
  ...probe ? { profiler: probe.profiler } : {}
353
377
  }, state).catch((e) => {
354
378
  logger.error(`Query failed: ${e.message}`);
@@ -364,7 +388,8 @@ const queryCommand = defineCommand({
364
388
  end: endDate
365
389
  },
366
390
  total: result.rows.length,
367
- data: result.rows
391
+ data: result.rows,
392
+ meta: { source: "local" }
368
393
  },
369
394
  format,
370
395
  path: args.output ? String(args.output) : void 0,
@@ -398,51 +423,46 @@ async function resolveDimensions(args) {
398
423
  }
399
424
  return ["page", "query"];
400
425
  }
401
- async function resolveRange(args) {
402
- let startDate = args.start != null ? String(args.start) : void 0;
403
- let endDate = args.end != null ? String(args.end) : void 0;
426
+ async function promptRange(args) {
427
+ let start = args.start != null ? String(args.start) : void 0;
428
+ let end = args.end != null ? String(args.end) : void 0;
404
429
  if (args.interactive) {
405
- if (startDate === void 0) {
430
+ if (start === void 0) {
406
431
  const startInput = await text({
407
- message: "Start date (YYYY-MM-DD)",
408
- placeholder: daysAgoUtc(28)
432
+ message: "Start date (YYYY-MM-DD, blank for the last 28 synced days)",
433
+ placeholder: ""
409
434
  });
410
435
  if (isCancel(startInput)) {
411
436
  cancel("Cancelled");
412
437
  process.exit(0);
413
438
  }
414
- startDate = String(startInput) || daysAgoUtc(28);
439
+ start = String(startInput) || void 0;
415
440
  }
416
- if (endDate === void 0) {
441
+ if (end === void 0) {
417
442
  const endInput = await text({
418
- message: "End date (YYYY-MM-DD)",
419
- placeholder: daysAgoUtc(3)
443
+ message: "End date (YYYY-MM-DD, blank for the newest synced day)",
444
+ placeholder: ""
420
445
  });
421
446
  if (isCancel(endInput)) {
422
447
  cancel("Cancelled");
423
448
  process.exit(0);
424
449
  }
425
- endDate = String(endInput) || daysAgoUtc(3);
450
+ end = String(endInput) || void 0;
426
451
  }
427
452
  }
428
- startDate ??= daysAgoUtc(31);
429
- endDate ??= daysAgoUtc(3);
430
- for (const [flag, value] of [["--start", startDate], ["--end", endDate]]) {
431
- const date = /* @__PURE__ */ new Date(`${value}T00:00:00Z`);
432
- if (!ISO_DATE_RE.test(value) || !Number.isFinite(date.getTime()) || date.toISOString().slice(0, 10) !== value) {
433
- logger.error(`Invalid ${flag}. Use a calendar date in YYYY-MM-DD format.`);
434
- process.exit(1);
435
- }
436
- }
437
- if (startDate > endDate) {
438
- logger.error("Invalid date range. --start must be on or before --end.");
439
- process.exit(1);
440
- }
441
453
  return {
442
- startDate,
443
- endDate
454
+ start,
455
+ end
444
456
  };
445
457
  }
458
+ function windowOrExit(flags, anchor) {
459
+ const window = parseWindowFlags(flags, DEFAULT_WINDOW, anchor);
460
+ if (!window.ok) {
461
+ logger.error(window.error.message);
462
+ process.exit(1);
463
+ }
464
+ return window.value;
465
+ }
446
466
  async function promptFilters(args) {
447
467
  if (!args.interactive) return;
448
468
  for (const dim of FILTER_DIMS) {
@@ -480,126 +500,118 @@ async function promptFilters(args) {
480
500
  if (ds && String(ds).length > 0) args["data-state"] = String(ds);
481
501
  }
482
502
  }
483
- async function resolveLocalSite(store, hint) {
484
- const siteIds = new Set((await store.engine.getWatermarks({ userId: store.userId })).map((w) => w.siteId).filter((siteId) => siteId !== void 0));
485
- const exact = store.siteIdFor(hint);
486
- if (siteIds.has(exact)) return hint;
487
- const lowered = exact.toLowerCase();
488
- const matches = [...siteIds].filter((siteId) => siteId.toLowerCase() === lowered);
489
- return matches.length === 1 ? decodeSiteId(matches[0]) : void 0;
490
- }
491
503
  function buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter) {
492
504
  const dims = dimNames.map((d) => DIM_COLUMNS[d]).filter((c) => Boolean(c));
493
505
  const dateFilter = between(date, startDate, endDate);
494
506
  const filter = dimensionFilter ? and(dateFilter, dimensionFilter) : dateFilter;
495
507
  return gsc.select(...dims).where(filter).limit(rowLimit).getState();
496
508
  }
497
- function buildDimensionFilter(args) {
498
- const leaves = [];
499
- for (const dim of FILTER_DIMS) {
500
- const raw = args[dim];
501
- if (raw == null || raw === "") continue;
502
- leaves.push(buildFilterFromArg(FILTER_COL[dim], String(raw)));
503
- }
504
- if (leaves.length === 0) return void 0;
505
- if (leaves.length === 1) return leaves[0];
506
- return and(...leaves);
507
- }
508
- async function assertRangeCovered(store, siteUrl, table, startDate, endDate, json, searchType) {
509
- const watermarks = await store.engine.getWatermarks({
510
- userId: store.userId,
511
- siteId: store.siteIdFor(siteUrl),
512
- table,
513
- ...searchType !== void 0 ? { searchType } : {}
514
- });
515
- const wm = watermarks[0];
516
- const states = await store.engine.getSyncStates({
517
- userId: store.userId,
518
- siteId: store.siteIdFor(siteUrl),
519
- table,
520
- searchType: searchType ?? "web",
521
- state: "done"
509
+ async function runSqlMode(opts) {
510
+ const ctx = await createCommandContext({ needsStore: true });
511
+ const store = ctx.store;
512
+ const found = opts.site ? await ctx.matchSite(opts.site, { scope: "store" }) : void 0;
513
+ if (found && found.kind !== "resolved") throw new Error(formatSiteResolution(found, "store"));
514
+ const site = found?.kind === "resolved" ? found.siteUrl : void 0;
515
+ const siteIds = site ? [store.siteIdFor(site)] : void 0;
516
+ const views = await openSqlViews(store, {
517
+ ...siteIds ? { siteIds } : {},
518
+ ...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
522
519
  });
523
- const completed = new Set(states.map((state) => state.date));
524
- const missingDates = [];
525
- for (let date = Date.parse(startDate); date <= Date.parse(endDate); date += 864e5) {
526
- const day = new Date(date).toISOString().slice(0, 10);
527
- if (!completed.has(day)) missingDates.push(day);
520
+ if (opts.mode.kind === "sql") {
521
+ const tables = referencedTables(opts.mode.sql, allTables());
522
+ if (tables.length > 0) {
523
+ const req = {
524
+ site,
525
+ siteHint: opts.site,
526
+ label: "query --sql",
527
+ localCapable: true,
528
+ liveCapable: false,
529
+ forceLive: opts.forceLive
530
+ };
531
+ const routeState = await readRouteState({
532
+ store,
533
+ site,
534
+ needs: [],
535
+ states: []
536
+ });
537
+ const stored = tables.some((table) => !views.emptyTables.includes(table));
538
+ const route = decideRoute(req, {
539
+ ...routeState,
540
+ coverage: [{
541
+ kind: "any",
542
+ tables,
543
+ stored
544
+ }]
545
+ });
546
+ if (route.kind === "syncing" || route.kind === "prompt") {
547
+ views.close();
548
+ stopAtRoute(route, req, routeState.auth, { json: opts.format === "json" });
549
+ }
550
+ }
528
551
  }
529
- if (missingDates.length === 0) return;
530
- const nextArgs = [
531
- "sync",
532
- "--site",
533
- siteUrl,
534
- "--start",
535
- startDate,
536
- "--end",
537
- endDate,
538
- "--tables",
539
- table,
540
- "--types",
541
- searchType ?? "web",
542
- "--json"
543
- ];
544
- const nextCommand = `gscdump ${nextArgs.map((value) => /^[\w:./=-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`).join(" ")}`;
545
- const message = !wm ? `No data synced for ${siteUrl} / ${table}.` : `Store coverage is incomplete for ${missingDates.length} requested dates.`;
546
- if (json) {
547
- const available = await store.engine.getWatermarks({
548
- userId: store.userId,
549
- siteId: store.siteIdFor(siteUrl)
550
- });
551
- console.log(JSON.stringify({ error: {
552
- code: "STORE_RANGE_NOT_COVERED",
553
- message,
554
- siteUrl,
555
- table,
556
- range: {
557
- start: startDate,
558
- end: endDate
559
- },
560
- missingDates,
561
- watermarks: watermarks.filter((w) => w.table === table),
562
- availableTables: available.map((w) => ({
563
- ...w,
564
- dimensions: tableDimensions(w.table)
565
- })),
566
- nextArgs,
567
- nextCommand
568
- } }, null, 2));
552
+ try {
553
+ const result = opts.mode.kind === "schema" ? await describeViews(views) : await runSql(views, opts.mode.sql);
554
+ const rows = result.rows;
555
+ const payload = opts.format === "table" && opts.mode.kind === "schema" ? renderSchema(rows) : opts.format === "table" ? renderTable(rows, columnsFor(rows), terminalOutputOptions(Boolean(opts.output && opts.output !== "-"))).join("\n") : opts.format === "csv" ? toCSV(rows.map((row) => Object.fromEntries(Object.entries(row).map(([key, value]) => [key, value !== null && typeof value === "object" ? JSON.stringify(value) : value]))), result.columns) : JSON.stringify({
556
+ total: rows.length,
557
+ data: rows,
558
+ ...result.warnings.length > 0 ? { warnings: result.warnings } : {}
559
+ }, null, 2);
560
+ if (opts.output && opts.output !== "-") {
561
+ await fs.writeFile(opts.output, payload);
562
+ if (!opts.quiet) logger.info(`Written to ${opts.output}`);
563
+ } else console.log(payload);
564
+ } finally {
565
+ views.close();
569
566
  }
570
- logger.error(`${message} Run ${nextCommand}, or pass --live.`);
571
- process.exit(1);
572
567
  }
573
- async function runRawSqlMode(opts) {
574
- if (!isKnownTable(opts.table)) {
575
- logger.error(`Unknown table "${opts.table}". Known: ${allTables().join(", ")}`);
576
- process.exit(1);
568
+ async function runSql(views, sql) {
569
+ const warnings = referencedTables(sql, views.emptyTables).map((table) => `No synced data for table ${table}. Run gscdump sync --tables ${table} to fill it.`);
570
+ for (const warning of warnings) logger.warn(warning);
571
+ return {
572
+ ...await views.run(sql).catch((e) => {
573
+ logger.error(`SQL failed: ${e.message}`);
574
+ process.exit(1);
575
+ }),
576
+ warnings
577
+ };
578
+ }
579
+ function renderSchema(rows) {
580
+ return rows.map((row) => {
581
+ const summary = row.rows === 0 ? "no synced data" : `${Number(row.rows).toLocaleString()} rows, ${row.start} to ${row.end}, ${row.search_types}, ${row.sites}`;
582
+ return `${row.view} ${summary}\n ${row.columns}`;
583
+ }).join("\n");
584
+ }
585
+ async function describeViews(views) {
586
+ const rows = [];
587
+ for (const view of views.views) {
588
+ const range = view.sources.length === 0 ? {
589
+ start: null,
590
+ end: null
591
+ } : (await views.run(`SELECT min(date) AS start, max(date) AS end FROM ${view.table}`)).rows[0];
592
+ rows.push({
593
+ view: view.table,
594
+ columns: view.columns.map((column) => `${column.name} ${column.type}`).join(", "),
595
+ sites: [...new Set(view.sources.map((source) => source.site))].join(", "),
596
+ search_types: [...new Set(view.sources.map((source) => source.searchType))].join(", "),
597
+ rows: view.sources.reduce((sum, source) => sum + source.rows, 0),
598
+ start: range.start ?? null,
599
+ end: range.end ?? null
600
+ });
577
601
  }
578
- const ctx = await createCommandContext({
579
- needsAuth: true,
580
- needsStore: true
581
- });
582
- const siteUrl = await ctx.resolveSite(opts.site);
583
- const store = ctx.store;
584
- if (!opts.quiet) logger.debug(`Running raw SQL over table "${opts.table}" for ${siteUrl}`);
585
- const { rows, sql } = await store.runRawSql({
586
- sql: opts.sql,
587
- siteUrl,
588
- table: opts.table,
589
- ...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
590
- }).catch((e) => {
591
- logger.error(`SQL failed: ${e.message}`);
592
- process.exit(1);
593
- });
594
- const payload = opts.format === "table" ? renderTable(rows, columnsFor(rows), terminalOutputOptions(Boolean(opts.output && opts.output !== "-"))).join("\n") : opts.format === "csv" ? toCSV(rows, Object.keys(rows[0] ?? {})) : JSON.stringify({
595
- sql,
596
- total: rows.length,
597
- data: rows
598
- }, (_key, value) => typeof value === "bigint" ? value.toString() : value, 2);
599
- if (opts.output && opts.output !== "-") {
600
- await fs.writeFile(opts.output, payload);
601
- if (!opts.quiet) logger.info(`Written to ${opts.output}`);
602
- } else console.log(payload);
602
+ return {
603
+ columns: [
604
+ "view",
605
+ "columns",
606
+ "sites",
607
+ "search_types",
608
+ "rows",
609
+ "start",
610
+ "end"
611
+ ],
612
+ rows,
613
+ warnings: []
614
+ };
603
615
  }
604
616
  function logProfile(spans) {
605
617
  if (spans.length === 0) {
@@ -632,7 +644,4 @@ async function writeOutput(opts) {
632
644
  if (!opts.quiet) logger.info(`Written to ${opts.path}`);
633
645
  } else console.log(content);
634
646
  }
635
- function isKnownTable(name) {
636
- return allTables().includes(name);
637
- }
638
647
  export { queryCommand };