@gscdump/cli 3.6.2 → 3.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-args.mjs +63 -0
- package/dist/cli.mjs +3 -0
- package/dist/commands/query.mjs +72 -16
- package/dist/commands/stats.mjs +12 -14
- package/dist/commands/store.mjs +1 -0
- package/dist/commands/sync.mjs +32 -6
- package/dist/local-store.mjs +13 -2
- package/dist/package.mjs +1 -1
- package/package.json +6 -6
- package/skills/gscdump/SKILL.md +46 -8
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
async function resolve(value) {
|
|
3
|
+
return typeof value === "function" ? value() : value;
|
|
4
|
+
}
|
|
5
|
+
async function checkCliArgs(command, rawArgs, path = "gscdump") {
|
|
6
|
+
const definitions = await resolve(command.args ?? {});
|
|
7
|
+
const options = {
|
|
8
|
+
"help": { type: "boolean" },
|
|
9
|
+
"h": { type: "boolean" },
|
|
10
|
+
"version": { type: "boolean" },
|
|
11
|
+
"no-color": { type: "boolean" }
|
|
12
|
+
};
|
|
13
|
+
for (const [name, definition] of Object.entries(definitions)) {
|
|
14
|
+
if (definition.type === "positional") continue;
|
|
15
|
+
const type = definition.type === "boolean" ? "boolean" : "string";
|
|
16
|
+
const alias = "alias" in definition ? definition.alias : void 0;
|
|
17
|
+
const names = [
|
|
18
|
+
name,
|
|
19
|
+
...typeof alias === "string" ? [alias] : alias ?? [],
|
|
20
|
+
name.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()),
|
|
21
|
+
name.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)
|
|
22
|
+
];
|
|
23
|
+
for (const key of names) {
|
|
24
|
+
options[key] = { type };
|
|
25
|
+
if (type === "boolean" && !key.startsWith("no-")) options[`no-${key}`] = { type };
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const { tokens } = parseArgs({
|
|
29
|
+
args: rawArgs,
|
|
30
|
+
options,
|
|
31
|
+
strict: false,
|
|
32
|
+
allowPositionals: true,
|
|
33
|
+
tokens: true
|
|
34
|
+
});
|
|
35
|
+
const subCommands = await resolve(command.subCommands ?? {});
|
|
36
|
+
const positional = tokens.find((token) => token.kind === "positional");
|
|
37
|
+
const hasSubCommands = Object.keys(subCommands).length > 0;
|
|
38
|
+
const boundary = hasSubCommands && positional ? positional.index : rawArgs.length;
|
|
39
|
+
for (const token of tokens) {
|
|
40
|
+
if (token.index >= boundary || token.kind !== "option") continue;
|
|
41
|
+
const option = options[token.name];
|
|
42
|
+
if (!option) {
|
|
43
|
+
const matches = Object.keys(definitions).filter((name) => definitions[name]?.type !== "positional" && name.startsWith(token.name) && name.length - token.name.length <= 2);
|
|
44
|
+
const suggestion = matches.length === 1 ? ` Use --${matches[0]}.` : "";
|
|
45
|
+
return `Unknown option ${token.rawName}.${suggestion} Run ${path} --help.`;
|
|
46
|
+
}
|
|
47
|
+
if (option.type === "string" && (token.value === void 0 || !token.inlineValue && token.value.startsWith("-"))) return `${token.rawName} requires a value. Use ${token.rawName}=VALUE.`;
|
|
48
|
+
}
|
|
49
|
+
if (hasSubCommands && positional?.kind === "positional") {
|
|
50
|
+
let selected = subCommands[positional.value];
|
|
51
|
+
if (!selected) for (const candidate of Object.values(subCommands)) {
|
|
52
|
+
const meta = await resolve((await resolve(candidate))?.meta ?? {});
|
|
53
|
+
if ((typeof meta.alias === "string" ? [meta.alias] : meta.alias ?? []).includes(positional.value)) {
|
|
54
|
+
selected = candidate;
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (!selected) return `Unknown command ${positional.value}. Run ${path} --help.`;
|
|
59
|
+
const child = await resolve(selected);
|
|
60
|
+
if (child) return checkCliArgs(child, rawArgs.slice(boundary + 1), `${path} ${positional.value}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
export { checkCliArgs };
|
package/dist/cli.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createCliRuntime, runWithCliRuntime, useCliRuntime } from "./runtime.mjs";
|
|
2
2
|
import { parseAuthMode } from "./auth-state.mjs";
|
|
3
|
+
import { checkCliArgs } from "./cli-args.mjs";
|
|
3
4
|
import { CLI_SUBCOMMANDS } from "./command-registry.mjs";
|
|
4
5
|
import { resolveCliEnvironment } from "./environment.mjs";
|
|
5
6
|
import { applyProfileFromCli } from "./commands/profile-selection.mjs";
|
|
@@ -78,6 +79,8 @@ async function runCli(opts = {}) {
|
|
|
78
79
|
if (opts.loadEnv !== false) loadEnvFromCwd();
|
|
79
80
|
const rawArgs = prepareCliArgs(input);
|
|
80
81
|
runtime.rawArgs = [...rawArgs];
|
|
82
|
+
const argumentError = await checkCliArgs(main, rawArgs);
|
|
83
|
+
if (argumentError) throw new Error(argumentError);
|
|
81
84
|
await withConfiguredOutput(() => runMain(main, { rawArgs }));
|
|
82
85
|
});
|
|
83
86
|
}
|
package/dist/commands/query.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import { queryCommandMeta } from "../command-meta.mjs";
|
|
|
2
2
|
import { loadConfig } from "../config.mjs";
|
|
3
3
|
import { terminalOutputOptions } from "../render/terminal.mjs";
|
|
4
4
|
import { ALL_SEARCH_TYPES, logger, parseSearchType, toCSV } from "../utils.mjs";
|
|
5
|
-
import { allTables, inferTable } from "../local-store.mjs";
|
|
5
|
+
import { allTables, inferTable, tableDimensions } from "../local-store.mjs";
|
|
6
6
|
import { createCommandContext } from "../context.mjs";
|
|
7
7
|
import { gscErrorHandler } from "../error-handler.mjs";
|
|
8
8
|
import { renderTable } from "../render/layout.mjs";
|
|
@@ -12,6 +12,7 @@ import process from "node:process";
|
|
|
12
12
|
import { defineCommand } from "citty";
|
|
13
13
|
import fs from "node:fs/promises";
|
|
14
14
|
import { and, between, contains, country, date, device, eq, gsc, hour, notRegex, page, query, regex, searchAppearance } from "gscdump/query";
|
|
15
|
+
import { decodeSiteId } from "gscdump/tenant";
|
|
15
16
|
import { cancel, isCancel, multiselect, text } from "@clack/prompts";
|
|
16
17
|
import { daysAgoUtc } from "gscdump/dates";
|
|
17
18
|
import { collectSpans } from "@gscdump/engine/profile";
|
|
@@ -279,7 +280,9 @@ const queryCommand = defineCommand({
|
|
|
279
280
|
needsStore: !args.live,
|
|
280
281
|
interactive: Boolean(args.interactive)
|
|
281
282
|
});
|
|
282
|
-
const
|
|
283
|
+
const hint = args.site ? String(args.site) : ctx.config.defaultSite;
|
|
284
|
+
const exactHint = !args.live && hint && /^(?:sc-domain:\S+|https?:\/\/\S+)$/.test(hint) ? hint : void 0;
|
|
285
|
+
const siteUrl = exactHint ? await resolveLocalSite(ctx.store, exactHint) ?? await ctx.resolveSite(hint) : await ctx.resolveSite(hint);
|
|
283
286
|
if (args.live) {
|
|
284
287
|
if (args.explain) {
|
|
285
288
|
const body = {
|
|
@@ -339,7 +342,7 @@ const queryCommand = defineCommand({
|
|
|
339
342
|
}, null, 2));
|
|
340
343
|
return;
|
|
341
344
|
}
|
|
342
|
-
await assertRangeCovered(store, siteUrl, table, startDate, endDate, searchType);
|
|
345
|
+
await assertRangeCovered(store, siteUrl, table, startDate, endDate, format === "json", searchType);
|
|
343
346
|
const probe = Boolean(args.profile) ? collectSpans() : void 0;
|
|
344
347
|
const result = await store.engine.query({
|
|
345
348
|
userId: store.userId,
|
|
@@ -477,6 +480,14 @@ async function promptFilters(args) {
|
|
|
477
480
|
if (ds && String(ds).length > 0) args["data-state"] = String(ds);
|
|
478
481
|
}
|
|
479
482
|
}
|
|
483
|
+
async function resolveLocalSite(store, hint) {
|
|
484
|
+
const siteIds = new Set((await store.engine.getWatermarks({ userId: store.userId })).map((w) => w.siteId).filter((siteId) => siteId !== void 0));
|
|
485
|
+
const exact = store.siteIdFor(hint);
|
|
486
|
+
if (siteIds.has(exact)) return hint;
|
|
487
|
+
const lowered = exact.toLowerCase();
|
|
488
|
+
const matches = [...siteIds].filter((siteId) => siteId.toLowerCase() === lowered);
|
|
489
|
+
return matches.length === 1 ? decodeSiteId(matches[0]) : void 0;
|
|
490
|
+
}
|
|
480
491
|
function buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter) {
|
|
481
492
|
const dims = dimNames.map((d) => DIM_COLUMNS[d]).filter((c) => Boolean(c));
|
|
482
493
|
const dateFilter = between(date, startDate, endDate);
|
|
@@ -494,25 +505,70 @@ function buildDimensionFilter(args) {
|
|
|
494
505
|
if (leaves.length === 1) return leaves[0];
|
|
495
506
|
return and(...leaves);
|
|
496
507
|
}
|
|
497
|
-
async function assertRangeCovered(store, siteUrl, table, startDate, endDate, searchType) {
|
|
498
|
-
const
|
|
508
|
+
async function assertRangeCovered(store, siteUrl, table, startDate, endDate, json, searchType) {
|
|
509
|
+
const watermarks = await store.engine.getWatermarks({
|
|
499
510
|
userId: store.userId,
|
|
500
511
|
siteId: store.siteIdFor(siteUrl),
|
|
501
512
|
table,
|
|
502
513
|
...searchType !== void 0 ? { searchType } : {}
|
|
503
|
-
})
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
514
|
+
});
|
|
515
|
+
const wm = watermarks[0];
|
|
516
|
+
const states = await store.engine.getSyncStates({
|
|
517
|
+
userId: store.userId,
|
|
518
|
+
siteId: store.siteIdFor(siteUrl),
|
|
519
|
+
table,
|
|
520
|
+
searchType: searchType ?? "web",
|
|
521
|
+
state: "done"
|
|
522
|
+
});
|
|
523
|
+
const completed = new Set(states.map((state) => state.date));
|
|
524
|
+
const missingDates = [];
|
|
525
|
+
for (let date = Date.parse(startDate); date <= Date.parse(endDate); date += 864e5) {
|
|
526
|
+
const day = new Date(date).toISOString().slice(0, 10);
|
|
527
|
+
if (!completed.has(day)) missingDates.push(day);
|
|
511
528
|
}
|
|
512
|
-
if (
|
|
513
|
-
|
|
514
|
-
|
|
529
|
+
if (missingDates.length === 0) return;
|
|
530
|
+
const nextArgs = [
|
|
531
|
+
"sync",
|
|
532
|
+
"--site",
|
|
533
|
+
siteUrl,
|
|
534
|
+
"--start",
|
|
535
|
+
startDate,
|
|
536
|
+
"--end",
|
|
537
|
+
endDate,
|
|
538
|
+
"--tables",
|
|
539
|
+
table,
|
|
540
|
+
"--types",
|
|
541
|
+
searchType ?? "web",
|
|
542
|
+
"--json"
|
|
543
|
+
];
|
|
544
|
+
const nextCommand = `gscdump ${nextArgs.map((value) => /^[\w:./=-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`).join(" ")}`;
|
|
545
|
+
const message = !wm ? `No data synced for ${siteUrl} / ${table}.` : `Store coverage is incomplete for ${missingDates.length} requested dates.`;
|
|
546
|
+
if (json) {
|
|
547
|
+
const available = await store.engine.getWatermarks({
|
|
548
|
+
userId: store.userId,
|
|
549
|
+
siteId: store.siteIdFor(siteUrl)
|
|
550
|
+
});
|
|
551
|
+
console.log(JSON.stringify({ error: {
|
|
552
|
+
code: "STORE_RANGE_NOT_COVERED",
|
|
553
|
+
message,
|
|
554
|
+
siteUrl,
|
|
555
|
+
table,
|
|
556
|
+
range: {
|
|
557
|
+
start: startDate,
|
|
558
|
+
end: endDate
|
|
559
|
+
},
|
|
560
|
+
missingDates,
|
|
561
|
+
watermarks: watermarks.filter((w) => w.table === table),
|
|
562
|
+
availableTables: available.map((w) => ({
|
|
563
|
+
...w,
|
|
564
|
+
dimensions: tableDimensions(w.table)
|
|
565
|
+
})),
|
|
566
|
+
nextArgs,
|
|
567
|
+
nextCommand
|
|
568
|
+
} }, null, 2));
|
|
515
569
|
}
|
|
570
|
+
logger.error(`${message} Run ${nextCommand}, or pass --live.`);
|
|
571
|
+
process.exit(1);
|
|
516
572
|
}
|
|
517
573
|
async function runRawSqlMode(opts) {
|
|
518
574
|
if (!isKnownTable(opts.table)) {
|
package/dist/commands/stats.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { terminalOutputOptions } from "../render/terminal.mjs";
|
|
2
2
|
import { OUTPUT_ARGS, applyOutputMode, displayPath, formatAge, logger } from "../utils.mjs";
|
|
3
|
-
import { allTables } from "../local-store.mjs";
|
|
3
|
+
import { allTables, tableDimensions } from "../local-store.mjs";
|
|
4
4
|
import { createCommandContext } from "../context.mjs";
|
|
5
5
|
import { renderTable, textLines } from "../render/layout.mjs";
|
|
6
6
|
import { formatMetric } from "../render/metrics.mjs";
|
|
@@ -26,15 +26,11 @@ const statsCommand = defineCommand({
|
|
|
26
26
|
const { json } = applyOutputMode(args);
|
|
27
27
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
28
28
|
const allEntries = await store.engine.listAll({ userId: store.userId });
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
logger.error(`No local data for --site=${args.site}. Known site IDs: ${known.size === 0 ? "(none — run `gscdump sync` first)" : Array.from(known).join(", ")}`);
|
|
35
|
-
process.exit(1);
|
|
36
|
-
}
|
|
37
|
-
siteId = candidate;
|
|
29
|
+
const knownSites = [...new Set(allEntries.filter((entry) => entry.retiredAt === void 0 && entry.siteId !== void 0).map((entry) => entry.siteId))];
|
|
30
|
+
const siteId = args.site ? store.siteIdFor(args.site) : void 0;
|
|
31
|
+
if (args.site && !json && !knownSites.includes(siteId)) {
|
|
32
|
+
logger.error(`No local data for --site=${args.site}. Known site IDs: ${knownSites.length === 0 ? "(none — run `gscdump sync` first)" : knownSites.join(", ")}`);
|
|
33
|
+
process.exit(1);
|
|
38
34
|
}
|
|
39
35
|
const selectedEntries = siteId === void 0 ? allEntries : allEntries.filter((entry) => entry.siteId === siteId);
|
|
40
36
|
const perTable = allTables().map((table) => {
|
|
@@ -48,16 +44,17 @@ const statsCommand = defineCommand({
|
|
|
48
44
|
const [watermarks, disk] = await Promise.all([store.engine.getWatermarks({
|
|
49
45
|
userId: store.userId,
|
|
50
46
|
siteId
|
|
51
|
-
}), filesystemStats(store.dataDir)
|
|
52
|
-
files: 0,
|
|
53
|
-
bytes: 0
|
|
54
|
-
}))]);
|
|
47
|
+
}), filesystemStats(store.dataDir)]);
|
|
55
48
|
if (json) {
|
|
56
49
|
const payload = {
|
|
57
50
|
dataDir: store.dataDir,
|
|
51
|
+
siteId: siteId ?? null,
|
|
52
|
+
knownSites,
|
|
53
|
+
nextCommand: `gscdump sync${args.site ? ` --site '${String(args.site).replaceAll("'", "'\\''")}'` : ""} --status --json`,
|
|
58
54
|
disk,
|
|
59
55
|
tables: perTable.map(({ table, live, retired }) => ({
|
|
60
56
|
table,
|
|
57
|
+
dimensions: tableDimensions(table),
|
|
61
58
|
liveFiles: live.length,
|
|
62
59
|
liveRows: sumRows(live),
|
|
63
60
|
liveBytes: sumBytes(live),
|
|
@@ -65,6 +62,7 @@ const statsCommand = defineCommand({
|
|
|
65
62
|
retiredBytes: sumBytes(retired),
|
|
66
63
|
watermarks: watermarks.filter((w) => w.table === table).map((w) => ({
|
|
67
64
|
siteId: w.siteId ?? null,
|
|
65
|
+
searchType: w.searchType ?? "web",
|
|
68
66
|
newestDateSynced: w.newestDateSynced,
|
|
69
67
|
oldestDateSynced: w.oldestDateSynced,
|
|
70
68
|
lastSyncAt: w.lastSyncAt
|
package/dist/commands/store.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import { resetCommand, rmSiteCommand } from "./store-purge.mjs";
|
|
|
9
9
|
import { defineCommand } from "citty";
|
|
10
10
|
const storeCommand = defineCommand({
|
|
11
11
|
meta: storeCommandMeta,
|
|
12
|
+
args: statsCommand.args,
|
|
12
13
|
subCommands: {
|
|
13
14
|
"stats": statsCommand,
|
|
14
15
|
"compact": compactCommand,
|
package/dist/commands/sync.mjs
CHANGED
|
@@ -276,10 +276,6 @@ const syncCommand = defineCommand({
|
|
|
276
276
|
}
|
|
277
277
|
types.push(t);
|
|
278
278
|
}
|
|
279
|
-
if (types.length === 0) {
|
|
280
|
-
logger.warn(`All requested types (${requestedTypes.join(", ")}) are marked empty for this site. Pass --force-types to re-probe.`);
|
|
281
|
-
return;
|
|
282
|
-
}
|
|
283
279
|
if (skippedTypes.length > 0 && !quiet) logger.info(`Skipping ${skippedTypes.join(", ")} (marked empty for this site; pass --force-types to re-probe).`);
|
|
284
280
|
const endDate = args.end ? String(args.end) : daysAgoUtc(DEFAULT_PENDING_DAYS);
|
|
285
281
|
let startDate;
|
|
@@ -292,6 +288,32 @@ const syncCommand = defineCommand({
|
|
|
292
288
|
logger.error(`No dates to sync (start=${startDate}, end=${endDate})`);
|
|
293
289
|
process.exit(1);
|
|
294
290
|
}
|
|
291
|
+
const printCompletion = async (status, totals, reason, rollupError) => {
|
|
292
|
+
if (!json) return;
|
|
293
|
+
console.log(JSON.stringify({
|
|
294
|
+
status,
|
|
295
|
+
...reason ? { reason } : {},
|
|
296
|
+
siteUrl,
|
|
297
|
+
range: {
|
|
298
|
+
start: startDate,
|
|
299
|
+
end: endDate
|
|
300
|
+
},
|
|
301
|
+
tables,
|
|
302
|
+
types,
|
|
303
|
+
skippedTypes,
|
|
304
|
+
totals,
|
|
305
|
+
watermarks: await store.engine.getWatermarks({
|
|
306
|
+
userId: store.userId,
|
|
307
|
+
siteId
|
|
308
|
+
}),
|
|
309
|
+
...rollupError ? { rollupError } : {}
|
|
310
|
+
}, null, 2));
|
|
311
|
+
};
|
|
312
|
+
if (types.length === 0) {
|
|
313
|
+
if (!quiet) logger.warn(`All requested types are marked empty. Pass --force-types to check them again.`);
|
|
314
|
+
await printCompletion("skipped", {}, "empty-types");
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
295
317
|
if (args["retry-failed"]) {
|
|
296
318
|
const failedSet = /* @__PURE__ */ new Set();
|
|
297
319
|
const selectedTables = new Set(tables);
|
|
@@ -303,7 +325,8 @@ const syncCommand = defineCommand({
|
|
|
303
325
|
for (const s of states) if (selectedTables.has(s.table) && selectedTypes.has(s.searchType ?? "web") && s.state === "failed" && s.date >= startDate && s.date <= endDate) failedSet.add(s.date);
|
|
304
326
|
dates = dates.filter((d) => failedSet.has(d));
|
|
305
327
|
if (dates.length === 0) {
|
|
306
|
-
logger.success("No failed dates in range
|
|
328
|
+
if (!quiet) logger.success("No failed dates in range. Nothing to retry.");
|
|
329
|
+
await printCompletion("skipped", {}, "no-failed-dates");
|
|
307
330
|
return;
|
|
308
331
|
}
|
|
309
332
|
args.force = true;
|
|
@@ -416,6 +439,7 @@ const syncCommand = defineCommand({
|
|
|
416
439
|
}
|
|
417
440
|
}
|
|
418
441
|
const noRollups = Boolean(args["no-rollups"]);
|
|
442
|
+
let rollupError;
|
|
419
443
|
const anyRowsSynced = Object.values(totals).some((t) => t.rows > 0);
|
|
420
444
|
if (!noRollups && anyRowsSynced) {
|
|
421
445
|
if (!quiet) logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)…`);
|
|
@@ -442,6 +466,7 @@ const syncCommand = defineCommand({
|
|
|
442
466
|
},
|
|
443
467
|
defs: DEFAULT_ROLLUPS
|
|
444
468
|
}).catch((err) => {
|
|
469
|
+
rollupError = err.message;
|
|
445
470
|
logger.warn(`Rollup rebuild failed: ${err.message}`);
|
|
446
471
|
return [];
|
|
447
472
|
});
|
|
@@ -451,7 +476,8 @@ const syncCommand = defineCommand({
|
|
|
451
476
|
logger.success(`Rebuilt ${results.length} rollup(s) in ${ms}ms — ${kb.toFixed(1)} KB`);
|
|
452
477
|
}
|
|
453
478
|
}
|
|
454
|
-
|
|
479
|
+
await printCompletion(anyFailed || rollupError ? "failed" : "completed", totals, void 0, rollupError);
|
|
480
|
+
if (anyFailed || rollupError) process.exit(1);
|
|
455
481
|
}
|
|
456
482
|
});
|
|
457
483
|
function isKnownTable(name) {
|
package/dist/local-store.mjs
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
import { createNodeHarness } from "@gscdump/engine/node";
|
|
2
|
+
import { SCHEMAS, allTables, inferTable } from "@gscdump/engine/schema";
|
|
2
3
|
import { TABLE_DIMS, assembleDatesRow } from "@gscdump/engine/ingest";
|
|
3
|
-
|
|
4
|
+
function tableDimensions(table) {
|
|
5
|
+
return SCHEMAS[table].columns.map((column) => column.name === "url" ? "page" : column.name === "search_appearance" ? "searchAppearance" : column.name).filter((name) => [
|
|
6
|
+
"page",
|
|
7
|
+
"query",
|
|
8
|
+
"date",
|
|
9
|
+
"hour",
|
|
10
|
+
"country",
|
|
11
|
+
"device",
|
|
12
|
+
"searchAppearance"
|
|
13
|
+
].includes(name));
|
|
14
|
+
}
|
|
4
15
|
function createLocalStore(opts) {
|
|
5
16
|
return createNodeHarness(opts);
|
|
6
17
|
}
|
|
7
|
-
export { TABLE_DIMS, allTables, assembleDatesRow, createLocalStore, inferTable };
|
|
18
|
+
export { TABLE_DIMS, allTables, assembleDatesRow, createLocalStore, inferTable, tableDimensions };
|
package/dist/package.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var version = "3.6.
|
|
1
|
+
var version = "3.6.3";
|
|
2
2
|
export { version };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/cli",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "3.6.
|
|
4
|
+
"version": "3.6.3",
|
|
5
5
|
"description": "CLI for Google Search Console and Bing with hosted or local authentication, data exports, and an MCP server",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -44,15 +44,15 @@
|
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@clack/prompts": "^1.8.0",
|
|
46
46
|
"@duckdb/node-api": "1.5.5-r.4",
|
|
47
|
-
"@gscdump/analysis": "^3.6.
|
|
48
|
-
"@gscdump/engine": "^3.6.
|
|
49
|
-
"@gscdump/engine-gsc-api": "^3.6.
|
|
50
|
-
"@gscdump/sdk": "^3.6.
|
|
47
|
+
"@gscdump/analysis": "^3.6.3",
|
|
48
|
+
"@gscdump/engine": "^3.6.3",
|
|
49
|
+
"@gscdump/engine-gsc-api": "^3.6.3",
|
|
50
|
+
"@gscdump/sdk": "^3.6.3",
|
|
51
51
|
"@modelcontextprotocol/sdk": "^1.30.0",
|
|
52
52
|
"citty": "^0.2.2",
|
|
53
53
|
"consola": "^3.4.2",
|
|
54
54
|
"google-auth-library": "^11.0.2",
|
|
55
|
-
"gscdump": "^3.6.
|
|
55
|
+
"gscdump": "^3.6.3",
|
|
56
56
|
"ofetch": "^1.5.1",
|
|
57
57
|
"open": "^11.0.2",
|
|
58
58
|
"sitemapd": "^0.2.2",
|
package/skills/gscdump/SKILL.md
CHANGED
|
@@ -7,6 +7,24 @@ description: Drive the `gscdump` CLI for Google Search Console and Bing with clo
|
|
|
7
7
|
|
|
8
8
|
`gscdump` reads Google Search Console and Bing with hosted or local authentication.
|
|
9
9
|
It keeps a local Parquet Store for Google rows. Every command has `--help`.
|
|
10
|
+
For `query`, `-s` means `--site`, `-d` means `--dimensions`, and `-f` means `--format`.
|
|
11
|
+
Use `--start` and `--end` for dates. `--site=SITE` also works.
|
|
12
|
+
Use each option once, with either its short or long spelling.
|
|
13
|
+
Example: `gscdump query --site=SITE --start=DATE --end=DATE -d page -f json`.
|
|
14
|
+
|
|
15
|
+
## Start each task
|
|
16
|
+
|
|
17
|
+
1. Before reading traffic, run `gscdump auth status --json`. Do this even when the user says authentication works.
|
|
18
|
+
2. Keep the requested Site, dates, dimensions, and task scope. A request for pages does not need query dimensions.
|
|
19
|
+
3. Before local queries, check coverage with `gscdump store stats --site SITE --json`.
|
|
20
|
+
Use `gscdump sync --site SITE --status --json` when you need sync-state details.
|
|
21
|
+
4. Read the table dimensions and watermarks. Sync only missing tables and the requested dates, once per task.
|
|
22
|
+
5. Use `sync --json`. Read its completion result before deciding what to do next. Never repeat a successful sync.
|
|
23
|
+
|
|
24
|
+
If the task only asks about deletion, explain the scope and ask for consent.
|
|
25
|
+
You may read Store metadata with `store stats` and `sync --status`.
|
|
26
|
+
Do not query traffic, sync rows, or delete data to explain deletion.
|
|
27
|
+
Call the local data directory the Store in your answer.
|
|
10
28
|
|
|
11
29
|
## Authentication mode
|
|
12
30
|
|
|
@@ -130,11 +148,14 @@ gscdump config set defaultSite sc-domain:example.com
|
|
|
130
148
|
|
|
131
149
|
## Output
|
|
132
150
|
|
|
133
|
-
Pass `--json` on every command that supports it. `
|
|
151
|
+
Pass `--json` on every command that supports it. `sync --json` includes completion, row counts, skipped dates, and failures.
|
|
152
|
+
`query` uses
|
|
134
153
|
`--format json` and prints rows to stdout. Progress goes to stderr, so stdout
|
|
135
154
|
stays parseable. `--quiet` drops progress lines.
|
|
136
155
|
|
|
137
156
|
Parse JSON. Never scrape human output.
|
|
157
|
+
If the user requests JSON, return the CLI JSON unchanged. Do not replace it with a table.
|
|
158
|
+
Do not rewrite rows, estimate metrics, or add manually calculated totals.
|
|
138
159
|
|
|
139
160
|
## Commands
|
|
140
161
|
|
|
@@ -172,9 +193,10 @@ The MCP server does not expose Bing tools. Use `gscdump bing` commands through t
|
|
|
172
193
|
## Sync before local analysis
|
|
173
194
|
|
|
174
195
|
```sh
|
|
196
|
+
gscdump store stats --site sc-domain:example.com --json
|
|
175
197
|
gscdump sync --site sc-domain:example.com --days 90 \
|
|
176
|
-
--tables pages,queries,page_queries,countries
|
|
177
|
-
gscdump sync --site sc-domain:example.com --status
|
|
198
|
+
--tables pages,queries,page_queries,countries --json
|
|
199
|
+
gscdump sync --site sc-domain:example.com --status --json
|
|
178
200
|
```
|
|
179
201
|
|
|
180
202
|
- Pass an explicit `--tables` list. The default list has a known daily-totals
|
|
@@ -183,6 +205,8 @@ gscdump sync --site sc-domain:example.com --status
|
|
|
183
205
|
- Sync skips completed dates. `--force` refreshes them. `--retry-failed`
|
|
184
206
|
reruns only failed dates.
|
|
185
207
|
- `--dry-run` prints the planned work without calling Google.
|
|
208
|
+
- Use the user's date range. The 90-day example does not authorize a wider sync.
|
|
209
|
+
- Empty Store metadata is expected before the first sync. It does not prove zero traffic.
|
|
186
210
|
|
|
187
211
|
## Query rows
|
|
188
212
|
|
|
@@ -195,8 +219,11 @@ gscdump query --site sc-domain:example.com --dimensions page,query \
|
|
|
195
219
|
- Filters: `--query`, `--page`, `--country`, `--device`,
|
|
196
220
|
`--search-appearance`. Prefixes: bare equals, `~` contains, `!~` not
|
|
197
221
|
contains, `re:` regex, `!re:` not regex, `!` not equals.
|
|
198
|
-
- `--live` bypasses the Store. `--
|
|
199
|
-
`--aggregation-type` apply to live mode only.
|
|
222
|
+
- `--live` bypasses the Store. `--type` selects a search type.
|
|
223
|
+
`--data-state` and `--aggregation-type` apply to live mode only.
|
|
224
|
+
- Metrics already include clicks, impressions, CTR, and position. There is no `--metrics` option.
|
|
225
|
+
- If Store coverage is missing, read the JSON error and its bounded `nextArgs` before syncing.
|
|
226
|
+
Do not switch dimensions to make a failed query succeed.
|
|
200
227
|
- `--explain` prints the request body or planned SQL without executing.
|
|
201
228
|
- `--sql` runs raw DuckDB SQL over the Store with `{{FILES}}` as the file list.
|
|
202
229
|
|
|
@@ -270,9 +297,9 @@ the failure and continue. Never retry an uncertain submission.
|
|
|
270
297
|
delete local data. `--yes` is consent you borrow from the user.
|
|
271
298
|
- **Never loop unattended.** One `sync` per Site per task. Inspection batches
|
|
272
299
|
spend a daily pool. Use `--dry-run` and `--explain` to plan first.
|
|
273
|
-
- **Report the result as the CLI gave it.**
|
|
274
|
-
|
|
275
|
-
rows
|
|
300
|
+
- **Report the result as the CLI gave it.** Zero clicks with impressions means rows exist with no clicks.
|
|
301
|
+
An empty row array means no rows matched. Missing coverage means the Store cannot answer that date range.
|
|
302
|
+
Check `sync --status --json` before interpreting empty rows. None of these results proves a clean Site.
|
|
276
303
|
- **Do not widen the Site.** A `sc-domain:` property includes every
|
|
277
304
|
subdomain. Filter with `--page` when the user means one host.
|
|
278
305
|
|
|
@@ -285,3 +312,14 @@ Hosted exports follow pagination and reject missing, unavailable, or changing da
|
|
|
285
312
|
Hosted date ranges span at most 366 days. The default range is the last 366 days.
|
|
286
313
|
Local date filters only narrow data currently returned by Bing.
|
|
287
314
|
Do not treat Bing crawl evidence as proof that a URL is indexed.
|
|
315
|
+
|
|
316
|
+
## Before the next command
|
|
317
|
+
|
|
318
|
+
For a traffic task, run `gscdump auth status --json` now, before any `query` command.
|
|
319
|
+
The user saying credentials work does not replace this check. It identifies the selected authentication mode.
|
|
320
|
+
Then check Store metadata, keep the requested dimensions, and read or sync only the requested dates.
|
|
321
|
+
If the user requests JSON, return the command's JSON unchanged, without a table or a separate totals summary.
|
|
322
|
+
Include that JSON in your final response. Tool output alone is not a final answer.
|
|
323
|
+
Include every returned row. Do not refer the user to results "above".
|
|
324
|
+
|
|
325
|
+
For a deletion explanation, read metadata only if needed. Explain the scope and ask for consent, then stop.
|