@gscdump/cli 3.7.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -30
- package/bin/gscdump.mjs +21 -1
- package/dist/analysis-local.mjs +172 -53
- package/dist/auth-state.mjs +21 -6
- package/dist/auth.mjs +27 -18
- package/dist/bing-data.mjs +6 -2
- package/dist/bing-hosted.mjs +1 -1
- package/dist/cli-args.mjs +8 -0
- package/dist/cli.d.mts +5 -1
- package/dist/cli.mjs +41 -10
- package/dist/cloud-google.mjs +8 -1
- package/dist/command-meta.mjs +4 -4
- package/dist/command-registry.mjs +86 -10
- package/dist/commands/analyze.mjs +84 -43
- package/dist/commands/auth.mjs +46 -37
- package/dist/commands/bing.mjs +3 -3
- package/dist/commands/compact.mjs +12 -8
- package/dist/commands/config.mjs +14 -14
- package/dist/commands/doctor.mjs +26 -26
- package/dist/commands/dump.mjs +203 -105
- package/dist/commands/entities.mjs +10 -113
- package/dist/commands/gc.mjs +4 -3
- package/dist/commands/indexing-urls.mjs +185 -0
- package/dist/commands/indexing.mjs +12 -8
- package/dist/commands/init.mjs +77 -18
- package/dist/commands/inspect.mjs +97 -110
- package/dist/commands/mcp.mjs +37 -57
- package/dist/commands/papercut.mjs +1 -1
- package/dist/commands/profile.mjs +9 -17
- package/dist/commands/query.mjs +219 -210
- package/dist/commands/report.mjs +68 -85
- package/dist/commands/rollups.mjs +9 -6
- package/dist/commands/sitemaps.mjs +51 -104
- package/dist/commands/sites.mjs +40 -24
- package/dist/commands/stats.mjs +14 -17
- package/dist/commands/store-purge.mjs +9 -5
- package/dist/commands/store.mjs +0 -12
- package/dist/commands/sync.mjs +774 -296
- package/dist/config.mjs +7 -4
- package/dist/context.mjs +51 -21
- package/dist/coverage.mjs +251 -0
- package/dist/dump-bing.mjs +71 -0
- package/dist/dump-writers.mjs +246 -0
- package/dist/error-handler.mjs +108 -38
- package/dist/filters.mjs +125 -0
- package/dist/hosted-site.mjs +91 -0
- package/dist/inspect-urls.mjs +76 -0
- package/dist/inspection-record.mjs +93 -0
- package/dist/local-entities.mjs +658 -0
- package/dist/mcp/errors.mjs +71 -18
- package/dist/mcp/handlers/diagnostics.mjs +15 -12
- package/dist/mcp/handlers/reports.mjs +33 -42
- package/dist/mcp/server/index.mjs +80 -38
- package/dist/mcp/types.mjs +7 -7
- package/dist/package.mjs +1 -1
- package/dist/quota-ledger.mjs +370 -0
- package/dist/render/analysis.mjs +8 -1
- package/dist/render/report.mjs +8 -1
- package/dist/request-pacer.mjs +29 -0
- package/dist/route.mjs +295 -0
- package/dist/sitemap.mjs +4 -0
- package/dist/sql-views.mjs +117 -0
- package/dist/store-sites.mjs +134 -0
- package/dist/sync-plan.mjs +129 -0
- package/dist/sync-run.mjs +96 -0
- package/dist/table-sources.mjs +57 -0
- package/dist/token-info.mjs +54 -0
- package/dist/utils.mjs +28 -19
- package/dist/window.mjs +159 -0
- package/package.json +17 -14
- package/skills/gscdump/SKILL.md +167 -37
- package/dist/commands/export.mjs +0 -76
- package/dist/native-duckdb.mjs +0 -46
package/dist/commands/report.mjs
CHANGED
|
@@ -1,69 +1,40 @@
|
|
|
1
1
|
import { reportCommandMeta } from "../command-meta.mjs";
|
|
2
2
|
import { terminalOutputOptions } from "../render/terminal.mjs";
|
|
3
|
-
import { logger } from "../utils.mjs";
|
|
4
|
-
import {
|
|
3
|
+
import { logger, parseFetchBudget } from "../utils.mjs";
|
|
4
|
+
import { COMPARISON_FLAGS, PERIOD_FLAGS, parseWindowFlags, windowFlagErrorToException } from "../window.mjs";
|
|
5
|
+
import { analysisNeeds, analyzerSources, analyzerTables, resolveAnalysisSource } from "../analysis-local.mjs";
|
|
5
6
|
import { renderCliReport } from "../render/report.mjs";
|
|
6
7
|
import { defineCommand } from "citty";
|
|
8
|
+
import { getLatestGscDate } from "gscdump/dates";
|
|
9
|
+
import { DEFAULT_FETCH_BUDGET, MAX_FETCH_BUDGET } from "@gscdump/engine/analysis-types";
|
|
10
|
+
import { unwrapResult } from "gscdump/result";
|
|
7
11
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
8
|
-
import { resolveWindow } from "@gscdump/engine/period";
|
|
9
|
-
import { err, ok, unwrapResult } from "gscdump/result";
|
|
10
12
|
import { defaultReportRegistry, dryRunReport, runReport } from "@gscdump/analysis/report";
|
|
11
|
-
function reportFlagErrorToException(error) {
|
|
12
|
-
const exception = new Error(error.message);
|
|
13
|
-
exception.reportFlagError = error;
|
|
14
|
-
return exception;
|
|
15
|
-
}
|
|
16
13
|
const REPORT_IDS = defaultReportRegistry.listReportIds();
|
|
17
|
-
|
|
18
|
-
"
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
"365d": "last-365d",
|
|
24
|
-
"last-7d": "last-7d",
|
|
25
|
-
"last-28d": "last-28d",
|
|
26
|
-
"last-30d": "last-30d",
|
|
27
|
-
"last-90d": "last-90d",
|
|
28
|
-
"last-180d": "last-180d",
|
|
29
|
-
"last-365d": "last-365d",
|
|
30
|
-
"mtd": "mtd",
|
|
31
|
-
"ytd": "ytd",
|
|
32
|
-
"custom": "custom"
|
|
33
|
-
};
|
|
34
|
-
const COMPARISON_ALIASES = {
|
|
35
|
-
"none": "none",
|
|
36
|
-
"prev": "prev-period",
|
|
37
|
-
"prev-period": "prev-period",
|
|
38
|
-
"prior": "prev-period",
|
|
39
|
-
"prior-period": "prev-period",
|
|
40
|
-
"yoy": "yoy"
|
|
41
|
-
};
|
|
42
|
-
function resolvePeriodResult(input, fallback) {
|
|
43
|
-
if (!input) return ok(fallback);
|
|
44
|
-
const preset = PERIOD_ALIASES[input.toLowerCase()];
|
|
45
|
-
if (!preset) return err({
|
|
46
|
-
kind: "unknown-period",
|
|
47
|
-
value: input,
|
|
48
|
-
message: `Unknown --period "${input}". Supported: 7d, 28d, 30d, 90d, 180d, 365d, mtd, ytd, custom.`
|
|
49
|
-
});
|
|
50
|
-
return ok(preset);
|
|
51
|
-
}
|
|
52
|
-
function resolvePeriod(input, fallback) {
|
|
53
|
-
return unwrapResult(resolvePeriodResult(input, fallback), reportFlagErrorToException);
|
|
14
|
+
function reportDefaults(report) {
|
|
15
|
+
if (report.defaultPeriod === "custom") throw new Error(`Report "${report.id}" declares a custom default period. Declare a preset.`);
|
|
16
|
+
return {
|
|
17
|
+
preset: report.defaultPeriod,
|
|
18
|
+
comparison: report.defaultComparison
|
|
19
|
+
};
|
|
54
20
|
}
|
|
55
|
-
function
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
21
|
+
function windowFlags(args) {
|
|
22
|
+
const optional = (key) => args[key] ? String(args[key]) : void 0;
|
|
23
|
+
return {
|
|
24
|
+
period: optional("period"),
|
|
25
|
+
vs: optional("vs"),
|
|
26
|
+
start: optional("start"),
|
|
27
|
+
end: optional("end"),
|
|
28
|
+
prevStart: optional("prev-start"),
|
|
29
|
+
prevEnd: optional("prev-end")
|
|
30
|
+
};
|
|
64
31
|
}
|
|
65
|
-
function
|
|
66
|
-
|
|
32
|
+
function reportSources(steps) {
|
|
33
|
+
const live = (type) => analyzerSources([type]).live;
|
|
34
|
+
return {
|
|
35
|
+
local: analyzerSources(steps.map((step) => step.type)).local,
|
|
36
|
+
live: steps.some((step) => live(step.type)) && steps.every((step) => !step.required || live(step.type))
|
|
37
|
+
};
|
|
67
38
|
}
|
|
68
39
|
function reportArgsToCitty(spec) {
|
|
69
40
|
const out = {};
|
|
@@ -107,29 +78,31 @@ function makeReportCommand(report) {
|
|
|
107
78
|
},
|
|
108
79
|
"period": {
|
|
109
80
|
type: "string",
|
|
110
|
-
description:
|
|
111
|
-
default: presetToFlag(report.defaultPeriod)
|
|
81
|
+
description: `Window: ${PERIOD_FLAGS.join("|")} (default: ${presetToFlag(report.defaultPeriod)}, ending on the newest synced day)`
|
|
112
82
|
},
|
|
113
83
|
"vs": {
|
|
114
84
|
type: "string",
|
|
115
|
-
description:
|
|
116
|
-
default: report.defaultComparison
|
|
85
|
+
description: `Comparison: ${COMPARISON_FLAGS.join("|")} (default: ${report.defaultComparison}). yoy compares with the same weekdays 52 weeks earlier`
|
|
117
86
|
},
|
|
118
87
|
"start": {
|
|
119
88
|
type: "string",
|
|
120
|
-
description: "Custom start date (YYYY-MM-DD)"
|
|
89
|
+
description: "Custom start date (YYYY-MM-DD). Implies --period custom"
|
|
121
90
|
},
|
|
122
91
|
"end": {
|
|
123
92
|
type: "string",
|
|
124
|
-
description: "Custom end date (YYYY-MM-DD)"
|
|
93
|
+
description: "Custom end date (YYYY-MM-DD). Implies --period custom"
|
|
125
94
|
},
|
|
126
95
|
"prev-start": {
|
|
127
96
|
type: "string",
|
|
128
|
-
description: "Override comparison start"
|
|
97
|
+
description: "Override comparison start (pass with --prev-end)"
|
|
129
98
|
},
|
|
130
99
|
"prev-end": {
|
|
131
100
|
type: "string",
|
|
132
|
-
description: "Override comparison end"
|
|
101
|
+
description: "Override comparison end (pass with --prev-start)"
|
|
102
|
+
},
|
|
103
|
+
"fetch-budget": {
|
|
104
|
+
type: "string",
|
|
105
|
+
description: `Max rows each live fetch reads (default: ${DEFAULT_FETCH_BUDGET}, max: ${MAX_FETCH_BUDGET})`
|
|
133
106
|
},
|
|
134
107
|
"live": {
|
|
135
108
|
type: "boolean",
|
|
@@ -154,20 +127,11 @@ function makeReportCommand(report) {
|
|
|
154
127
|
...reportArgs
|
|
155
128
|
},
|
|
156
129
|
async run({ args }) {
|
|
157
|
-
const
|
|
158
|
-
const comparison = resolveComparison(args.vs, report.defaultComparison);
|
|
159
|
-
const window = resolveWindow({
|
|
160
|
-
preset,
|
|
161
|
-
comparison,
|
|
162
|
-
start: args.start,
|
|
163
|
-
end: args.end
|
|
164
|
-
});
|
|
165
|
-
if (args["prev-start"] && args["prev-end"]) window.comparison = {
|
|
166
|
-
start: String(args["prev-start"]),
|
|
167
|
-
end: String(args["prev-end"])
|
|
168
|
-
};
|
|
130
|
+
const flags = windowFlags(args);
|
|
169
131
|
const params = buildReportParams(report, args);
|
|
132
|
+
const fetchBudget = parseFetchBudget(args["fetch-budget"]);
|
|
170
133
|
if (args.explain || args["dry-run"]) {
|
|
134
|
+
const window = unwrapResult(parseWindowFlags(flags, reportDefaults(report), getLatestGscDate()), windowFlagErrorToException);
|
|
171
135
|
const ctx = {
|
|
172
136
|
site: args.site ? String(args.site) : "(unresolved)",
|
|
173
137
|
window,
|
|
@@ -178,21 +142,35 @@ function makeReportCommand(report) {
|
|
|
178
142
|
console.log(JSON.stringify({
|
|
179
143
|
id: report.id,
|
|
180
144
|
window,
|
|
181
|
-
comparison,
|
|
182
145
|
plan: dry.steps
|
|
183
146
|
}, null, 2));
|
|
184
147
|
return;
|
|
185
148
|
}
|
|
186
|
-
const
|
|
149
|
+
const steps = (window) => report.plan(params, window).map((step) => ({
|
|
150
|
+
type: step.type,
|
|
151
|
+
params: {
|
|
152
|
+
...step.params,
|
|
153
|
+
type: step.type
|
|
154
|
+
},
|
|
155
|
+
required: step.required === true
|
|
156
|
+
}));
|
|
157
|
+
const provisional = steps(unwrapResult(parseWindowFlags(flags, reportDefaults(report), getLatestGscDate()), windowFlagErrorToException));
|
|
158
|
+
const { source, siteUrl, anchor, isLive } = await resolveAnalysisSource({
|
|
187
159
|
site: args.site,
|
|
188
160
|
live: !!args.live,
|
|
189
|
-
json: !!args.json
|
|
161
|
+
json: !!args.json,
|
|
162
|
+
label: `report ${report.id}`,
|
|
163
|
+
types: provisional.map((step) => step.type),
|
|
164
|
+
sources: reportSources(provisional),
|
|
165
|
+
anchorTables: [...new Set(provisional.flatMap((step) => analyzerTables(step.params)))],
|
|
166
|
+
needs: (anchor) => steps(unwrapResult(parseWindowFlags(flags, reportDefaults(report), anchor), windowFlagErrorToException)).flatMap((step) => analysisNeeds(step.params))
|
|
190
167
|
});
|
|
191
168
|
const ctx = {
|
|
192
169
|
site: siteUrl,
|
|
193
|
-
window,
|
|
170
|
+
window: unwrapResult(parseWindowFlags(flags, reportDefaults(report), anchor), windowFlagErrorToException),
|
|
194
171
|
params,
|
|
195
|
-
registryVersion: defaultReportRegistry.version
|
|
172
|
+
registryVersion: defaultReportRegistry.version,
|
|
173
|
+
...fetchBudget !== void 0 ? { fetchBudget } : {}
|
|
196
174
|
};
|
|
197
175
|
const result = await runReport(report, {
|
|
198
176
|
source,
|
|
@@ -200,7 +178,13 @@ function makeReportCommand(report) {
|
|
|
200
178
|
ctx
|
|
201
179
|
});
|
|
202
180
|
if (args.json) {
|
|
203
|
-
console.log(JSON.stringify(
|
|
181
|
+
console.log(JSON.stringify({
|
|
182
|
+
...result,
|
|
183
|
+
meta: {
|
|
184
|
+
...result.meta,
|
|
185
|
+
source: isLive ? "live" : "local"
|
|
186
|
+
}
|
|
187
|
+
}, null, 2));
|
|
204
188
|
return;
|
|
205
189
|
}
|
|
206
190
|
console.log(renderCliReport(result, terminalOutputOptions()));
|
|
@@ -209,8 +193,7 @@ function makeReportCommand(report) {
|
|
|
209
193
|
});
|
|
210
194
|
}
|
|
211
195
|
function presetToFlag(preset) {
|
|
212
|
-
|
|
213
|
-
return preset.replace(/^last-/, "");
|
|
196
|
+
return /^last-\d+d$/.test(preset) ? preset.replace(/^last-/, "") : preset;
|
|
214
197
|
}
|
|
215
198
|
const listCommand = defineCommand({
|
|
216
199
|
meta: {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { OUTPUT_ARGS, applyOutputMode, logger } from "../utils.mjs";
|
|
2
|
+
import { readSiteMap, siteUrlForId } from "../store-sites.mjs";
|
|
2
3
|
import { createCommandContext } from "../context.mjs";
|
|
3
4
|
import { defineCommand } from "citty";
|
|
4
5
|
import { buildQueryDimRecords, createQueryDimStore } from "@gscdump/engine/entities";
|
|
@@ -43,7 +44,7 @@ const rebuildSubCommand = defineCommand({
|
|
|
43
44
|
"site": {
|
|
44
45
|
type: "string",
|
|
45
46
|
alias: "s",
|
|
46
|
-
description: "Restrict to
|
|
47
|
+
description: "Restrict to one Site, for example example.com (default: every Site with local data)"
|
|
47
48
|
},
|
|
48
49
|
"with-canonical": {
|
|
49
50
|
type: "boolean",
|
|
@@ -54,8 +55,10 @@ const rebuildSubCommand = defineCommand({
|
|
|
54
55
|
async run({ args }) {
|
|
55
56
|
const { json } = applyOutputMode(args);
|
|
56
57
|
const defs = args["with-canonical"] ? [...DEFAULT_ROLLUPS, ...CANONICAL_ROLLUPS] : DEFAULT_ROLLUPS;
|
|
57
|
-
const
|
|
58
|
-
const
|
|
58
|
+
const ctx = await createCommandContext({ needsStore: true });
|
|
59
|
+
const store = ctx.store;
|
|
60
|
+
const explicitSiteId = args.site ? store.siteIdFor(await ctx.resolveSite(String(args.site), { scope: "store" })) : void 0;
|
|
61
|
+
const siteMap = await readSiteMap(store.dataDir, store.userId);
|
|
59
62
|
const allSiteIds = /* @__PURE__ */ new Set();
|
|
60
63
|
if (explicitSiteId) allSiteIds.add(explicitSiteId);
|
|
61
64
|
else {
|
|
@@ -75,9 +78,9 @@ const rebuildSubCommand = defineCommand({
|
|
|
75
78
|
for (const siteId of allSiteIds) {
|
|
76
79
|
if (args["with-canonical"]) {
|
|
77
80
|
const dimRows = await buildSiteQueryDim(store, siteId);
|
|
78
|
-
if (!json) logger.info(`Built query dimension for [${siteId}] (${dimRows} distinct queries, normalizer v${NORMALIZER_VERSION})`);
|
|
81
|
+
if (!json) logger.info(`Built query dimension for [${siteUrlForId(siteMap, siteId)}] (${dimRows} distinct queries, normalizer v${NORMALIZER_VERSION})`);
|
|
79
82
|
}
|
|
80
|
-
logger.info(`Rebuilding rollups for [${siteId}] (${defs.length} rollups)`);
|
|
83
|
+
logger.info(`Rebuilding rollups for [${siteUrlForId(siteMap, siteId)}] (${defs.length} rollups)`);
|
|
81
84
|
const results = await rebuildRollups({
|
|
82
85
|
engine: {
|
|
83
86
|
runSQL: (opts) => store.engine.runSQL(opts),
|
|
@@ -101,7 +104,7 @@ const rebuildSubCommand = defineCommand({
|
|
|
101
104
|
defs
|
|
102
105
|
});
|
|
103
106
|
const site = {
|
|
104
|
-
siteId,
|
|
107
|
+
siteUrl: siteUrlForId(siteMap, siteId),
|
|
105
108
|
rollups: []
|
|
106
109
|
};
|
|
107
110
|
for (const r of results) {
|
|
@@ -1,36 +1,16 @@
|
|
|
1
|
-
import { parseAuthentication, resolveAuthentication } from "../auth-state.mjs";
|
|
2
1
|
import { sitemapsCommandMeta } from "../command-meta.mjs";
|
|
3
|
-
import {
|
|
4
|
-
import { OUTPUT_ARGS, applyOutputMode, logger, noSubcommandSelected, parseIntegerOption } from "../utils.mjs";
|
|
2
|
+
import { OUTPUT_ARGS, applyOutputMode, logger, parseIntegerOption } from "../utils.mjs";
|
|
5
3
|
import { createCommandContext } from "../context.mjs";
|
|
6
|
-
import { gscErrorHandler } from "../error-handler.mjs";
|
|
7
4
|
import { discoverLiveSitemap, loadSitemapUrls } from "../sitemap.mjs";
|
|
5
|
+
import { HOSTED_ARGS, resolveHostedSite } from "../hosted-site.mjs";
|
|
8
6
|
import process from "node:process";
|
|
9
7
|
import { defineCommand } from "citty";
|
|
10
|
-
import { createGscdumpV1Client } from "@gscdump/sdk/v1";
|
|
11
8
|
import { fetchSitemap } from "gscdump/sites";
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
"api-key": {
|
|
18
|
-
type: "string",
|
|
19
|
-
description: "Hosted API key; defaults to GSCDUMP_API_KEY"
|
|
20
|
-
}
|
|
21
|
-
};
|
|
22
|
-
async function hostedClient(args) {
|
|
23
|
-
const environment = resolveCliEnvironment().values;
|
|
24
|
-
const authentication = args["api-key"] ? parseAuthentication({
|
|
25
|
-
_tag: "Cloud",
|
|
26
|
-
apiKey: args["api-key"],
|
|
27
|
-
apiRoot: String(args["api-root"] || environment.GSCDUMP_API_ROOT || "https://gscdump.com/api")
|
|
28
|
-
}) : await resolveAuthentication();
|
|
29
|
-
if (authentication._tag !== "Cloud") throw new Error("Hosted sitemap reads require cloud authentication. Run `gscdump auth login --mode cloud` or supply --api-key.");
|
|
30
|
-
if (args["api-root"] && String(args["api-root"]).replace(/\/+$/, "") !== authentication.apiRoot) throw new Error("The API root changed. Supply --api-key explicitly for the new API root.");
|
|
31
|
-
return createGscdumpV1Client({
|
|
32
|
-
apiRoot: authentication.apiRoot,
|
|
33
|
-
credential: authentication.apiKey
|
|
9
|
+
const HOSTED_SITEMAP_ALTERNATIVE = "read a live sitemap with `gscdump sitemaps urls <sitemap-url>`";
|
|
10
|
+
function hostedSitemapSite(args, name) {
|
|
11
|
+
return resolveHostedSite(args, {
|
|
12
|
+
name: `sitemaps ${name}`,
|
|
13
|
+
localAlternative: HOSTED_SITEMAP_ALTERNATIVE
|
|
34
14
|
});
|
|
35
15
|
}
|
|
36
16
|
const listCommand = defineCommand({
|
|
@@ -43,7 +23,7 @@ const listCommand = defineCommand({
|
|
|
43
23
|
site: {
|
|
44
24
|
type: "string",
|
|
45
25
|
alias: "s",
|
|
46
|
-
description: "Site
|
|
26
|
+
description: "Site, for example example.com"
|
|
47
27
|
},
|
|
48
28
|
pending: {
|
|
49
29
|
type: "boolean",
|
|
@@ -60,7 +40,7 @@ const listCommand = defineCommand({
|
|
|
60
40
|
const { json } = applyOutputMode(args);
|
|
61
41
|
const ctx = await createCommandContext({ needsAuth: true });
|
|
62
42
|
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
63
|
-
let sitemaps = (await ctx.client.sitemaps.list(siteUrl)
|
|
43
|
+
let sitemaps = (await ctx.client.sitemaps.list(siteUrl)).map((sm) => ({
|
|
64
44
|
path: sm.path,
|
|
65
45
|
type: sm.type || void 0,
|
|
66
46
|
isPending: sm.isPending || false,
|
|
@@ -100,7 +80,7 @@ const getCommand = defineCommand({
|
|
|
100
80
|
site: {
|
|
101
81
|
type: "string",
|
|
102
82
|
alias: "s",
|
|
103
|
-
description: "Site
|
|
83
|
+
description: "Site, for example example.com; defaults to config.defaultSite or a prompt"
|
|
104
84
|
},
|
|
105
85
|
url: {
|
|
106
86
|
type: "positional",
|
|
@@ -113,7 +93,7 @@ const getCommand = defineCommand({
|
|
|
113
93
|
const ctx = await createCommandContext({ needsAuth: true });
|
|
114
94
|
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
115
95
|
const client = ctx.client;
|
|
116
|
-
const sitemap = await fetchSitemap(client, siteUrl, args.url)
|
|
96
|
+
const sitemap = await fetchSitemap(client, siteUrl, args.url);
|
|
117
97
|
if (json) {
|
|
118
98
|
console.log(JSON.stringify(sitemap, null, 2));
|
|
119
99
|
return;
|
|
@@ -143,7 +123,7 @@ const submitCommand = defineCommand({
|
|
|
143
123
|
site: {
|
|
144
124
|
type: "string",
|
|
145
125
|
alias: "s",
|
|
146
|
-
description: "Site
|
|
126
|
+
description: "Site, for example example.com; defaults to config.defaultSite or a prompt"
|
|
147
127
|
},
|
|
148
128
|
url: {
|
|
149
129
|
type: "positional",
|
|
@@ -155,7 +135,7 @@ const submitCommand = defineCommand({
|
|
|
155
135
|
const { json } = applyOutputMode(args);
|
|
156
136
|
const ctx = await createCommandContext({ needsAuth: true });
|
|
157
137
|
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
158
|
-
await ctx.client.sitemaps.submit(siteUrl, args.url)
|
|
138
|
+
await ctx.client.sitemaps.submit(siteUrl, args.url);
|
|
159
139
|
if (json) {
|
|
160
140
|
console.log(JSON.stringify({
|
|
161
141
|
siteUrl,
|
|
@@ -177,7 +157,7 @@ const deleteCommand = defineCommand({
|
|
|
177
157
|
site: {
|
|
178
158
|
type: "string",
|
|
179
159
|
alias: "s",
|
|
180
|
-
description: "Site
|
|
160
|
+
description: "Site, for example example.com; defaults to config.defaultSite or a prompt"
|
|
181
161
|
},
|
|
182
162
|
url: {
|
|
183
163
|
type: "positional",
|
|
@@ -189,7 +169,7 @@ const deleteCommand = defineCommand({
|
|
|
189
169
|
const { json } = applyOutputMode(args);
|
|
190
170
|
const ctx = await createCommandContext({ needsAuth: true });
|
|
191
171
|
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
192
|
-
await ctx.client.sitemaps.delete(siteUrl, args.url)
|
|
172
|
+
await ctx.client.sitemaps.delete(siteUrl, args.url);
|
|
193
173
|
if (json) {
|
|
194
174
|
console.log(JSON.stringify({
|
|
195
175
|
siteUrl,
|
|
@@ -238,6 +218,7 @@ const discoverCommand = defineCommand({
|
|
|
238
218
|
logger.success(`Discovered sitemap: ${url}`);
|
|
239
219
|
}
|
|
240
220
|
});
|
|
221
|
+
const SITEMAP_WALK_MAX_URLS = 1e6;
|
|
241
222
|
const urlsCommand = defineCommand({
|
|
242
223
|
meta: {
|
|
243
224
|
name: "urls",
|
|
@@ -253,7 +234,7 @@ const urlsCommand = defineCommand({
|
|
|
253
234
|
"limit": {
|
|
254
235
|
type: "string",
|
|
255
236
|
alias: "l",
|
|
256
|
-
description: "
|
|
237
|
+
description: "Print at most N URLs across all nested sitemaps"
|
|
257
238
|
},
|
|
258
239
|
"max-depth": {
|
|
259
240
|
type: "string",
|
|
@@ -265,18 +246,17 @@ const urlsCommand = defineCommand({
|
|
|
265
246
|
const limit = parseIntegerOption(args.limit, "--limit");
|
|
266
247
|
const maxDepth = parseIntegerOption(args["max-depth"], "--max-depth", 0);
|
|
267
248
|
const result = await loadSitemapUrls(String(args.url), {
|
|
268
|
-
maxUrls:
|
|
249
|
+
maxUrls: SITEMAP_WALK_MAX_URLS,
|
|
269
250
|
maxDepth
|
|
270
251
|
});
|
|
271
|
-
if (result._tag === "error") {
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
}
|
|
275
|
-
const { urls, complete, documentsRead } = result.value;
|
|
252
|
+
if (result._tag === "error") throw new Error(`Sitemap fetch failed: ${result.message}`);
|
|
253
|
+
const { complete, documentsRead } = result.value;
|
|
254
|
+
const urls = limit === void 0 ? result.value.urls : result.value.urls.slice(0, limit);
|
|
276
255
|
if (json) {
|
|
277
256
|
console.log(JSON.stringify({
|
|
278
257
|
sitemap: args.url,
|
|
279
258
|
count: urls.length,
|
|
259
|
+
found: result.value.urls.length,
|
|
280
260
|
complete,
|
|
281
261
|
documentsRead,
|
|
282
262
|
urls
|
|
@@ -290,20 +270,16 @@ const urlsCommand = defineCommand({
|
|
|
290
270
|
const currentCommand = defineCommand({
|
|
291
271
|
meta: {
|
|
292
272
|
name: "current",
|
|
293
|
-
description: "Read the
|
|
273
|
+
description: "Read the canonical sitemap generation and feed list (hosted)"
|
|
294
274
|
},
|
|
295
275
|
args: {
|
|
296
276
|
...OUTPUT_ARGS,
|
|
297
|
-
...HOSTED_ARGS
|
|
298
|
-
"site-id": {
|
|
299
|
-
type: "positional",
|
|
300
|
-
required: true,
|
|
301
|
-
description: "Hosted gscdump site id"
|
|
302
|
-
}
|
|
277
|
+
...HOSTED_ARGS
|
|
303
278
|
},
|
|
304
279
|
async run({ args }) {
|
|
305
280
|
const { json } = applyOutputMode(args);
|
|
306
|
-
const
|
|
281
|
+
const { client, site } = await hostedSitemapSite(args, "current");
|
|
282
|
+
const result = await client.getSiteSitemaps({ params: { siteId: site.siteId } });
|
|
307
283
|
if (json) {
|
|
308
284
|
console.log(JSON.stringify(result.data, null, 2));
|
|
309
285
|
return;
|
|
@@ -316,17 +292,12 @@ const currentCommand = defineCommand({
|
|
|
316
292
|
const historyCommand = defineCommand({
|
|
317
293
|
meta: {
|
|
318
294
|
name: "history",
|
|
319
|
-
description: "Read
|
|
295
|
+
description: "Read canonical sitemap membership and lastmod changes (hosted)"
|
|
320
296
|
},
|
|
321
297
|
args: {
|
|
322
298
|
...OUTPUT_ARGS,
|
|
323
299
|
...HOSTED_ARGS,
|
|
324
|
-
|
|
325
|
-
type: "positional",
|
|
326
|
-
required: true,
|
|
327
|
-
description: "Hosted gscdump site id"
|
|
328
|
-
},
|
|
329
|
-
"days": {
|
|
300
|
+
days: {
|
|
330
301
|
type: "string",
|
|
331
302
|
description: "History window in days"
|
|
332
303
|
}
|
|
@@ -334,8 +305,9 @@ const historyCommand = defineCommand({
|
|
|
334
305
|
async run({ args }) {
|
|
335
306
|
const { json } = applyOutputMode(args);
|
|
336
307
|
const days = parseIntegerOption(args.days, "--days");
|
|
337
|
-
const
|
|
338
|
-
|
|
308
|
+
const { client, site } = await hostedSitemapSite(args, "history");
|
|
309
|
+
const result = await client.getSiteSitemapChanges({
|
|
310
|
+
params: { siteId: site.siteId },
|
|
339
311
|
query: { ...days ? { days } : {} }
|
|
340
312
|
});
|
|
341
313
|
console.log(JSON.stringify(result.data, null, json ? 2 : 0));
|
|
@@ -344,22 +316,17 @@ const historyCommand = defineCommand({
|
|
|
344
316
|
const membershipCommand = defineCommand({
|
|
345
317
|
meta: {
|
|
346
318
|
name: "membership",
|
|
347
|
-
description: "Query exact
|
|
319
|
+
description: "Query exact sitemap membership evidence for comma-separated URLs (hosted)"
|
|
348
320
|
},
|
|
349
321
|
args: {
|
|
350
322
|
...OUTPUT_ARGS,
|
|
351
323
|
...HOSTED_ARGS,
|
|
352
|
-
|
|
353
|
-
type: "positional",
|
|
354
|
-
required: true,
|
|
355
|
-
description: "Hosted gscdump site id"
|
|
356
|
-
},
|
|
357
|
-
"urls": {
|
|
324
|
+
urls: {
|
|
358
325
|
type: "string",
|
|
359
326
|
required: true,
|
|
360
327
|
description: "Comma-separated exact URLs"
|
|
361
328
|
},
|
|
362
|
-
|
|
329
|
+
generation: {
|
|
363
330
|
type: "string",
|
|
364
331
|
description: "Pin the query to a generation id"
|
|
365
332
|
}
|
|
@@ -367,8 +334,9 @@ const membershipCommand = defineCommand({
|
|
|
367
334
|
async run({ args }) {
|
|
368
335
|
const { json } = applyOutputMode(args);
|
|
369
336
|
const urls = String(args.urls).split(",").map((url) => url.trim()).filter(Boolean);
|
|
370
|
-
const
|
|
371
|
-
|
|
337
|
+
const { client, site } = await hostedSitemapSite(args, "membership");
|
|
338
|
+
const result = await client.querySitemapMembership({
|
|
339
|
+
params: { siteId: site.siteId },
|
|
372
340
|
body: {
|
|
373
341
|
urls,
|
|
374
342
|
...args.generation ? { generationId: String(args.generation) } : {}
|
|
@@ -387,29 +355,24 @@ const membershipCommand = defineCommand({
|
|
|
387
355
|
const lastmodCommand = defineCommand({
|
|
388
356
|
meta: {
|
|
389
357
|
name: "lastmod",
|
|
390
|
-
description: "List a generation-pinned
|
|
358
|
+
description: "List a generation-pinned page of URL lastmod evidence (hosted)"
|
|
391
359
|
},
|
|
392
360
|
args: {
|
|
393
361
|
...OUTPUT_ARGS,
|
|
394
362
|
...HOSTED_ARGS,
|
|
395
|
-
|
|
396
|
-
type: "positional",
|
|
397
|
-
required: true,
|
|
398
|
-
description: "Hosted gscdump site id"
|
|
399
|
-
},
|
|
400
|
-
"generation": {
|
|
363
|
+
generation: {
|
|
401
364
|
type: "string",
|
|
402
365
|
description: "Pin to a generation id"
|
|
403
366
|
},
|
|
404
|
-
|
|
367
|
+
feedpath: {
|
|
405
368
|
type: "string",
|
|
406
369
|
description: "Restrict to one exact sitemap feed URL"
|
|
407
370
|
},
|
|
408
|
-
|
|
371
|
+
cursor: {
|
|
409
372
|
type: "string",
|
|
410
373
|
description: "Opaque cursor from the prior page"
|
|
411
374
|
},
|
|
412
|
-
|
|
375
|
+
limit: {
|
|
413
376
|
type: "string",
|
|
414
377
|
description: "Page size, maximum 10000"
|
|
415
378
|
}
|
|
@@ -417,8 +380,9 @@ const lastmodCommand = defineCommand({
|
|
|
417
380
|
async run({ args }) {
|
|
418
381
|
const { json } = applyOutputMode(args);
|
|
419
382
|
const limit = parseIntegerOption(args.limit, "--limit");
|
|
420
|
-
const
|
|
421
|
-
|
|
383
|
+
const { client, site } = await hostedSitemapSite(args, "lastmod");
|
|
384
|
+
const result = await client.listSitemapUrls({
|
|
385
|
+
params: { siteId: site.siteId },
|
|
422
386
|
query: {
|
|
423
387
|
...args.generation ? { generationId: String(args.generation) } : {},
|
|
424
388
|
...args.feedpath ? { feedpath: String(args.feedpath) } : {},
|
|
@@ -437,29 +401,25 @@ const lastmodCommand = defineCommand({
|
|
|
437
401
|
const exportCommand = defineCommand({
|
|
438
402
|
meta: {
|
|
439
403
|
name: "export",
|
|
440
|
-
description: "Get a generation-pinned
|
|
404
|
+
description: "Get a generation-pinned bulk sitemap export descriptor (hosted)"
|
|
441
405
|
},
|
|
442
406
|
args: {
|
|
443
407
|
...OUTPUT_ARGS,
|
|
444
408
|
...HOSTED_ARGS,
|
|
445
|
-
|
|
446
|
-
type: "positional",
|
|
447
|
-
required: true,
|
|
448
|
-
description: "Hosted gscdump site id"
|
|
449
|
-
},
|
|
450
|
-
"generation": {
|
|
409
|
+
generation: {
|
|
451
410
|
type: "string",
|
|
452
411
|
description: "Pin to a generation id"
|
|
453
412
|
},
|
|
454
|
-
|
|
413
|
+
feedpath: {
|
|
455
414
|
type: "string",
|
|
456
415
|
description: "Restrict to one exact sitemap feed URL"
|
|
457
416
|
}
|
|
458
417
|
},
|
|
459
418
|
async run({ args }) {
|
|
460
419
|
const { json } = applyOutputMode(args);
|
|
461
|
-
const
|
|
462
|
-
|
|
420
|
+
const { client, site } = await hostedSitemapSite(args, "export");
|
|
421
|
+
const result = await client.getSitemapExport({
|
|
422
|
+
params: { siteId: site.siteId },
|
|
463
423
|
query: {
|
|
464
424
|
...args.generation ? { generationId: String(args.generation) } : {},
|
|
465
425
|
...args.feedpath ? { feedpath: String(args.feedpath) } : {}
|
|
@@ -489,19 +449,6 @@ const sitemapsCommand = defineCommand({
|
|
|
489
449
|
export: exportCommand
|
|
490
450
|
},
|
|
491
451
|
async run({ args }) {
|
|
492
|
-
if (!noSubcommandSelected("sitemaps", [
|
|
493
|
-
"list",
|
|
494
|
-
"get",
|
|
495
|
-
"submit",
|
|
496
|
-
"delete",
|
|
497
|
-
"discover",
|
|
498
|
-
"urls",
|
|
499
|
-
"current",
|
|
500
|
-
"history",
|
|
501
|
-
"membership",
|
|
502
|
-
"lastmod",
|
|
503
|
-
"export"
|
|
504
|
-
])) return;
|
|
505
452
|
await listCommand.run?.({
|
|
506
453
|
args,
|
|
507
454
|
cmd: listCommand,
|