@gscdump/cli 3.8.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -30
- package/bin/gscdump.mjs +21 -1
- package/dist/analysis-local.mjs +172 -53
- package/dist/auth-state.mjs +21 -6
- package/dist/auth.mjs +27 -18
- package/dist/bing-data.mjs +6 -2
- package/dist/bing-hosted.mjs +1 -1
- package/dist/cli-args.mjs +8 -0
- package/dist/cli.d.mts +5 -1
- package/dist/cli.mjs +41 -10
- package/dist/cloud-google.mjs +8 -1
- package/dist/command-meta.mjs +4 -4
- package/dist/command-registry.mjs +86 -10
- package/dist/commands/analyze.mjs +84 -43
- package/dist/commands/auth.mjs +46 -37
- package/dist/commands/bing.mjs +3 -3
- package/dist/commands/compact.mjs +12 -8
- package/dist/commands/config.mjs +14 -14
- package/dist/commands/doctor.mjs +26 -26
- package/dist/commands/dump.mjs +203 -105
- package/dist/commands/entities.mjs +10 -113
- package/dist/commands/gc.mjs +4 -3
- package/dist/commands/indexing-urls.mjs +185 -0
- package/dist/commands/indexing.mjs +12 -8
- package/dist/commands/init.mjs +77 -18
- package/dist/commands/inspect.mjs +97 -110
- package/dist/commands/mcp.mjs +37 -57
- package/dist/commands/papercut.mjs +1 -1
- package/dist/commands/profile.mjs +9 -17
- package/dist/commands/query.mjs +219 -210
- package/dist/commands/report.mjs +68 -85
- package/dist/commands/rollups.mjs +9 -6
- package/dist/commands/sitemaps.mjs +51 -104
- package/dist/commands/sites.mjs +40 -24
- package/dist/commands/stats.mjs +14 -17
- package/dist/commands/store-purge.mjs +9 -5
- package/dist/commands/store.mjs +0 -12
- package/dist/commands/sync.mjs +774 -296
- package/dist/config.mjs +7 -4
- package/dist/context.mjs +51 -21
- package/dist/coverage.mjs +251 -0
- package/dist/dump-bing.mjs +71 -0
- package/dist/dump-writers.mjs +246 -0
- package/dist/error-handler.mjs +108 -38
- package/dist/filters.mjs +125 -0
- package/dist/hosted-site.mjs +91 -0
- package/dist/inspect-urls.mjs +76 -0
- package/dist/inspection-record.mjs +93 -0
- package/dist/local-entities.mjs +658 -0
- package/dist/mcp/errors.mjs +71 -18
- package/dist/mcp/handlers/diagnostics.mjs +15 -12
- package/dist/mcp/handlers/reports.mjs +33 -42
- package/dist/mcp/server/index.mjs +80 -38
- package/dist/mcp/types.mjs +7 -7
- package/dist/package.mjs +1 -1
- package/dist/quota-ledger.mjs +370 -0
- package/dist/render/analysis.mjs +8 -1
- package/dist/render/report.mjs +8 -1
- package/dist/request-pacer.mjs +29 -0
- package/dist/route.mjs +301 -0
- package/dist/sitemap.mjs +4 -0
- package/dist/sql-views.mjs +117 -0
- package/dist/store-sites.mjs +134 -0
- package/dist/sync-plan.mjs +129 -0
- package/dist/sync-run.mjs +96 -0
- package/dist/table-sources.mjs +57 -0
- package/dist/token-info.mjs +54 -0
- package/dist/utils.mjs +28 -19
- package/dist/window.mjs +159 -0
- package/package.json +17 -14
- package/skills/gscdump/SKILL.md +172 -37
- package/dist/commands/export.mjs +0 -76
- package/dist/native-duckdb.mjs +0 -46
package/dist/commands/dump.mjs
CHANGED
|
@@ -1,26 +1,28 @@
|
|
|
1
1
|
import { dumpCommandMeta } from "../command-meta.mjs";
|
|
2
|
-
import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger,
|
|
2
|
+
import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger, parseNameList, parseSearchType } from "../utils.mjs";
|
|
3
3
|
import { allTables } from "../local-store.mjs";
|
|
4
|
+
import { readSiteMap, siteUrlForId } from "../store-sites.mjs";
|
|
4
5
|
import { createCommandContext } from "../context.mjs";
|
|
5
|
-
import {
|
|
6
|
+
import { ENTITY_DATASETS, readEntityDatasets } from "../local-entities.mjs";
|
|
7
|
+
import "../sync-plan.mjs";
|
|
8
|
+
import { nextCommand, readStoreCoverage, renderCoverage } from "../coverage.mjs";
|
|
9
|
+
import { isProcessAlive, readSyncRun, syncRunStatus } from "../sync-run.mjs";
|
|
10
|
+
import { dumpBing } from "../dump-bing.mjs";
|
|
11
|
+
import { groupTableSources } from "../table-sources.mjs";
|
|
12
|
+
import { DUMP_FORMATS, isDumpFormat, openDumpSink } from "../dump-writers.mjs";
|
|
6
13
|
import process from "node:process";
|
|
7
14
|
import { defineCommand } from "citty";
|
|
8
15
|
import fs from "node:fs/promises";
|
|
9
16
|
import path from "node:path";
|
|
17
|
+
import { Buffer } from "node:buffer";
|
|
10
18
|
const DEFAULT_OUT = "./gscdump-export";
|
|
11
|
-
const FORMATS = [
|
|
12
|
-
"parquet",
|
|
13
|
-
"json",
|
|
14
|
-
"ndjson",
|
|
15
|
-
"csv"
|
|
16
|
-
];
|
|
17
19
|
const dumpCommand = defineCommand({
|
|
18
20
|
meta: dumpCommandMeta,
|
|
19
21
|
args: {
|
|
20
22
|
"site": {
|
|
21
23
|
type: "string",
|
|
22
24
|
alias: "s",
|
|
23
|
-
description: "Site
|
|
25
|
+
description: "Site, for example example.com; ignored with --all-sites"
|
|
24
26
|
},
|
|
25
27
|
"out": {
|
|
26
28
|
type: "string",
|
|
@@ -32,12 +34,16 @@ const dumpCommand = defineCommand({
|
|
|
32
34
|
type: "string",
|
|
33
35
|
alias: "F",
|
|
34
36
|
default: "parquet",
|
|
35
|
-
description: `Output format: ${
|
|
37
|
+
description: `Output format: ${DUMP_FORMATS.join(", ")} (default: parquet). sqlite and duckdb write one database file`
|
|
36
38
|
},
|
|
37
39
|
"tables": {
|
|
38
40
|
type: "string",
|
|
39
41
|
alias: "t",
|
|
40
|
-
description: `Comma-separated
|
|
42
|
+
description: `Comma-separated tables and datasets (default: all). Known: ${[
|
|
43
|
+
...allTables(),
|
|
44
|
+
...ENTITY_DATASETS,
|
|
45
|
+
"bing"
|
|
46
|
+
].join(", ")}`
|
|
41
47
|
},
|
|
42
48
|
"all-sites": {
|
|
43
49
|
type: "boolean",
|
|
@@ -49,6 +55,12 @@ const dumpCommand = defineCommand({
|
|
|
49
55
|
default: false,
|
|
50
56
|
description: "Compact every closed month into a single file before exporting"
|
|
51
57
|
},
|
|
58
|
+
"bing": {
|
|
59
|
+
type: "boolean",
|
|
60
|
+
default: true,
|
|
61
|
+
description: "Also export Bing data for matching sites when a Bing login exists",
|
|
62
|
+
negativeDescription: "Skip Bing"
|
|
63
|
+
},
|
|
52
64
|
"search-type": {
|
|
53
65
|
type: "string",
|
|
54
66
|
description: `Restrict dump to a single GSC search-type slice (${ALL_SEARCH_TYPES.join(", ")}). Default: all slices.`
|
|
@@ -58,26 +70,28 @@ const dumpCommand = defineCommand({
|
|
|
58
70
|
async run({ args }) {
|
|
59
71
|
const { json, quiet } = applyOutputMode(args);
|
|
60
72
|
const format = String(args.format);
|
|
61
|
-
if (!
|
|
62
|
-
logger.error(`Invalid --format: ${format}. Allowed: ${
|
|
73
|
+
if (!isDumpFormat(format)) {
|
|
74
|
+
logger.error(`Invalid --format: ${format}. Allowed: ${DUMP_FORMATS.join(", ")}`);
|
|
63
75
|
process.exit(1);
|
|
64
76
|
}
|
|
65
|
-
const tablesFilter = args.tables ? new Set(
|
|
77
|
+
const tablesFilter = args.tables ? new Set(parseNameList(args.tables, [
|
|
78
|
+
...allTables(),
|
|
79
|
+
...ENTITY_DATASETS,
|
|
80
|
+
"bing"
|
|
81
|
+
], "--tables")) : null;
|
|
66
82
|
const searchType = parseSearchType(args["search-type"]);
|
|
67
|
-
const ctx = await createCommandContext({
|
|
68
|
-
needsAuth: !args["all-sites"],
|
|
69
|
-
needsStore: true
|
|
70
|
-
});
|
|
83
|
+
const ctx = await createCommandContext({ needsStore: true });
|
|
71
84
|
const store = ctx.store;
|
|
72
85
|
const outDir = path.resolve(String(args.out));
|
|
86
|
+
const siteMap = await readSiteMap(store.dataDir, store.userId);
|
|
73
87
|
let preloadedEntries = args["all-sites"] ? await store.engine.listLive({
|
|
74
88
|
userId: store.userId,
|
|
75
89
|
...searchType !== void 0 ? { searchType } : {}
|
|
76
90
|
}) : void 0;
|
|
77
91
|
const targets = args["all-sites"] ? [...new Set(preloadedEntries.flatMap((entry) => entry.siteId ? [entry.siteId] : []))].map((siteId) => ({
|
|
78
|
-
site: siteId,
|
|
92
|
+
site: siteUrlForId(siteMap, siteId),
|
|
79
93
|
siteId
|
|
80
|
-
})) : await ctx.resolveSite(args.site ? String(args.site) : void 0).then((site) => [{
|
|
94
|
+
})) : await ctx.resolveSite(args.site ? String(args.site) : void 0, { scope: "store" }).then((site) => [{
|
|
81
95
|
site,
|
|
82
96
|
siteId: store.siteIdFor(site)
|
|
83
97
|
}]);
|
|
@@ -92,105 +106,189 @@ const dumpCommand = defineCommand({
|
|
|
92
106
|
...searchType !== void 0 ? { searchType } : {}
|
|
93
107
|
});
|
|
94
108
|
}
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
summary.push({
|
|
114
|
-
site: target.site,
|
|
115
|
-
files: written.files,
|
|
116
|
-
rows: written.rows,
|
|
117
|
-
format,
|
|
118
|
-
outPath: outDir
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
}
|
|
109
|
+
const bing = args.bing === false || tablesFilter && !tablesFilter.has("bing") ? { _tag: "disabled" } : await dumpBing({
|
|
110
|
+
googleSites: args["all-sites"] ? "all" : targets.map((target) => target.site),
|
|
111
|
+
outDir,
|
|
112
|
+
format
|
|
113
|
+
}).catch((error) => ({
|
|
114
|
+
_tag: "failed",
|
|
115
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
116
|
+
}));
|
|
117
|
+
const result = await dumpSites({
|
|
118
|
+
store,
|
|
119
|
+
targets,
|
|
120
|
+
outDir,
|
|
121
|
+
format,
|
|
122
|
+
...tablesFilter ? { tables: tablesFilter } : {},
|
|
123
|
+
...searchType !== void 0 ? { searchType } : {},
|
|
124
|
+
...preloadedEntries ? { entries: preloadedEntries } : {},
|
|
125
|
+
bing
|
|
126
|
+
});
|
|
122
127
|
if (json) {
|
|
123
128
|
console.log(JSON.stringify({
|
|
124
|
-
|
|
125
|
-
|
|
129
|
+
...result,
|
|
130
|
+
bing
|
|
126
131
|
}, null, 2));
|
|
127
132
|
return;
|
|
128
133
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
134
|
+
const bytesByPath = new Map(result.files.map((file) => [file.path, file.bytes]));
|
|
135
|
+
const database = format === "sqlite" || format === "duckdb";
|
|
136
|
+
if (database && result.sites.some((site) => site.datasets.length > 0)) {
|
|
137
|
+
const file = result.files[0];
|
|
138
|
+
logger.success(`Wrote ${displayPath(file.path)} (${formatBytes(file.bytes)})`);
|
|
139
|
+
}
|
|
140
|
+
for (const site of result.sites) {
|
|
141
|
+
if (site.datasets.length === 0) {
|
|
142
|
+
if (!quiet) logger.warn(`No data for ${site.site}; skipping`);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
logger.success(`[${site.site}] ${site.totals.datasets} dataset(s), ${site.totals.rows.toLocaleString()} rows`);
|
|
146
|
+
if (quiet) continue;
|
|
147
|
+
for (const dataset of site.datasets) {
|
|
148
|
+
const label = database ? `${dataset.dataset}${dataset.searchType ? ` (search_type = ${dataset.searchType})` : ""}` : `${path.relative(outDir, dataset.path)} ${formatBytes(bytesByPath.get(dataset.path) ?? 0)}`;
|
|
149
|
+
console.log(` ${label} ${dataset.rows.toLocaleString()} rows`);
|
|
150
|
+
}
|
|
151
|
+
for (const skip of site.skipped) console.log(` \x1B[90m${skip.dataset}: no rows yet; skipped\x1B[0m`);
|
|
152
|
+
for (const line of renderCoverage(site.coverage)) console.log(` ${line}`);
|
|
153
|
+
for (const job of site.coverage.analytics) if (job.coverage.kind === "partial" && job.coverage.failed > 0) logger.warn(`${site.site} ${job.table} (${job.searchType}): ${job.coverage.failed} failed date(s). Run ${nextCommand(site.coverage)} to retry them.`);
|
|
132
154
|
}
|
|
155
|
+
if (bing._tag === "dumped") {
|
|
156
|
+
for (const site of bing.sites) {
|
|
157
|
+
const bytes = site.files.reduce((sum, file) => sum + file.bytes, 0);
|
|
158
|
+
logger.success(`[Bing ${site.siteUrl}] ${site.files.length} file(s), ${formatBytes(bytes)}`);
|
|
159
|
+
if (!quiet) for (const file of site.files) console.log(` ${path.relative(outDir, file.path)} ${formatBytes(file.bytes)} ${file.rows.toLocaleString()} rows`);
|
|
160
|
+
}
|
|
161
|
+
for (const failure of bing.failures) logger.warn(`Bing ${failure.siteUrl} not exported: ${failure.error}`);
|
|
162
|
+
} else if (bing._tag === "failed") logger.warn(`Bing not exported: ${bing.reason}`);
|
|
163
|
+
else if (bing._tag === "skipped" && !quiet) logger.info(`Bing skipped: ${bing.reason}`);
|
|
164
|
+
if (!quiet) for (const file of result.metadataFiles) console.log(` ${path.relative(outDir, file.path)} ${formatBytes(file.bytes)}`);
|
|
133
165
|
}
|
|
134
166
|
});
|
|
135
|
-
|
|
136
|
-
return
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
ready = fs.mkdir(dir, { recursive: true }).then(() => void 0);
|
|
149
|
-
readyDirectories.set(dir, ready);
|
|
150
|
-
ready.catch(() => readyDirectories.delete(dir));
|
|
151
|
-
}
|
|
152
|
-
await ready;
|
|
167
|
+
function formatBytes(bytes) {
|
|
168
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
169
|
+
const units = [
|
|
170
|
+
"KB",
|
|
171
|
+
"MB",
|
|
172
|
+
"GB",
|
|
173
|
+
"TB"
|
|
174
|
+
];
|
|
175
|
+
let value = bytes / 1024;
|
|
176
|
+
let unit = 0;
|
|
177
|
+
while (value >= 1024 && unit < units.length - 1) {
|
|
178
|
+
value /= 1024;
|
|
179
|
+
unit++;
|
|
153
180
|
}
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
181
|
+
return `${value.toFixed(value >= 100 ? 0 : 1)} ${units[unit]}`;
|
|
182
|
+
}
|
|
183
|
+
async function dumpSites(opts) {
|
|
184
|
+
const { outDir, format } = opts;
|
|
185
|
+
const sink = await openDumpSink(outDir, format);
|
|
186
|
+
const summary = await dumpEachSite(sink, opts).catch(async (error) => {
|
|
187
|
+
await sink.close();
|
|
188
|
+
throw error;
|
|
161
189
|
});
|
|
162
|
-
|
|
190
|
+
const files = await sink.close();
|
|
191
|
+
const metadataFiles = [];
|
|
192
|
+
const sites = opts.targets.map((target) => ({
|
|
193
|
+
siteUrl: target.site,
|
|
194
|
+
siteId: target.siteId
|
|
195
|
+
}));
|
|
196
|
+
metadataFiles.push(await writeJsonFile(path.join(outDir, "sites.json"), { sites }));
|
|
197
|
+
metadataFiles.push(await writeJsonFile(path.join(outDir, "manifest.json"), {
|
|
198
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
199
|
+
format,
|
|
200
|
+
files: files.map((file) => ({
|
|
201
|
+
...file,
|
|
202
|
+
path: path.relative(outDir, file.path)
|
|
203
|
+
})),
|
|
204
|
+
sites: summary.map((site) => ({
|
|
205
|
+
...site,
|
|
206
|
+
datasets: site.datasets.map((dataset) => ({
|
|
207
|
+
...dataset,
|
|
208
|
+
path: path.relative(outDir, dataset.path)
|
|
209
|
+
}))
|
|
210
|
+
})),
|
|
211
|
+
...opts.bing ? { bing: manifestBing(opts.bing, outDir) } : {}
|
|
212
|
+
}));
|
|
213
|
+
return {
|
|
214
|
+
outDir,
|
|
215
|
+
format,
|
|
216
|
+
files,
|
|
217
|
+
metadataFiles,
|
|
218
|
+
sites: summary
|
|
219
|
+
};
|
|
163
220
|
}
|
|
164
|
-
async function
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
221
|
+
async function dumpEachSite(sink, opts) {
|
|
222
|
+
const { store, tables } = opts;
|
|
223
|
+
const wantedEntities = ENTITY_DATASETS.filter((dataset) => !tables || tables.has(dataset));
|
|
224
|
+
const summary = [];
|
|
225
|
+
for (const target of opts.targets) {
|
|
226
|
+
const entries = (opts.entries ? opts.entries.filter((entry) => entry.siteId === target.siteId) : await listLiveEntries(store, target.siteId, opts.searchType)).filter((e) => !tables || tables.has(e.table));
|
|
227
|
+
const datasets = [];
|
|
228
|
+
for (const source of groupTableSources(entries, store.dataDir, { [target.siteId]: target.site })) datasets.push(await sink.writeTable(source));
|
|
229
|
+
const skipped = [];
|
|
230
|
+
for (const dataset of await readEntityDatasets(store.dataSource, {
|
|
231
|
+
userId: store.userId,
|
|
232
|
+
siteId: target.siteId
|
|
233
|
+
}, wantedEntities)) if (dataset.rows.length === 0) skipped.push({
|
|
234
|
+
dataset: dataset.dataset,
|
|
235
|
+
reason: "empty"
|
|
236
|
+
});
|
|
237
|
+
else datasets.push(await sink.writeEntity(target.site, dataset));
|
|
238
|
+
summary.push({
|
|
239
|
+
site: target.site,
|
|
240
|
+
siteId: target.siteId,
|
|
241
|
+
coverage: await readDumpCoverage(store, target.site),
|
|
242
|
+
datasets,
|
|
243
|
+
skipped,
|
|
244
|
+
totals: {
|
|
245
|
+
datasets: datasets.length,
|
|
246
|
+
rows: datasets.reduce((sum, dataset) => sum + dataset.rows, 0)
|
|
247
|
+
}
|
|
248
|
+
});
|
|
188
249
|
}
|
|
250
|
+
return summary;
|
|
251
|
+
}
|
|
252
|
+
function manifestBing(step, outDir) {
|
|
253
|
+
if (step._tag !== "dumped") return step;
|
|
189
254
|
return {
|
|
190
|
-
|
|
191
|
-
|
|
255
|
+
...step,
|
|
256
|
+
sites: step.sites.map((site) => ({
|
|
257
|
+
siteUrl: site.siteUrl,
|
|
258
|
+
files: site.files.map((file) => ({
|
|
259
|
+
...file,
|
|
260
|
+
path: path.relative(outDir, file.path)
|
|
261
|
+
}))
|
|
262
|
+
}))
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
async function writeJsonFile(target, value) {
|
|
266
|
+
const body = `${JSON.stringify(value, null, 2)}\n`;
|
|
267
|
+
await fs.writeFile(target, body);
|
|
268
|
+
return {
|
|
269
|
+
path: target,
|
|
270
|
+
bytes: Buffer.byteLength(body)
|
|
192
271
|
};
|
|
193
272
|
}
|
|
273
|
+
async function readDumpCoverage(store, site) {
|
|
274
|
+
const run = syncRunStatus(await readSyncRun(store.dataDir), {
|
|
275
|
+
now: Date.now(),
|
|
276
|
+
isAlive: isProcessAlive
|
|
277
|
+
});
|
|
278
|
+
return readStoreCoverage({
|
|
279
|
+
store,
|
|
280
|
+
site,
|
|
281
|
+
inspectLimit: 50,
|
|
282
|
+
run
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
async function listLiveEntries(store, siteId, searchType) {
|
|
286
|
+
return store.engine.listLive({
|
|
287
|
+
userId: store.userId,
|
|
288
|
+
siteId,
|
|
289
|
+
...searchType !== void 0 ? { searchType } : {}
|
|
290
|
+
});
|
|
291
|
+
}
|
|
194
292
|
async function compactClosedMonths(store, siteId, quiet) {
|
|
195
293
|
for (const table of allTables()) {
|
|
196
294
|
if (!quiet) logger.info(`Compacting ${table} (raw→d7→d30→d90)`);
|
|
@@ -201,4 +299,4 @@ async function compactClosedMonths(store, siteId, quiet) {
|
|
|
201
299
|
});
|
|
202
300
|
}
|
|
203
301
|
}
|
|
204
|
-
export { dumpCommand };
|
|
302
|
+
export { dumpCommand, dumpSites, formatBytes };
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { entitiesCommandMeta } from "../command-meta.mjs";
|
|
2
2
|
import { OUTPUT_ARGS, applyOutputMode, logger, parseIntegerOption, progressBar, runWithConcurrency } from "../utils.mjs";
|
|
3
|
+
import { formatSiteIdCollision, recordStoreSite } from "../store-sites.mjs";
|
|
3
4
|
import { createCommandContext } from "../context.mjs";
|
|
4
5
|
import process from "node:process";
|
|
5
6
|
import { defineCommand } from "citty";
|
|
6
7
|
import { readFile } from "node:fs/promises";
|
|
7
8
|
import { Buffer } from "node:buffer";
|
|
8
9
|
import { createIndexingMetadataStore, createInspectionStore } from "@gscdump/engine/entities";
|
|
9
|
-
const INSPECTION_QPD_PER_PROPERTY = 2e3;
|
|
10
10
|
const INDEXING_NOT_FOUND_RE = /\b404\b|NOT_FOUND/i;
|
|
11
11
|
const INSPECTION_HISTORY_LOOKBACK_MONTHS = 24;
|
|
12
12
|
async function findLatestInspection(inspector, ctx, url) {
|
|
@@ -28,110 +28,6 @@ async function readUrlList(opts) {
|
|
|
28
28
|
for await (const chunk of process.stdin) chunks.push(chunk);
|
|
29
29
|
return Buffer.concat(chunks).toString("utf8").split("\n").map((l) => l.trim()).filter(Boolean);
|
|
30
30
|
}
|
|
31
|
-
const inspectSubCommand = defineCommand({
|
|
32
|
-
meta: {
|
|
33
|
-
name: "inspect",
|
|
34
|
-
description: "Run URL Inspection for a list of URLs and persist results to the local entity store"
|
|
35
|
-
},
|
|
36
|
-
args: {
|
|
37
|
-
site: {
|
|
38
|
-
type: "string",
|
|
39
|
-
alias: "s",
|
|
40
|
-
description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
|
|
41
|
-
},
|
|
42
|
-
file: {
|
|
43
|
-
type: "string",
|
|
44
|
-
alias: "f",
|
|
45
|
-
description: "Path to a file with one URL per line. If omitted, reads from stdin."
|
|
46
|
-
},
|
|
47
|
-
limit: {
|
|
48
|
-
type: "string",
|
|
49
|
-
description: `Max URLs to inspect this run (default: ${INSPECTION_QPD_PER_PROPERTY}, the per-property GSC daily quota)`
|
|
50
|
-
},
|
|
51
|
-
concurrency: {
|
|
52
|
-
type: "string",
|
|
53
|
-
alias: "c",
|
|
54
|
-
default: "4",
|
|
55
|
-
description: "Concurrent in-flight inspect calls (default: 4)"
|
|
56
|
-
},
|
|
57
|
-
...OUTPUT_ARGS
|
|
58
|
-
},
|
|
59
|
-
async run({ args }) {
|
|
60
|
-
const { json, quiet } = applyOutputMode(args);
|
|
61
|
-
const limit = parseIntegerOption(args.limit, "--limit") ?? INSPECTION_QPD_PER_PROPERTY;
|
|
62
|
-
const concurrency = parseIntegerOption(args.concurrency, "--concurrency") ?? 4;
|
|
63
|
-
const ctx = await createCommandContext({
|
|
64
|
-
needsAuth: true,
|
|
65
|
-
needsStore: true
|
|
66
|
-
});
|
|
67
|
-
const client = ctx.client;
|
|
68
|
-
const store = ctx.store;
|
|
69
|
-
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
70
|
-
const urls = (await readUrlList({ file: args.file ? String(args.file) : void 0 })).slice(0, limit);
|
|
71
|
-
if (urls.length === 0) {
|
|
72
|
-
logger.warn("No URLs to inspect.");
|
|
73
|
-
return;
|
|
74
|
-
}
|
|
75
|
-
if (urls.length === limit && limit < INSPECTION_QPD_PER_PROPERTY) logger.info(`Capping at --limit ${limit}`);
|
|
76
|
-
if (urls.length === INSPECTION_QPD_PER_PROPERTY) logger.info(`Hit per-property daily inspection quota (${INSPECTION_QPD_PER_PROPERTY}); remaining URLs will be queued for tomorrow.`);
|
|
77
|
-
const inspector = createInspectionStore({ dataSource: store.dataSource });
|
|
78
|
-
let completed = 0;
|
|
79
|
-
let failed = 0;
|
|
80
|
-
const records = [];
|
|
81
|
-
const failures = [];
|
|
82
|
-
await runWithConcurrency(urls, concurrency, async (url) => {
|
|
83
|
-
const result = await client.inspect(siteUrl, url).catch((err) => err);
|
|
84
|
-
if (result instanceof Error) {
|
|
85
|
-
failed++;
|
|
86
|
-
failures.push({
|
|
87
|
-
url,
|
|
88
|
-
error: result.message
|
|
89
|
-
});
|
|
90
|
-
} else {
|
|
91
|
-
const ix = result.inspectionResult;
|
|
92
|
-
const indexStatus = ix?.indexStatusResult;
|
|
93
|
-
records.push({
|
|
94
|
-
url,
|
|
95
|
-
inspectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
96
|
-
indexStatus: indexStatus?.verdict ?? void 0,
|
|
97
|
-
lastCrawlTime: indexStatus?.lastCrawlTime ?? void 0,
|
|
98
|
-
googleCanonical: indexStatus?.googleCanonical ?? void 0,
|
|
99
|
-
userCanonical: indexStatus?.userCanonical ?? void 0,
|
|
100
|
-
coverageState: indexStatus?.coverageState ?? void 0,
|
|
101
|
-
robotsTxtState: indexStatus?.robotsTxtState ?? void 0,
|
|
102
|
-
indexingState: indexStatus?.indexingState ?? void 0,
|
|
103
|
-
pageFetchState: indexStatus?.pageFetchState ?? void 0,
|
|
104
|
-
mobileUsabilityVerdict: ix?.mobileUsabilityResult?.verdict ?? void 0,
|
|
105
|
-
richResultsVerdict: ix?.richResultsResult?.verdict ?? void 0,
|
|
106
|
-
raw: ix
|
|
107
|
-
});
|
|
108
|
-
}
|
|
109
|
-
completed++;
|
|
110
|
-
if (!quiet) process.stdout.write(`\r${progressBar(completed, urls.length, `${url.slice(0, 60)}`)}`);
|
|
111
|
-
});
|
|
112
|
-
if (!quiet) process.stdout.write("\n");
|
|
113
|
-
await inspector.appendHistory({
|
|
114
|
-
userId: store.userId,
|
|
115
|
-
siteId: store.siteIdFor(siteUrl)
|
|
116
|
-
}, records);
|
|
117
|
-
if (json) console.log(JSON.stringify({
|
|
118
|
-
site: siteUrl,
|
|
119
|
-
inspected: records.length,
|
|
120
|
-
failed,
|
|
121
|
-
failures,
|
|
122
|
-
records
|
|
123
|
-
}, null, 2));
|
|
124
|
-
else if (!quiet) {
|
|
125
|
-
logger.success(`Inspected ${records.length}/${urls.length} URL(s)`);
|
|
126
|
-
if (failed > 0) {
|
|
127
|
-
logger.warn(`${failed} failed:`);
|
|
128
|
-
for (const f of failures.slice(0, 5)) console.log(` ${f.url}: ${f.error}`);
|
|
129
|
-
if (failures.length > 5) console.log(` ... and ${failures.length - 5} more`);
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
if (failed > 0) process.exit(1);
|
|
133
|
-
}
|
|
134
|
-
});
|
|
135
31
|
const showSubCommand = defineCommand({
|
|
136
32
|
meta: {
|
|
137
33
|
name: "show",
|
|
@@ -142,7 +38,7 @@ const showSubCommand = defineCommand({
|
|
|
142
38
|
site: {
|
|
143
39
|
type: "string",
|
|
144
40
|
alias: "s",
|
|
145
|
-
description: "Site
|
|
41
|
+
description: "Site, for example example.com; defaults to config.defaultSite or a prompt"
|
|
146
42
|
},
|
|
147
43
|
url: {
|
|
148
44
|
type: "positional",
|
|
@@ -152,12 +48,9 @@ const showSubCommand = defineCommand({
|
|
|
152
48
|
},
|
|
153
49
|
async run({ args }) {
|
|
154
50
|
const { json } = applyOutputMode(args);
|
|
155
|
-
const ctx = await createCommandContext({
|
|
156
|
-
needsAuth: true,
|
|
157
|
-
needsStore: true
|
|
158
|
-
});
|
|
51
|
+
const ctx = await createCommandContext({ needsStore: true });
|
|
159
52
|
const store = ctx.store;
|
|
160
|
-
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
53
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0, { scope: "store" });
|
|
161
54
|
const record = await findLatestInspection(createInspectionStore({ dataSource: store.dataSource }), {
|
|
162
55
|
userId: store.userId,
|
|
163
56
|
siteId: store.siteIdFor(siteUrl)
|
|
@@ -190,7 +83,7 @@ const indexingSnapshotSubCommand = defineCommand({
|
|
|
190
83
|
site: {
|
|
191
84
|
type: "string",
|
|
192
85
|
alias: "s",
|
|
193
|
-
description: "Site
|
|
86
|
+
description: "Site, for example example.com; defaults to config.defaultSite or a prompt"
|
|
194
87
|
},
|
|
195
88
|
file: {
|
|
196
89
|
type: "string",
|
|
@@ -220,6 +113,11 @@ const indexingSnapshotSubCommand = defineCommand({
|
|
|
220
113
|
logger.warn("No URLs to fetch metadata for.");
|
|
221
114
|
return;
|
|
222
115
|
}
|
|
116
|
+
const claim = await recordStoreSite(store.dataDir, siteUrl, { userId: store.userId });
|
|
117
|
+
if (!claim.ok) {
|
|
118
|
+
logger.error(formatSiteIdCollision(claim.error));
|
|
119
|
+
process.exit(1);
|
|
120
|
+
}
|
|
223
121
|
const records = [];
|
|
224
122
|
const failures = [];
|
|
225
123
|
let completed = 0;
|
|
@@ -270,7 +168,6 @@ const indexingSubCommand = defineCommand({
|
|
|
270
168
|
const entitiesCommand = defineCommand({
|
|
271
169
|
meta: entitiesCommandMeta,
|
|
272
170
|
subCommands: {
|
|
273
|
-
inspect: inspectSubCommand,
|
|
274
171
|
show: showSubCommand,
|
|
275
172
|
indexing: indexingSubCommand
|
|
276
173
|
}
|
package/dist/commands/gc.mjs
CHANGED
|
@@ -16,7 +16,7 @@ const gcCommand = defineCommand({
|
|
|
16
16
|
"site": {
|
|
17
17
|
type: "string",
|
|
18
18
|
alias: "s",
|
|
19
|
-
description: "Restrict to
|
|
19
|
+
description: "Restrict to one Site, for example example.com (default: every Site)"
|
|
20
20
|
},
|
|
21
21
|
"dry-run": {
|
|
22
22
|
type: "boolean",
|
|
@@ -27,8 +27,9 @@ const gcCommand = defineCommand({
|
|
|
27
27
|
},
|
|
28
28
|
async run({ args }) {
|
|
29
29
|
const { json } = applyOutputMode(args);
|
|
30
|
-
const
|
|
31
|
-
const
|
|
30
|
+
const ctx = await createCommandContext({ needsStore: true });
|
|
31
|
+
const store = ctx.store;
|
|
32
|
+
const siteId = args.site ? store.siteIdFor(await ctx.resolveSite(String(args.site), { scope: "store" })) : void 0;
|
|
32
33
|
const graceMs = Number(args["grace-hours"]) * 36e5;
|
|
33
34
|
if (args["dry-run"]) {
|
|
34
35
|
const cutoff = Date.now() - graceMs;
|