@gscdump/cli 3.8.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +113 -30
  2. package/bin/gscdump.mjs +21 -1
  3. package/dist/analysis-local.mjs +172 -53
  4. package/dist/auth-state.mjs +21 -6
  5. package/dist/auth.mjs +27 -18
  6. package/dist/bing-data.mjs +6 -2
  7. package/dist/bing-hosted.mjs +1 -1
  8. package/dist/cli-args.mjs +8 -0
  9. package/dist/cli.d.mts +5 -1
  10. package/dist/cli.mjs +41 -10
  11. package/dist/cloud-google.mjs +8 -1
  12. package/dist/command-meta.mjs +4 -4
  13. package/dist/command-registry.mjs +86 -10
  14. package/dist/commands/analyze.mjs +84 -43
  15. package/dist/commands/auth.mjs +46 -37
  16. package/dist/commands/bing.mjs +3 -3
  17. package/dist/commands/compact.mjs +12 -8
  18. package/dist/commands/config.mjs +14 -14
  19. package/dist/commands/doctor.mjs +26 -26
  20. package/dist/commands/dump.mjs +203 -105
  21. package/dist/commands/entities.mjs +10 -113
  22. package/dist/commands/gc.mjs +4 -3
  23. package/dist/commands/indexing-urls.mjs +185 -0
  24. package/dist/commands/indexing.mjs +12 -8
  25. package/dist/commands/init.mjs +77 -18
  26. package/dist/commands/inspect.mjs +97 -110
  27. package/dist/commands/mcp.mjs +37 -57
  28. package/dist/commands/papercut.mjs +1 -1
  29. package/dist/commands/profile.mjs +9 -17
  30. package/dist/commands/query.mjs +219 -210
  31. package/dist/commands/report.mjs +68 -85
  32. package/dist/commands/rollups.mjs +9 -6
  33. package/dist/commands/sitemaps.mjs +51 -104
  34. package/dist/commands/sites.mjs +40 -24
  35. package/dist/commands/stats.mjs +14 -17
  36. package/dist/commands/store-purge.mjs +9 -5
  37. package/dist/commands/store.mjs +0 -12
  38. package/dist/commands/sync.mjs +774 -296
  39. package/dist/config.mjs +7 -4
  40. package/dist/context.mjs +51 -21
  41. package/dist/coverage.mjs +251 -0
  42. package/dist/dump-bing.mjs +71 -0
  43. package/dist/dump-writers.mjs +246 -0
  44. package/dist/error-handler.mjs +108 -38
  45. package/dist/filters.mjs +125 -0
  46. package/dist/hosted-site.mjs +91 -0
  47. package/dist/inspect-urls.mjs +76 -0
  48. package/dist/inspection-record.mjs +93 -0
  49. package/dist/local-entities.mjs +658 -0
  50. package/dist/mcp/errors.mjs +71 -18
  51. package/dist/mcp/handlers/diagnostics.mjs +15 -12
  52. package/dist/mcp/handlers/reports.mjs +33 -42
  53. package/dist/mcp/server/index.mjs +80 -38
  54. package/dist/mcp/types.mjs +7 -7
  55. package/dist/package.mjs +1 -1
  56. package/dist/quota-ledger.mjs +370 -0
  57. package/dist/render/analysis.mjs +8 -1
  58. package/dist/render/report.mjs +8 -1
  59. package/dist/request-pacer.mjs +29 -0
  60. package/dist/route.mjs +295 -0
  61. package/dist/sitemap.mjs +4 -0
  62. package/dist/sql-views.mjs +117 -0
  63. package/dist/store-sites.mjs +134 -0
  64. package/dist/sync-plan.mjs +129 -0
  65. package/dist/sync-run.mjs +96 -0
  66. package/dist/table-sources.mjs +57 -0
  67. package/dist/token-info.mjs +54 -0
  68. package/dist/utils.mjs +28 -19
  69. package/dist/window.mjs +159 -0
  70. package/package.json +17 -14
  71. package/skills/gscdump/SKILL.md +167 -37
  72. package/dist/commands/export.mjs +0 -76
  73. package/dist/native-duckdb.mjs +0 -46
@@ -1,26 +1,28 @@
1
1
  import { dumpCommandMeta } from "../command-meta.mjs";
2
- import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger, parseSearchType, runWithConcurrency, toCSV } from "../utils.mjs";
2
+ import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger, parseNameList, parseSearchType } from "../utils.mjs";
3
3
  import { allTables } from "../local-store.mjs";
4
+ import { readSiteMap, siteUrlForId } from "../store-sites.mjs";
4
5
  import { createCommandContext } from "../context.mjs";
5
- import { readParquetRows } from "../native-duckdb.mjs";
6
+ import { ENTITY_DATASETS, readEntityDatasets } from "../local-entities.mjs";
7
+ import "../sync-plan.mjs";
8
+ import { nextCommand, readStoreCoverage, renderCoverage } from "../coverage.mjs";
9
+ import { isProcessAlive, readSyncRun, syncRunStatus } from "../sync-run.mjs";
10
+ import { dumpBing } from "../dump-bing.mjs";
11
+ import { groupTableSources } from "../table-sources.mjs";
12
+ import { DUMP_FORMATS, isDumpFormat, openDumpSink } from "../dump-writers.mjs";
6
13
  import process from "node:process";
7
14
  import { defineCommand } from "citty";
8
15
  import fs from "node:fs/promises";
9
16
  import path from "node:path";
17
+ import { Buffer } from "node:buffer";
10
18
  const DEFAULT_OUT = "./gscdump-export";
11
- const FORMATS = [
12
- "parquet",
13
- "json",
14
- "ndjson",
15
- "csv"
16
- ];
17
19
  const dumpCommand = defineCommand({
18
20
  meta: dumpCommandMeta,
19
21
  args: {
20
22
  "site": {
21
23
  type: "string",
22
24
  alias: "s",
23
- description: "Site URL (e.g., sc-domain:example.com); ignored with --all-sites"
25
+ description: "Site, for example example.com; ignored with --all-sites"
24
26
  },
25
27
  "out": {
26
28
  type: "string",
@@ -32,12 +34,16 @@ const dumpCommand = defineCommand({
32
34
  type: "string",
33
35
  alias: "F",
34
36
  default: "parquet",
35
- description: `Output format: ${FORMATS.join(", ")} (default: parquet copies raw files)`
37
+ description: `Output format: ${DUMP_FORMATS.join(", ")} (default: parquet). sqlite and duckdb write one database file`
36
38
  },
37
39
  "tables": {
38
40
  type: "string",
39
41
  alias: "t",
40
- description: `Comma-separated table list (default: all). Known: ${allTables().join(", ")}`
42
+ description: `Comma-separated tables and datasets (default: all). Known: ${[
43
+ ...allTables(),
44
+ ...ENTITY_DATASETS,
45
+ "bing"
46
+ ].join(", ")}`
41
47
  },
42
48
  "all-sites": {
43
49
  type: "boolean",
@@ -49,6 +55,12 @@ const dumpCommand = defineCommand({
49
55
  default: false,
50
56
  description: "Compact every closed month into a single file before exporting"
51
57
  },
58
+ "bing": {
59
+ type: "boolean",
60
+ default: true,
61
+ description: "Also export Bing data for matching sites when a Bing login exists",
62
+ negativeDescription: "Skip Bing"
63
+ },
52
64
  "search-type": {
53
65
  type: "string",
54
66
  description: `Restrict dump to a single GSC search-type slice (${ALL_SEARCH_TYPES.join(", ")}). Default: all slices.`
@@ -58,26 +70,28 @@ const dumpCommand = defineCommand({
58
70
  async run({ args }) {
59
71
  const { json, quiet } = applyOutputMode(args);
60
72
  const format = String(args.format);
61
- if (!FORMATS.includes(format)) {
62
- logger.error(`Invalid --format: ${format}. Allowed: ${FORMATS.join(", ")}`);
73
+ if (!isDumpFormat(format)) {
74
+ logger.error(`Invalid --format: ${format}. Allowed: ${DUMP_FORMATS.join(", ")}`);
63
75
  process.exit(1);
64
76
  }
65
- const tablesFilter = args.tables ? new Set(String(args.tables).split(",").map((t) => t.trim()).filter(Boolean)) : null;
77
+ const tablesFilter = args.tables ? new Set(parseNameList(args.tables, [
78
+ ...allTables(),
79
+ ...ENTITY_DATASETS,
80
+ "bing"
81
+ ], "--tables")) : null;
66
82
  const searchType = parseSearchType(args["search-type"]);
67
- const ctx = await createCommandContext({
68
- needsAuth: !args["all-sites"],
69
- needsStore: true
70
- });
83
+ const ctx = await createCommandContext({ needsStore: true });
71
84
  const store = ctx.store;
72
85
  const outDir = path.resolve(String(args.out));
86
+ const siteMap = await readSiteMap(store.dataDir, store.userId);
73
87
  let preloadedEntries = args["all-sites"] ? await store.engine.listLive({
74
88
  userId: store.userId,
75
89
  ...searchType !== void 0 ? { searchType } : {}
76
90
  }) : void 0;
77
91
  const targets = args["all-sites"] ? [...new Set(preloadedEntries.flatMap((entry) => entry.siteId ? [entry.siteId] : []))].map((siteId) => ({
78
- site: siteId,
92
+ site: siteUrlForId(siteMap, siteId),
79
93
  siteId
80
- })) : await ctx.resolveSite(args.site ? String(args.site) : void 0).then((site) => [{
94
+ })) : await ctx.resolveSite(args.site ? String(args.site) : void 0, { scope: "store" }).then((site) => [{
81
95
  site,
82
96
  siteId: store.siteIdFor(site)
83
97
  }]);
@@ -92,105 +106,189 @@ const dumpCommand = defineCommand({
92
106
  ...searchType !== void 0 ? { searchType } : {}
93
107
  });
94
108
  }
95
- const summary = [];
96
- for (const target of targets) {
97
- const entries = (preloadedEntries ? preloadedEntries.filter((entry) => entry.siteId === target.siteId) : await listLiveEntries(store, target.siteId, searchType)).filter((e) => !tablesFilter || tablesFilter.has(e.table));
98
- if (entries.length === 0) {
99
- if (!quiet) logger.warn(`No data for ${target.site}; skipping`);
100
- continue;
101
- }
102
- if (format === "parquet") {
103
- const written = await dumpParquet(store, entries, outDir);
104
- summary.push({
105
- site: target.site,
106
- files: written,
107
- rows: 0,
108
- format,
109
- outPath: outDir
110
- });
111
- } else {
112
- const written = await dumpRowFormat(store, entries, outDir, target.site, format);
113
- summary.push({
114
- site: target.site,
115
- files: written.files,
116
- rows: written.rows,
117
- format,
118
- outPath: outDir
119
- });
120
- }
121
- }
109
+ const bing = args.bing === false || tablesFilter && !tablesFilter.has("bing") ? { _tag: "disabled" } : await dumpBing({
110
+ googleSites: args["all-sites"] ? "all" : targets.map((target) => target.site),
111
+ outDir,
112
+ format
113
+ }).catch((error) => ({
114
+ _tag: "failed",
115
+ reason: error instanceof Error ? error.message : String(error)
116
+ }));
117
+ const result = await dumpSites({
118
+ store,
119
+ targets,
120
+ outDir,
121
+ format,
122
+ ...tablesFilter ? { tables: tablesFilter } : {},
123
+ ...searchType !== void 0 ? { searchType } : {},
124
+ ...preloadedEntries ? { entries: preloadedEntries } : {},
125
+ bing
126
+ });
122
127
  if (json) {
123
128
  console.log(JSON.stringify({
124
- outDir,
125
- sites: summary
129
+ ...result,
130
+ bing
126
131
  }, null, 2));
127
132
  return;
128
133
  }
129
- for (const s of summary) {
130
- const rows = s.rows ? `, ${s.rows.toLocaleString()} rows` : "";
131
- logger.success(`[${s.site}] ${s.files} ${s.format} file(s)${rows} → ${displayPath(s.outPath)}`);
134
+ const bytesByPath = new Map(result.files.map((file) => [file.path, file.bytes]));
135
+ const database = format === "sqlite" || format === "duckdb";
136
+ if (database && result.sites.some((site) => site.datasets.length > 0)) {
137
+ const file = result.files[0];
138
+ logger.success(`Wrote ${displayPath(file.path)} (${formatBytes(file.bytes)})`);
139
+ }
140
+ for (const site of result.sites) {
141
+ if (site.datasets.length === 0) {
142
+ if (!quiet) logger.warn(`No data for ${site.site}; skipping`);
143
+ continue;
144
+ }
145
+ logger.success(`[${site.site}] ${site.totals.datasets} dataset(s), ${site.totals.rows.toLocaleString()} rows`);
146
+ if (quiet) continue;
147
+ for (const dataset of site.datasets) {
148
+ const label = database ? `${dataset.dataset}${dataset.searchType ? ` (search_type = ${dataset.searchType})` : ""}` : `${path.relative(outDir, dataset.path)} ${formatBytes(bytesByPath.get(dataset.path) ?? 0)}`;
149
+ console.log(` ${label} ${dataset.rows.toLocaleString()} rows`);
150
+ }
151
+ for (const skip of site.skipped) console.log(` \x1B[90m${skip.dataset}: no rows yet; skipped\x1B[0m`);
152
+ for (const line of renderCoverage(site.coverage)) console.log(` ${line}`);
153
+ for (const job of site.coverage.analytics) if (job.coverage.kind === "partial" && job.coverage.failed > 0) logger.warn(`${site.site} ${job.table} (${job.searchType}): ${job.coverage.failed} failed date(s). Run ${nextCommand(site.coverage)} to retry them.`);
132
154
  }
155
+ if (bing._tag === "dumped") {
156
+ for (const site of bing.sites) {
157
+ const bytes = site.files.reduce((sum, file) => sum + file.bytes, 0);
158
+ logger.success(`[Bing ${site.siteUrl}] ${site.files.length} file(s), ${formatBytes(bytes)}`);
159
+ if (!quiet) for (const file of site.files) console.log(` ${path.relative(outDir, file.path)} ${formatBytes(file.bytes)} ${file.rows.toLocaleString()} rows`);
160
+ }
161
+ for (const failure of bing.failures) logger.warn(`Bing ${failure.siteUrl} not exported: ${failure.error}`);
162
+ } else if (bing._tag === "failed") logger.warn(`Bing not exported: ${bing.reason}`);
163
+ else if (bing._tag === "skipped" && !quiet) logger.info(`Bing skipped: ${bing.reason}`);
164
+ if (!quiet) for (const file of result.metadataFiles) console.log(` ${path.relative(outDir, file.path)} ${formatBytes(file.bytes)}`);
133
165
  }
134
166
  });
135
- async function listLiveEntries(store, siteId, searchType) {
136
- return store.engine.listLive({
137
- userId: store.userId,
138
- siteId,
139
- ...searchType !== void 0 ? { searchType } : {}
140
- });
141
- }
142
- async function dumpParquet(store, entries, outDir) {
143
- await fs.mkdir(outDir, { recursive: true });
144
- const readyDirectories = /* @__PURE__ */ new Map();
145
- async function ensureDirectory(dir) {
146
- let ready = readyDirectories.get(dir);
147
- if (!ready) {
148
- ready = fs.mkdir(dir, { recursive: true }).then(() => void 0);
149
- readyDirectories.set(dir, ready);
150
- ready.catch(() => readyDirectories.delete(dir));
151
- }
152
- await ready;
167
+ function formatBytes(bytes) {
168
+ if (bytes < 1024) return `${bytes} B`;
169
+ const units = [
170
+ "KB",
171
+ "MB",
172
+ "GB",
173
+ "TB"
174
+ ];
175
+ let value = bytes / 1024;
176
+ let unit = 0;
177
+ while (value >= 1024 && unit < units.length - 1) {
178
+ value /= 1024;
179
+ unit++;
153
180
  }
154
- let copied = 0;
155
- await runWithConcurrency(entries, 8, async (entry) => {
156
- const bytes = await store.engine.readObject(entry.objectKey);
157
- const target = path.join(outDir, entry.objectKey);
158
- await ensureDirectory(path.dirname(target));
159
- await fs.writeFile(target, bytes);
160
- copied++;
181
+ return `${value.toFixed(value >= 100 ? 0 : 1)} ${units[unit]}`;
182
+ }
183
+ async function dumpSites(opts) {
184
+ const { outDir, format } = opts;
185
+ const sink = await openDumpSink(outDir, format);
186
+ const summary = await dumpEachSite(sink, opts).catch(async (error) => {
187
+ await sink.close();
188
+ throw error;
161
189
  });
162
- return copied;
190
+ const files = await sink.close();
191
+ const metadataFiles = [];
192
+ const sites = opts.targets.map((target) => ({
193
+ siteUrl: target.site,
194
+ siteId: target.siteId
195
+ }));
196
+ metadataFiles.push(await writeJsonFile(path.join(outDir, "sites.json"), { sites }));
197
+ metadataFiles.push(await writeJsonFile(path.join(outDir, "manifest.json"), {
198
+ generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
199
+ format,
200
+ files: files.map((file) => ({
201
+ ...file,
202
+ path: path.relative(outDir, file.path)
203
+ })),
204
+ sites: summary.map((site) => ({
205
+ ...site,
206
+ datasets: site.datasets.map((dataset) => ({
207
+ ...dataset,
208
+ path: path.relative(outDir, dataset.path)
209
+ }))
210
+ })),
211
+ ...opts.bing ? { bing: manifestBing(opts.bing, outDir) } : {}
212
+ }));
213
+ return {
214
+ outDir,
215
+ format,
216
+ files,
217
+ metadataFiles,
218
+ sites: summary
219
+ };
163
220
  }
164
- async function dumpRowFormat(store, entries, outDir, siteUrl, format) {
165
- const byTable = /* @__PURE__ */ new Map();
166
- for (const e of entries) {
167
- const arr = byTable.get(e.table) ?? [];
168
- arr.push(e);
169
- byTable.set(e.table, arr);
170
- }
171
- const safeSite = siteUrl.replace(/[^a-z0-9]+/gi, "_");
172
- const siteDir = path.join(outDir, safeSite);
173
- await fs.mkdir(siteDir, { recursive: true });
174
- let files = 0;
175
- let totalRows = 0;
176
- for (const [table, tableEntries] of byTable) {
177
- const filePaths = tableEntries.map((e) => path.join(store.dataDir, e.objectKey));
178
- const rows = await readParquetRows(filePaths, table);
179
- const ext = format === "csv" ? "csv" : format === "ndjson" ? "ndjson" : "json";
180
- const target = path.join(siteDir, `${table}.${ext}`);
181
- let body;
182
- if (format === "json") body = JSON.stringify(rows, null, 2);
183
- else if (format === "ndjson") body = rows.map((r) => JSON.stringify(r)).join("\n");
184
- else body = rows.length > 0 ? toCSV(rows, Object.keys(rows[0])) : "";
185
- await fs.writeFile(target, body);
186
- files++;
187
- totalRows += rows.length;
221
+ async function dumpEachSite(sink, opts) {
222
+ const { store, tables } = opts;
223
+ const wantedEntities = ENTITY_DATASETS.filter((dataset) => !tables || tables.has(dataset));
224
+ const summary = [];
225
+ for (const target of opts.targets) {
226
+ const entries = (opts.entries ? opts.entries.filter((entry) => entry.siteId === target.siteId) : await listLiveEntries(store, target.siteId, opts.searchType)).filter((e) => !tables || tables.has(e.table));
227
+ const datasets = [];
228
+ for (const source of groupTableSources(entries, store.dataDir, { [target.siteId]: target.site })) datasets.push(await sink.writeTable(source));
229
+ const skipped = [];
230
+ for (const dataset of await readEntityDatasets(store.dataSource, {
231
+ userId: store.userId,
232
+ siteId: target.siteId
233
+ }, wantedEntities)) if (dataset.rows.length === 0) skipped.push({
234
+ dataset: dataset.dataset,
235
+ reason: "empty"
236
+ });
237
+ else datasets.push(await sink.writeEntity(target.site, dataset));
238
+ summary.push({
239
+ site: target.site,
240
+ siteId: target.siteId,
241
+ coverage: await readDumpCoverage(store, target.site),
242
+ datasets,
243
+ skipped,
244
+ totals: {
245
+ datasets: datasets.length,
246
+ rows: datasets.reduce((sum, dataset) => sum + dataset.rows, 0)
247
+ }
248
+ });
188
249
  }
250
+ return summary;
251
+ }
252
+ function manifestBing(step, outDir) {
253
+ if (step._tag !== "dumped") return step;
189
254
  return {
190
- files,
191
- rows: totalRows
255
+ ...step,
256
+ sites: step.sites.map((site) => ({
257
+ siteUrl: site.siteUrl,
258
+ files: site.files.map((file) => ({
259
+ ...file,
260
+ path: path.relative(outDir, file.path)
261
+ }))
262
+ }))
263
+ };
264
+ }
265
+ async function writeJsonFile(target, value) {
266
+ const body = `${JSON.stringify(value, null, 2)}\n`;
267
+ await fs.writeFile(target, body);
268
+ return {
269
+ path: target,
270
+ bytes: Buffer.byteLength(body)
192
271
  };
193
272
  }
273
+ async function readDumpCoverage(store, site) {
274
+ const run = syncRunStatus(await readSyncRun(store.dataDir), {
275
+ now: Date.now(),
276
+ isAlive: isProcessAlive
277
+ });
278
+ return readStoreCoverage({
279
+ store,
280
+ site,
281
+ inspectLimit: 50,
282
+ run
283
+ });
284
+ }
285
+ async function listLiveEntries(store, siteId, searchType) {
286
+ return store.engine.listLive({
287
+ userId: store.userId,
288
+ siteId,
289
+ ...searchType !== void 0 ? { searchType } : {}
290
+ });
291
+ }
194
292
  async function compactClosedMonths(store, siteId, quiet) {
195
293
  for (const table of allTables()) {
196
294
  if (!quiet) logger.info(`Compacting ${table} (raw→d7→d30→d90)`);
@@ -201,4 +299,4 @@ async function compactClosedMonths(store, siteId, quiet) {
201
299
  });
202
300
  }
203
301
  }
204
- export { dumpCommand };
302
+ export { dumpCommand, dumpSites, formatBytes };
@@ -1,12 +1,12 @@
1
1
  import { entitiesCommandMeta } from "../command-meta.mjs";
2
2
  import { OUTPUT_ARGS, applyOutputMode, logger, parseIntegerOption, progressBar, runWithConcurrency } from "../utils.mjs";
3
+ import { formatSiteIdCollision, recordStoreSite } from "../store-sites.mjs";
3
4
  import { createCommandContext } from "../context.mjs";
4
5
  import process from "node:process";
5
6
  import { defineCommand } from "citty";
6
7
  import { readFile } from "node:fs/promises";
7
8
  import { Buffer } from "node:buffer";
8
9
  import { createIndexingMetadataStore, createInspectionStore } from "@gscdump/engine/entities";
9
- const INSPECTION_QPD_PER_PROPERTY = 2e3;
10
10
  const INDEXING_NOT_FOUND_RE = /\b404\b|NOT_FOUND/i;
11
11
  const INSPECTION_HISTORY_LOOKBACK_MONTHS = 24;
12
12
  async function findLatestInspection(inspector, ctx, url) {
@@ -28,110 +28,6 @@ async function readUrlList(opts) {
28
28
  for await (const chunk of process.stdin) chunks.push(chunk);
29
29
  return Buffer.concat(chunks).toString("utf8").split("\n").map((l) => l.trim()).filter(Boolean);
30
30
  }
31
- const inspectSubCommand = defineCommand({
32
- meta: {
33
- name: "inspect",
34
- description: "Run URL Inspection for a list of URLs and persist results to the local entity store"
35
- },
36
- args: {
37
- site: {
38
- type: "string",
39
- alias: "s",
40
- description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
41
- },
42
- file: {
43
- type: "string",
44
- alias: "f",
45
- description: "Path to a file with one URL per line. If omitted, reads from stdin."
46
- },
47
- limit: {
48
- type: "string",
49
- description: `Max URLs to inspect this run (default: ${INSPECTION_QPD_PER_PROPERTY}, the per-property GSC daily quota)`
50
- },
51
- concurrency: {
52
- type: "string",
53
- alias: "c",
54
- default: "4",
55
- description: "Concurrent in-flight inspect calls (default: 4)"
56
- },
57
- ...OUTPUT_ARGS
58
- },
59
- async run({ args }) {
60
- const { json, quiet } = applyOutputMode(args);
61
- const limit = parseIntegerOption(args.limit, "--limit") ?? INSPECTION_QPD_PER_PROPERTY;
62
- const concurrency = parseIntegerOption(args.concurrency, "--concurrency") ?? 4;
63
- const ctx = await createCommandContext({
64
- needsAuth: true,
65
- needsStore: true
66
- });
67
- const client = ctx.client;
68
- const store = ctx.store;
69
- const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
70
- const urls = (await readUrlList({ file: args.file ? String(args.file) : void 0 })).slice(0, limit);
71
- if (urls.length === 0) {
72
- logger.warn("No URLs to inspect.");
73
- return;
74
- }
75
- if (urls.length === limit && limit < INSPECTION_QPD_PER_PROPERTY) logger.info(`Capping at --limit ${limit}`);
76
- if (urls.length === INSPECTION_QPD_PER_PROPERTY) logger.info(`Hit per-property daily inspection quota (${INSPECTION_QPD_PER_PROPERTY}); remaining URLs will be queued for tomorrow.`);
77
- const inspector = createInspectionStore({ dataSource: store.dataSource });
78
- let completed = 0;
79
- let failed = 0;
80
- const records = [];
81
- const failures = [];
82
- await runWithConcurrency(urls, concurrency, async (url) => {
83
- const result = await client.inspect(siteUrl, url).catch((err) => err);
84
- if (result instanceof Error) {
85
- failed++;
86
- failures.push({
87
- url,
88
- error: result.message
89
- });
90
- } else {
91
- const ix = result.inspectionResult;
92
- const indexStatus = ix?.indexStatusResult;
93
- records.push({
94
- url,
95
- inspectedAt: (/* @__PURE__ */ new Date()).toISOString(),
96
- indexStatus: indexStatus?.verdict ?? void 0,
97
- lastCrawlTime: indexStatus?.lastCrawlTime ?? void 0,
98
- googleCanonical: indexStatus?.googleCanonical ?? void 0,
99
- userCanonical: indexStatus?.userCanonical ?? void 0,
100
- coverageState: indexStatus?.coverageState ?? void 0,
101
- robotsTxtState: indexStatus?.robotsTxtState ?? void 0,
102
- indexingState: indexStatus?.indexingState ?? void 0,
103
- pageFetchState: indexStatus?.pageFetchState ?? void 0,
104
- mobileUsabilityVerdict: ix?.mobileUsabilityResult?.verdict ?? void 0,
105
- richResultsVerdict: ix?.richResultsResult?.verdict ?? void 0,
106
- raw: ix
107
- });
108
- }
109
- completed++;
110
- if (!quiet) process.stdout.write(`\r${progressBar(completed, urls.length, `${url.slice(0, 60)}`)}`);
111
- });
112
- if (!quiet) process.stdout.write("\n");
113
- await inspector.appendHistory({
114
- userId: store.userId,
115
- siteId: store.siteIdFor(siteUrl)
116
- }, records);
117
- if (json) console.log(JSON.stringify({
118
- site: siteUrl,
119
- inspected: records.length,
120
- failed,
121
- failures,
122
- records
123
- }, null, 2));
124
- else if (!quiet) {
125
- logger.success(`Inspected ${records.length}/${urls.length} URL(s)`);
126
- if (failed > 0) {
127
- logger.warn(`${failed} failed:`);
128
- for (const f of failures.slice(0, 5)) console.log(` ${f.url}: ${f.error}`);
129
- if (failures.length > 5) console.log(` ... and ${failures.length - 5} more`);
130
- }
131
- }
132
- if (failed > 0) process.exit(1);
133
- }
134
- });
135
31
  const showSubCommand = defineCommand({
136
32
  meta: {
137
33
  name: "show",
@@ -142,7 +38,7 @@ const showSubCommand = defineCommand({
142
38
  site: {
143
39
  type: "string",
144
40
  alias: "s",
145
- description: "Site URL (defaults to config.defaultSite or prompt)"
41
+ description: "Site, for example example.com; defaults to config.defaultSite or a prompt"
146
42
  },
147
43
  url: {
148
44
  type: "positional",
@@ -152,12 +48,9 @@ const showSubCommand = defineCommand({
152
48
  },
153
49
  async run({ args }) {
154
50
  const { json } = applyOutputMode(args);
155
- const ctx = await createCommandContext({
156
- needsAuth: true,
157
- needsStore: true
158
- });
51
+ const ctx = await createCommandContext({ needsStore: true });
159
52
  const store = ctx.store;
160
- const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
53
+ const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0, { scope: "store" });
161
54
  const record = await findLatestInspection(createInspectionStore({ dataSource: store.dataSource }), {
162
55
  userId: store.userId,
163
56
  siteId: store.siteIdFor(siteUrl)
@@ -190,7 +83,7 @@ const indexingSnapshotSubCommand = defineCommand({
190
83
  site: {
191
84
  type: "string",
192
85
  alias: "s",
193
- description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
86
+ description: "Site, for example example.com; defaults to config.defaultSite or a prompt"
194
87
  },
195
88
  file: {
196
89
  type: "string",
@@ -220,6 +113,11 @@ const indexingSnapshotSubCommand = defineCommand({
220
113
  logger.warn("No URLs to fetch metadata for.");
221
114
  return;
222
115
  }
116
+ const claim = await recordStoreSite(store.dataDir, siteUrl, { userId: store.userId });
117
+ if (!claim.ok) {
118
+ logger.error(formatSiteIdCollision(claim.error));
119
+ process.exit(1);
120
+ }
223
121
  const records = [];
224
122
  const failures = [];
225
123
  let completed = 0;
@@ -270,7 +168,6 @@ const indexingSubCommand = defineCommand({
270
168
  const entitiesCommand = defineCommand({
271
169
  meta: entitiesCommandMeta,
272
170
  subCommands: {
273
- inspect: inspectSubCommand,
274
171
  show: showSubCommand,
275
172
  indexing: indexingSubCommand
276
173
  }
@@ -16,7 +16,7 @@ const gcCommand = defineCommand({
16
16
  "site": {
17
17
  type: "string",
18
18
  alias: "s",
19
- description: "Restrict to a single site (default: all sites)"
19
+ description: "Restrict to one Site, for example example.com (default: every Site)"
20
20
  },
21
21
  "dry-run": {
22
22
  type: "boolean",
@@ -27,8 +27,9 @@ const gcCommand = defineCommand({
27
27
  },
28
28
  async run({ args }) {
29
29
  const { json } = applyOutputMode(args);
30
- const store = (await createCommandContext({ needsStore: true })).store;
31
- const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
30
+ const ctx = await createCommandContext({ needsStore: true });
31
+ const store = ctx.store;
32
+ const siteId = args.site ? store.siteIdFor(await ctx.resolveSite(String(args.site), { scope: "store" })) : void 0;
32
33
  const graceMs = Number(args["grace-hours"]) * 36e5;
33
34
  if (args["dry-run"]) {
34
35
  const cutoff = Date.now() - graceMs;