@gscdump/cli 6.0.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,7 @@ const HOSTED_SESSION_REJECTED = "gscdump.com rejected the CLI session. Run `gscd
9
9
  const HOSTED_KEY_REJECTED = "gscdump.com rejected the API key. Run `gscdump auth login --mode hosted --api-key KEY` with a valid key.";
10
10
  const LOCAL_MODE_REQUIRED = [
11
11
  "This command calls Google, so it needs Local mode.",
12
- "Hosted mode reads your gscdump.com record and never calls Google.",
12
+ "This command needs your own Google credentials. Hosted inspections use the platform separately.",
13
13
  "If you want Local mode, run `gscdump auth login --mode local`."
14
14
  ].join("\n");
15
15
  function localModeRequired(detail) {
@@ -28,7 +28,7 @@ const storeCommandMeta = {
28
28
  };
29
29
  const inspectCommandMeta = {
30
30
  name: "inspect",
31
- description: "Inspect the indexing status of one or more URLs and save the results to the Store"
31
+ description: "Read Google URL Inspection results and save them to the Store"
32
32
  };
33
33
  const indexingCommandMeta = {
34
34
  name: "indexing",
@@ -36,7 +36,7 @@ const indexingCommandMeta = {
36
36
  };
37
37
  const entitiesCommandMeta = {
38
38
  name: "entities",
39
- description: "Manage local entity snapshots (URL inspections and indexing metadata)"
39
+ description: "Read saved URL inspections and save Google Indexing API metadata"
40
40
  };
41
41
  const analyzeCommandMeta = {
42
42
  name: "analyze",
@@ -44,7 +44,7 @@ const analyzeCommandMeta = {
44
44
  };
45
45
  const reportCommandMeta = {
46
46
  name: "report",
47
- description: "Run an intent-keyed report (composes analyzers into bounded sections)"
47
+ description: "Run a Report to find traffic changes, SEO opportunities, and risks"
48
48
  };
49
49
  const authCommandMeta = {
50
50
  status: {
@@ -3,7 +3,7 @@ import { terminalOutputOptions } from "../render/terminal.mjs";
3
3
  import { logger, parseFetchBudget, parseIntegerOption, setJsonOutput, toCSV } from "../utils.mjs";
4
4
  import { DEFAULT_WINDOW, PERIOD_FLAGS, parseWindowFlags, windowFlagErrorToException } from "../window.mjs";
5
5
  import { analysisNeeds, analyzerTables, resolveAnalysisSource } from "../analysis-local.mjs";
6
- import { coverageWarning, renderAnalysis } from "../render/analysis.mjs";
6
+ import { ANALYZER_DISPLAY, coverageWarning, renderAnalysis } from "../render/analysis.mjs";
7
7
  import { defineCommand } from "citty";
8
8
  import { DEFAULT_FETCH_BUDGET, MAX_FETCH_BUDGET } from "@gscdump/engine/analysis-types";
9
9
  import { unwrapResult } from "gscdump/result";
@@ -120,7 +120,7 @@ function makeToolCommand(tool) {
120
120
  return defineCommand({
121
121
  meta: {
122
122
  name: tool,
123
- description: `Run ${tool} analysis`
123
+ description: ANALYZER_DISPLAY[tool]?.description ?? `Run ${tool} analysis`
124
124
  },
125
125
  args: {
126
126
  "site": {
@@ -228,7 +228,7 @@ const listCommand = defineCommand({
228
228
  console.log(JSON.stringify(ANALYSIS_TOOLS, null, 2));
229
229
  return;
230
230
  }
231
- for (const id of ANALYSIS_TOOLS) console.log(id);
231
+ for (const id of ANALYSIS_TOOLS) console.log(ANALYZER_DISPLAY[id] ? `${id}: ${ANALYZER_DISPLAY[id].description}` : id);
232
232
  }
233
233
  });
234
234
  const analyzeCommand = defineCommand({
@@ -31,7 +31,7 @@ async function readUrlList(opts) {
31
31
  const showSubCommand = defineCommand({
32
32
  meta: {
33
33
  name: "show",
34
- description: "Print the latest inspection record for a URL from the local entity store"
34
+ description: "Read the latest saved URL inspection from the Store"
35
35
  },
36
36
  args: {
37
37
  ...OUTPUT_ARGS,
@@ -74,7 +74,7 @@ const showSubCommand = defineCommand({
74
74
  const indexingSnapshotSubCommand = defineCommand({
75
75
  meta: {
76
76
  name: "snapshot",
77
- description: "Fetch Indexing API metadata (latest update/remove per URL) and persist to the local entity store"
77
+ description: "Save the latest Google Indexing API update and removal notification times for each URL to the Store"
78
78
  },
79
79
  args: {
80
80
  site: {
@@ -155,7 +155,7 @@ const indexingSnapshotSubCommand = defineCommand({
155
155
  const indexingSubCommand = defineCommand({
156
156
  meta: {
157
157
  name: "indexing",
158
- description: "Snapshot Indexing API metadata per URL"
158
+ description: "Save Google Indexing API notification times for URLs"
159
159
  },
160
160
  subCommands: { snapshot: indexingSnapshotSubCommand }
161
161
  });
@@ -5,13 +5,13 @@ const DEFAULT_GRACE_HOURS = 24;
5
5
  const gcCommand = defineCommand({
6
6
  meta: {
7
7
  name: "gc",
8
- description: "Delete orphaned object-store files not referenced by any manifest entry"
8
+ description: "Delete retired and unreferenced files; expire hourly files created over 90 days ago"
9
9
  },
10
10
  args: {
11
11
  "grace-hours": {
12
12
  type: "string",
13
13
  default: String(DEFAULT_GRACE_HOURS),
14
- description: `Spare orphans younger than this (default: ${DEFAULT_GRACE_HOURS}h)`
14
+ description: `Grace period in hours for retired and unreferenced files (default: ${DEFAULT_GRACE_HOURS}h)`
15
15
  },
16
16
  "site": {
17
17
  type: "string",
@@ -21,7 +21,7 @@ const gcCommand = defineCommand({
21
21
  "dry-run": {
22
22
  type: "boolean",
23
23
  default: false,
24
- description: "List retired manifest entries past the grace window without deleting"
24
+ description: "Preview retired entries only; excludes expired hourly files and unreferenced files"
25
25
  },
26
26
  ...OUTPUT_ARGS
27
27
  },
@@ -55,7 +55,8 @@ const gcCommand = defineCommand({
55
55
  console.log();
56
56
  for (const c of candidates) console.log(` ${c.objectKey} \x1B[90m(retired ${new Date(c.retiredAt).toISOString()})\x1B[0m`);
57
57
  console.log();
58
- logger.info(`gc --dry-run: ${candidates.length} retired manifest entry(ies) past ${args["grace-hours"]}h grace; pass without --dry-run to delete`);
58
+ logger.info(`gc --dry-run: ${candidates.length} retired manifest entry(ies) past ${args["grace-hours"]}h grace.`);
59
+ logger.warn("This preview excludes expired hourly files and unreferenced files. The deletion run also removes those files.");
59
60
  return;
60
61
  }
61
62
  const result = await store.engine.gcOrphans({
@@ -69,7 +70,7 @@ const gcCommand = defineCommand({
69
70
  }, null, 2));
70
71
  return;
71
72
  }
72
- logger.success(`gc: deleted ${result.deleted} orphan file(s)`);
73
+ logger.success(`gc: deleted ${result.deleted} retired, expired hourly, or unreferenced file(s)`);
73
74
  }
74
75
  });
75
76
  export { gcCommand };
@@ -30,7 +30,7 @@ function captureLines(capture) {
30
30
  unrecorded: "an unrecorded scope"
31
31
  }[capture.scope];
32
32
  const lines = [
33
- `Counted ${capture.capturedAt} from stored URL Inspection verdicts (${capture.source}).`,
33
+ `${capture.source === "live" ? "Counted during this request" : "Saved daily count"} at ${capture.capturedAt}, from stored URL Inspection verdicts.`,
34
34
  `The counts cover ${scope}.`,
35
35
  `Verdicts inspected between ${capture.oldestVerdictAt ?? "unknown"} and ${capture.newestVerdictAt ?? "unknown"}.`
36
36
  ];
@@ -95,12 +95,12 @@ const indexingSummaryCommand = defineCommand({
95
95
  },
96
96
  {
97
97
  key: "discovered_not_indexed",
98
- label: "Discovered",
98
+ label: "Discovered, not indexed",
99
99
  numeric: true
100
100
  },
101
101
  {
102
102
  key: "crawled_not_indexed",
103
- label: "Crawled",
103
+ label: "Crawled, not indexed",
104
104
  numeric: true
105
105
  },
106
106
  {
@@ -41,7 +41,7 @@ function watchedRow(entry) {
41
41
  };
42
42
  }
43
43
  function skipHint(reason, manager) {
44
- return reason === "inspection_disabled" ? `gscdump never inspects this Site, so a Watched URL gets no Checkpoint. To use Watched URLs, turn on URL Inspection ${siteSettingsPlace(manager)}.` : void 0;
44
+ return reason === "inspection_disabled" ? `gscdump never inspects this Site, so a Watched URL gets no Watched URL inspection. To use Watched URLs, turn on URL Inspection ${siteSettingsPlace(manager)}.` : void 0;
45
45
  }
46
46
  function printChange(verb, result, siteUrl, manager) {
47
47
  for (const url of result.changed) console.log(`${verb}: ${url}`);
@@ -81,7 +81,7 @@ async function changeInChunks(urls, send) {
81
81
  const listCommand = defineCommand({
82
82
  meta: {
83
83
  name: "list",
84
- description: "List Watched URLs with the latest Checkpoint of each (hosted)"
84
+ description: "List Watched URLs with the latest Watched URL inspection of each (hosted)"
85
85
  },
86
86
  args: {
87
87
  ...HOSTED_ARGS,
@@ -157,7 +157,7 @@ const addCommand = defineCommand({
157
157
  const removeCommand = defineCommand({
158
158
  meta: {
159
159
  name: "remove",
160
- description: "Remove Watched URLs and their Checkpoints (hosted)"
160
+ description: "Remove Watched URLs and their Watched URL inspections (hosted)"
161
161
  },
162
162
  args: {
163
163
  ...HOSTED_ARGS,
@@ -80,7 +80,7 @@ const removeCommand = defineCommand({
80
80
  console.log(JSON.stringify(result, null, 2));
81
81
  return;
82
82
  }
83
- logger.success(`Removed: ${result.url}`);
83
+ logger.success(`Removal notification accepted: ${result.url}`);
84
84
  if (result.notifyTime) console.log(` Notified: ${result.notifyTime}`);
85
85
  }
86
86
  });
@@ -262,7 +262,7 @@ function createGscMcpServer(options) {
262
262
  }] };
263
263
  });
264
264
  registerTool("request-indexing", {
265
- description: "Request Google to index or remove a URL via the Indexing API",
265
+ description: "Send a URL update or removal notification to Google. Acceptance does not confirm indexing or removal.",
266
266
  inputSchema: requestIndexingInput.shape
267
267
  }, async (args) => {
268
268
  const result = await requestIndexing$1(args, await getContext());
@@ -272,7 +272,7 @@ function createGscMcpServer(options) {
272
272
  }] };
273
273
  });
274
274
  registerTool("get-indexing-status", {
275
- description: "Get indexing status metadata for a URL",
275
+ description: "Read the latest Google Indexing API notification times for a URL. Use inspect-url for URL Inspection results.",
276
276
  inputSchema: getIndexingStatusInput.shape
277
277
  }, async (args) => {
278
278
  const result = await getIndexingStatus(args, await getContext());
@@ -282,7 +282,7 @@ function createGscMcpServer(options) {
282
282
  }] };
283
283
  });
284
284
  registerTool("batch-request-indexing", {
285
- description: "Batch request indexing for multiple URLs with rate limiting",
285
+ description: "Send Google Indexing API notifications for multiple URLs with rate limiting. Acceptance does not confirm indexing.",
286
286
  inputSchema: batchRequestIndexingInput.shape
287
287
  }, async (args) => {
288
288
  const result = await batchRequestIndexing$1(args, await getContext());
package/dist/package.mjs CHANGED
@@ -1,2 +1,2 @@
1
- var version = "6.0.0";
1
+ var version = "7.0.0";
2
2
  export { version };
@@ -3,6 +3,28 @@ import { finite, formatMetric, metricLabel } from "./metrics.mjs";
3
3
  import { barColumn, calendarBuckets, renderBars, renderShare, renderSparklines } from "./charts.mjs";
4
4
  import { MAX_FETCH_BUDGET } from "@gscdump/engine/analysis-types";
5
5
  import { parseGscSiteUrl } from "gscdump";
6
+ const ANALYZER_DISPLAY = {
7
+ "zero-click": {
8
+ title: "Low-CTR queries",
9
+ description: "Find low-CTR search queries with high impressions and strong positions; clicks can be nonzero"
10
+ },
11
+ "content-velocity": {
12
+ title: "Newly observed search queries",
13
+ description: "Count search queries first observed within the selected window, by week"
14
+ },
15
+ "intent-atlas": {
16
+ title: "Shared-word query groups",
17
+ description: "Group search queries by shared words, without classifying Search Intent"
18
+ },
19
+ "dark-traffic": {
20
+ title: "Clicks without reported queries",
21
+ description: "Compare page clicks with clicks attributed to reported search queries"
22
+ },
23
+ "keyword-breadth": {
24
+ title: "Search queries per page",
25
+ description: "Count distinct reported search queries with impressions per page"
26
+ }
27
+ };
6
28
  function asRecord(value) {
7
29
  return value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
8
30
  }
@@ -56,7 +78,7 @@ function renderAnalysis(result, context, options) {
56
78
  const meta = result.meta;
57
79
  const start = typeof meta.startDate === "string" ? meta.startDate : context.start;
58
80
  const end = typeof meta.endDate === "string" ? meta.endDate : context.end;
59
- const lines = [...textLines(`${parseGscSiteUrl(context.site).hostname} / ${context.id}`, options, "accent"), ...textLines(`${start} to ${end}`, options, "muted")];
81
+ const lines = [...textLines(`${parseGscSiteUrl(context.site).hostname} / ${ANALYZER_DISPLAY[context.id]?.title ?? context.id}`, options, "accent"), ...textLines(`${start} to ${end}`, options, "muted")];
60
82
  if (context.previous) lines.push(...textLines(`vs ${context.previous.start} to ${context.previous.end}`, options, "muted"));
61
83
  const warning = coverageWarning(meta.coverage);
62
84
  if (warning) lines.push(...textLines(warning, options, "warning"));
@@ -125,4 +147,4 @@ function renderAnalysis(result, context, options) {
125
147
  if (total !== null && total > rows.length) lines.push("", ...textLines(`${rows.length} of ${formatMetric("clicks", total)} rows`, options, "muted"));
126
148
  return lines.join("\n");
127
149
  }
128
- export { asRecord, columnsFor, coverageWarning, renderAnalysis, rowLabel };
150
+ export { ANALYZER_DISPLAY, asRecord, columnsFor, coverageWarning, renderAnalysis, rowLabel };
@@ -1,4 +1,36 @@
1
1
  const METRICS = {
2
+ query: {
3
+ label: "Search query",
4
+ unit: "count"
5
+ },
6
+ keyword: {
7
+ label: "Search query",
8
+ unit: "count"
9
+ },
10
+ keywordCount: {
11
+ label: "Search queries",
12
+ unit: "count"
13
+ },
14
+ newKeywords: {
15
+ label: "First observed queries",
16
+ unit: "count"
17
+ },
18
+ totalKeywords: {
19
+ label: "Reported queries",
20
+ unit: "count"
21
+ },
22
+ darkClicks: {
23
+ label: "Clicks without queries",
24
+ unit: "count"
25
+ },
26
+ darkPercent: {
27
+ label: "Click gap share",
28
+ unit: "share"
29
+ },
30
+ attributedClicks: {
31
+ label: "Clicks with queries",
32
+ unit: "count"
33
+ },
2
34
  clicks: {
3
35
  label: "Clicks",
4
36
  unit: "count"
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@gscdump/cli",
3
3
  "type": "module",
4
- "version": "6.0.0",
4
+ "version": "7.0.0",
5
5
  "description": "CLI for Google Search Console and Bing with hosted or local authentication, data exports, and an MCP server",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -44,16 +44,16 @@
44
44
  "dependencies": {
45
45
  "@clack/prompts": "^1.8.1",
46
46
  "@duckdb/node-api": "1.5.5-r.5",
47
- "@gscdump/analysis": "^6.0.0",
48
- "@gscdump/contracts": "^6.0.0",
49
- "@gscdump/engine": "^6.0.0",
50
- "@gscdump/engine-gsc-api": "^6.0.0",
51
- "@gscdump/sdk": "^6.0.0",
47
+ "@gscdump/analysis": "^7.0.0",
48
+ "@gscdump/contracts": "^7.0.0",
49
+ "@gscdump/engine": "^7.0.0",
50
+ "@gscdump/engine-gsc-api": "^7.0.0",
51
+ "@gscdump/sdk": "^7.0.0",
52
52
  "@modelcontextprotocol/sdk": "^1.30.1",
53
53
  "citty": "^0.2.2",
54
54
  "consola": "^3.4.2",
55
55
  "google-auth-library": "^11.1.0",
56
- "gscdump": "^6.0.0",
56
+ "gscdump": "^7.0.0",
57
57
  "ofetch": "^1.5.1",
58
58
  "open": "^11.0.4",
59
59
  "proper-lockfile": "^4.1.2",
@@ -1,6 +1,8 @@
1
1
  ---
2
2
  name: gscdump
3
3
  description: Drive the `gscdump` CLI for Google Search Console and Bing in Local or Hosted mode. Sync Google rows to a local Store, export Bing datasets, run SEO Analyzers and Reports, inspect Indexing Evidence, and manage sitemaps. Use when the user mentions gscdump, Search Console data, Bing Webmaster data, or GSC automation.
4
+ license: MIT
5
+ compatibility: "Requires the gscdump CLI and Node.js 22.13+ in the 22 release line, or Node.js 24+. API tasks need network access and credentials."
4
6
  ---
5
7
 
6
8
  # gscdump CLI
@@ -278,7 +280,7 @@ A routing stop also has `siteUrl`. `STORE_RANGE_NOT_COVERED` adds `missingDates`
278
280
  | `gscdump indexing` | Indexing API notifications and quota; hosted URL Inspection results |
279
281
  | `gscdump indexing inspect <url...> --yes` | Hosted: refresh up to 10 URLs through the platform, spending shared URL Inspections. Never retry automatically |
280
282
  | `gscdump indexing summary` | Hosted: the coverage ladder per day, with the time gscdump counted the verdicts |
281
- | `gscdump indexing watch` | Hosted: list, add, and remove Watched URLs and read their Checkpoints |
283
+ | `gscdump indexing watch` | Hosted: list, add, and remove Watched URLs and read their Watched URL inspections |
282
284
  | `gscdump dump` | Export Store tables, inspections, sitemaps, and Bing data as Parquet, CSV, JSON, NDJSON, SQLite, or DuckDB |
283
285
  | `gscdump store` | Store stats, compaction, garbage collection, resets |
284
286
  | `gscdump entities` | Read saved inspections; snapshot Indexing API metadata |
@@ -337,68 +339,9 @@ gscdump sync --site example.com --json
337
339
  - If the user excludes rollups, pass `--no-rollups` on the sync command.
338
340
  - Empty Store metadata is expected before the first sync. It does not prove zero traffic.
339
341
 
340
- ## Query rows
342
+ ## Store queries
341
343
 
342
- ```sh
343
- gscdump query --site example.com --dimensions page,query \
344
- --start 2026-08-01 --end 2026-08-28 --limit 1000 --format json
345
- ```
346
-
347
- - Dimension names are singular: `page`, `query`, `date`, `country`, `device`.
348
- - A page breakdown uses `--tables pages` for sync and `-d page` for query.
349
- `-d page,query` needs `page_queries`; syncing only `pages` does not fill that table.
350
- - Filters: `--query`, `--page`, `--country`, `--device`,
351
- `--search-appearance`. Prefixes: bare equals, `~` contains, `!~` not
352
- contains, `re:` regex, `!re:` not regex, `!` not equals.
353
- - `--page` takes a path or a full URL. The Store compares paths.
354
- - Without dates, `query` reads the 28 days ending on the newest synced day.
355
- - `--live` bypasses the Store and needs Local mode. `--type` selects a search type. The default is `web`.
356
- `--data-state` and `--aggregation-type` apply to live mode only.
357
- - Metrics already include clicks, impressions, CTR, and position. There is no `--metrics` option.
358
- - If Store coverage is missing, read the JSON error and run its `nextCommand`. It syncs only the missing dates and tables.
359
- Do not switch dimensions to make a failed query succeed.
360
- - `--explain` prints the request body or planned SQL without executing.
361
-
362
- ## SQL over the Store
363
-
364
- ```sh
365
- gscdump query --schema --format json
366
- gscdump query --format json --sql "SELECT search_type, SUM(clicks) AS clicks,
367
- gsc_position(sum_position, impressions) AS position
368
- FROM pages WHERE date >= DATE '2026-08-01' GROUP BY search_type"
369
- ```
370
-
371
- - `--sql` runs DuckDB SQL over one view per Store table: `pages`, `queries`,
372
- `page_queries`, `countries`, `dates`, `hourly_pages`, and the
373
- `search_appearance*` tables. Join views on `site`, `search_type`, `url`, and `date`.
374
- - `--schema` lists each view, its columns, its Sites, and its date range.
375
- - Every view has `site` (the Site URL) and `search_type`. The Store keeps
376
- every search type, so filter or group by `search_type`. A plain `SUM` adds
377
- web, image, and Discover rows together.
378
- - `url` holds the page path. `page` is the same value.
379
- - `sum_position` is the zero-based position times impressions. Use
380
- `gsc_position(sum_position, impressions)` for the average position. It adds 1
381
- and weights by impressions. Never average a per-row position.
382
- - The views cover every Site in the Store. `--site` and `--type` narrow them.
383
- - Dates return as `YYYY-MM-DD`. Integers return as numbers.
384
- - If a query names a table with no synced data, the JSON has a `warnings` list.
385
-
386
- ## Export the Store
387
-
388
- ```sh
389
- gscdump dump --site example.com --format parquet --out ./export
390
- gscdump dump --all-sites --format sqlite --out ./export
391
- ```
392
-
393
- - `dump` reads only the Store. It never calls Google to fill a gap.
394
- - Every exported row has `site` and `search_type`.
395
- - File formats write `<site>/<search_type>/<table>.<ext>` and
396
- `<site>/<dataset>.<ext>` for inspections, sitemaps, and Indexing API metadata.
397
- - `csv`, `json`, and `ndjson` rows also have `position`: `sum_position / impressions + 1`.
398
- - `sqlite` and `duckdb` write one file, `gscdump.sqlite` or `gscdump.duckdb`,
399
- with one table per dataset for every Site and search type.
400
- - `manifest.json` lists every dataset with its row count, plus the coverage that `sync --status --json` reports. Partial coverage is progress: daily sync fills the rest.
401
- - `sites.json` lists each exported Site URL with its Store ID.
344
+ Before row queries, SQL, or Store exports, read [Store queries](references/store-queries.md).
402
345
 
403
346
  ## Analyze and report
404
347
 
@@ -441,112 +384,9 @@ gscdump analyze striking-distance --site example.com --json
441
384
  with its dates, and quote `basis`. `basis._tag` is `clicks` when impressions cannot carry the read, for example
442
385
  when the peak falls in the period Search Console over-counted impressions, which ended 2026-04-27.
443
386
 
444
- ## Inspect and index
445
-
446
- ```sh
447
- gscdump inspect https://example.com/page https://example.com/other --site example.com --json
448
- gscdump inspect --site example.com --file urls.txt --json
449
- gscdump indexing quota --json
450
- ```
451
-
452
- Inspection spends Google's separate quota: 2,000 requests per day and 600 per minute for each property.
453
- `inspect` refuses more than 2,000 URLs in one run. It saves each result to the Store.
454
- On a quota error it stops and reports `remaining`. It exits 1 when any URL fails or remains.
455
- `indexing quota` describes Indexing API limits. It does not report remaining URL Inspection requests.
456
- Report the Indexing Evidence fields as Google returned them.
457
-
458
- ## Find URLs Google has not indexed (hosted)
459
-
460
- ```sh
461
- gscdump indexing urls --site example.com --status not_indexed --json
462
- gscdump indexing urls --site example.com --status not_indexed --all --format csv
463
- ```
464
-
465
- - The command reads URL Inspection results that gscdump.com already saved. It spends no inspection quota.
466
- - `--status` takes `indexed`, `not_indexed`, or `pending`. `--search` keeps URLs that contain the text.
467
- - Each row lists the sitemaps that contain the URL.
468
- - Pages hold 100 rows by default and 500 at most. Use `--offset` for the next page, or `--all` for every page.
469
- - With local authentication, the command fails. Pipe `gscdump sitemaps urls <sitemap-url>` into `gscdump inspect --site <site>` instead.
387
+ ## Hosted inspection
470
388
 
471
- ## Inspect one Sitemap (hosted)
472
-
473
- ```sh
474
- gscdump sitemaps inspect https://example.com/sitemap-old.xml --site example.com --engine google --json
475
- gscdump sitemaps inspect https://example.com/sitemap-old.xml --site example.com --engine bing --json
476
- ```
477
-
478
- Google reads that exact Sitemap from Search Console during the request.
479
- That read uses Google's API quota, without spending URL Inspections.
480
- Bing reads its stored provider list. Quote `capture.capturedAt` and `capture.source` with the answer.
481
- `listed` carries provider dates, flags, and counts. A null date means the provider supplied no date.
482
- `missing` means that exact Sitemap was absent at Capture time.
483
- `unavailable` means gscdump could not establish the evidence. The command exits with code 1.
484
- The command never submits a Sitemap or inspects URLs inside it.
485
- For Hosted MCP, use `inspect-sitemap` at `https://gscdump.com/mcp`.
486
-
487
- ## Read the coverage ladder (hosted)
488
-
489
- ```sh
490
- gscdump indexing summary --site example.com --json
491
- gscdump indexing summary --site example.com --days 90 --json
492
- ```
493
-
494
- The command reads saved URL Inspection verdicts. It spends no inspection quota.
495
- Each day in `trend` has `coverageStates`:
496
-
497
- - `counted` has `counts`, one URL count per coverage state, and `capturedAt`, the time gscdump counted them.
498
- - `not_counted` means gscdump stored the day before it counted coverage states. That day has no ladder. Say so.
499
-
500
- Read the 4 ladder states in `counts`, lowest rung first. Name each state with these words:
501
-
502
- | Key | Name | Meaning |
503
- | --- | --- | --- |
504
- | `unknown_to_google` | unknown | URL Inspection reported the URL as unknown at its check time |
505
- | `discovered_not_indexed` | discovered | URL Inspection reported discovery without a crawl at its check time |
506
- | `crawled_not_indexed` | crawled | URL Inspection reported a crawl without indexing at its check time |
507
- | `indexed` | indexed | URL Inspection reported the URL as indexed at its check time |
508
-
509
- The other keys in `counts` are exclusions, such as `noindex`, `not_found`, and `redirect`.
510
- `unrecognized` counts coverage text gscdump cannot map yet. `not_reported` counts results with no coverage text.
511
- A newer host can add keys to `counts`. Treat a key you do not know as its own state. Treat a missing key as 0.
512
-
513
- Rules for every answer:
514
-
515
- - `capture.source` is `stored` for counts gscdump saved on its daily run, and `live` for counts made during the request.
516
- - Quote `capturedAt` with every count. Also quote `capture.oldestVerdictAt` and the `capture.freshness` percents: `olderThan7dPercent` and `olderThan30dPercent`.
517
- - If `capture._tag` is `empty`, gscdump holds no verdicts for the Site. Report no ladder.
518
- - The counts are stored URL Inspection verdicts. They are not Google's live index.
519
- They are also not the Search Console Page indexing report. The two can differ a lot, and the Page indexing report can list many more URLs.
520
- - `olderThan7d` and `olderThan30d` count verdicts inspected more than 7 times 24 hours and 30 times 24 hours before `capturedAt`. They are not calendar days.
521
- - A verdict can be months older than its capture. Unchanged URLs are rechecked less often over time. Say when most verdicts are old.
522
- - Unknown and discovered verdicts do not show Google's current crawl queue. They do not prove noindex or content rejection.
523
- - An accepted indexing request does not confirm crawling or indexing. Compare its receipt date with the saved inspection date.
524
- - If the saved inspection predates a request, it cannot show that request's outcome. A live indexability test also cannot confirm indexing.
525
- - A move between rungs shows in the trend. Compare two `counted` days and name the dates.
526
- - `capture.scope` says which URLs the counts cover. `sitemap_members` means every counted URL is in a live sitemap.
527
- `inspected_urls` means sitemap membership was not available, so the counts can include URLs outside every sitemap.
528
- - If `unknown_to_google` URLs are already in a sitemap, the sitemap is not the gap. Do not tell the owner to add them to it.
529
- Read the sitemap's last download with `gscdump sitemaps current --site SITE --json`. Compare that date with each saved inspection date.
530
- A sitemap download confirms that Google read the feed. It does not confirm discovery, crawling, or indexing of every URL.
531
- Check one URL's membership with `gscdump sitemaps membership`.
532
- - Name no cause beyond what the evidence shows.
533
- - Cite the command behind each number.
534
-
535
- ## Watch URLs (hosted)
536
-
537
- ```sh
538
- gscdump indexing watch add --site example.com https://example.com/guide https://example.com/pricing
539
- gscdump indexing watch list --site example.com --json
540
- gscdump indexing watch remove --site example.com https://example.com/pricing
541
- ```
542
-
543
- - A Site can hold 50 Watched URLs. `add` and `remove` send 50 URLs per request, so a longer list works. Every URL must be absolute. `add` fails before any request if one is relative.
544
- - If `add` skips a URL with `inspection_disabled`, the Site has URL Inspection off. gscdump never inspects it, so the URL gets no Checkpoint. Tell the owner to turn on URL Inspection in the Site settings.
545
- - gscdump inspects each Watched URL every 7 days, before other scheduled URLs. It never backs off.
546
- - Each scheduled inspection spends the Site's daily URL Inspection budget and the URL Inspections meter. Get consent before `add`.
547
- - Each scheduled inspection stores one Checkpoint. `watch list --json` returns Checkpoints newest first, with `checkedAt`, the `coverageState` key, and Google's text.
548
- - Compare Checkpoints by date to see when a URL moves up the ladder. A new Watched URL has no Checkpoint until its first inspection.
549
- - `remove` deletes the URL's Checkpoints. The saved URL Inspection results stay.
389
+ Before hosted inspection, indexing, Sitemap coverage, or Watch tasks, read [Hosted inspection](references/hosted-inspection.md).
550
390
 
551
391
  ## Report a papercut
552
392
 
@@ -581,7 +421,7 @@ the failure and continue. Never retry an uncertain submission.
581
421
  `sites verify`, `sitemaps submit`, `sitemaps delete`, `indexing submit`,
582
422
  `indexing remove`, `store reset`, and `store rm-site` change Google or
583
423
  delete local data. `indexing watch add` spends URL Inspections every week,
584
- and `indexing watch remove` deletes Checkpoints. `--yes` is consent you
424
+ and `indexing watch remove` deletes Watched URL inspections. `--yes` is consent you
585
425
  borrow from the user.
586
426
  - **Never loop unattended.** One `sync` per Site per task. Inspection batches
587
427
  spend a daily pool. Use `--dry-run` and `--explain` to plan first.
@@ -614,13 +454,4 @@ For a deletion explanation, read metadata only if needed. Explain the scope and
614
454
 
615
455
  ## Refresh hosted Google URL Inspection results
616
456
 
617
- ```sh
618
- gscdump indexing inspect https://example.com/page --site example.com --yes --json
619
- ```
620
-
621
- This asks Google for its current indexed-page verdict. It does not run a live-page test or request indexing.
622
- Pass 1 to 10 unique URLs, or use `--file` or stdin. The platform enforces its shared daily quota.
623
- The command sends one request. It never splits batches or retries automatically.
624
- If the API returns 429, wait for the reported reset before another request.
625
- If `errors` or `skipped` contains URLs, the command exits 1 and preserves its result on stdout.
626
- Read `rateLimit.remaining` before deciding whether to inspect more URLs.
457
+ Before a hosted Google URL Inspection refresh, read [Hosted inspection](references/hosted-inspection.md).
@@ -0,0 +1,131 @@
1
+ # Hosted inspection
2
+
3
+ - [Inspect and index](#inspect-and-index)
4
+ - [Find URLs Google has not indexed (hosted)](#find-urls-google-has-not-indexed-hosted)
5
+ - [Inspect one Sitemap (hosted)](#inspect-one-sitemap-hosted)
6
+ - [Read the coverage ladder (hosted)](#read-the-coverage-ladder-hosted)
7
+ - [Watch URLs (hosted)](#watch-urls-hosted)
8
+ - [Refresh hosted Google URL Inspection results](#refresh-hosted-google-url-inspection-results)
9
+
10
+ ## Inspect and index
11
+
12
+ ```sh
13
+ gscdump inspect https://example.com/page https://example.com/other --site example.com --json
14
+ gscdump inspect --site example.com --file urls.txt --json
15
+ gscdump indexing quota --json
16
+ ```
17
+
18
+ Inspection spends Google's separate quota: 2,000 requests per day and 600 per minute for each property.
19
+ `inspect` refuses more than 2,000 URLs in one run. It saves each result to the Store.
20
+ On a quota error it stops and reports `remaining`. It exits 1 when any URL fails or remains.
21
+ `indexing quota` describes Indexing API limits. It does not report remaining URL Inspection requests.
22
+ Report the Indexing Evidence fields as Google returned them.
23
+
24
+ ## Find URLs Google has not indexed (hosted)
25
+
26
+ ```sh
27
+ gscdump indexing urls --site example.com --status not_indexed --json
28
+ gscdump indexing urls --site example.com --status not_indexed --all --format csv
29
+ ```
30
+
31
+ - The command reads URL Inspection results that gscdump.com already saved. It spends no inspection quota.
32
+ - `--status` takes `indexed`, `not_indexed`, or `pending`. `--search` keeps URLs that contain the text.
33
+ - Each row lists the sitemaps that contain the URL.
34
+ - Pages hold 100 rows by default and 500 at most. Use `--offset` for the next page, or `--all` for every page.
35
+ - With local authentication, the command fails. Pipe `gscdump sitemaps urls <sitemap-url>` into `gscdump inspect --site <site>` instead.
36
+
37
+ ## Inspect one Sitemap (hosted)
38
+
39
+ ```sh
40
+ gscdump sitemaps inspect https://example.com/sitemap-old.xml --site example.com --engine google --json
41
+ gscdump sitemaps inspect https://example.com/sitemap-old.xml --site example.com --engine bing --json
42
+ ```
43
+
44
+ Google reads that exact Sitemap from Search Console during the request.
45
+ That read uses Google's API quota, without spending URL Inspections.
46
+ Bing reads its stored provider list. Quote `capture.capturedAt` and `capture.source` with the answer.
47
+ `listed` carries provider dates, flags, and counts. A null date means the provider supplied no date.
48
+ `missing` means that exact Sitemap was absent at Capture time.
49
+ `unavailable` means gscdump could not establish the evidence. The command exits with code 1.
50
+ The command never submits a Sitemap or inspects URLs inside it.
51
+ For Hosted MCP, use `inspect-sitemap` at `https://gscdump.com/mcp`.
52
+
53
+ ## Read the coverage ladder (hosted)
54
+
55
+ ```sh
56
+ gscdump indexing summary --site example.com --json
57
+ gscdump indexing summary --site example.com --days 90 --json
58
+ ```
59
+
60
+ The command reads saved URL Inspection verdicts. It spends no inspection quota.
61
+ Each day in `trend` has `coverageStates`:
62
+
63
+ - `counted` has `counts`, one URL count per coverage state, and `capturedAt`, the time gscdump counted them.
64
+ - `not_counted` means gscdump stored the day before it counted coverage states. That day has no ladder. Say so.
65
+
66
+ Read the 4 ladder states in `counts`, lowest rung first. Name each state with these words:
67
+
68
+ | Key | Name | Meaning |
69
+ | --- | --- | --- |
70
+ | `unknown_to_google` | unknown | URL Inspection reported the URL as unknown at its check time |
71
+ | `discovered_not_indexed` | discovered, not indexed | URL Inspection reported discovery without a crawl at its check time |
72
+ | `crawled_not_indexed` | crawled, not indexed | URL Inspection reported a crawl without indexing at its check time |
73
+ | `indexed` | indexed | URL Inspection reported the URL as indexed at its check time |
74
+
75
+ The other keys in `counts` are exclusions, such as `noindex`, `not_found`, and `redirect`.
76
+ `unrecognized` counts coverage text gscdump cannot map yet. `not_reported` counts results with no coverage text.
77
+ A newer host can add keys to `counts`. Treat a key you do not know as its own state. Treat a missing key as 0.
78
+
79
+ Rules for every answer:
80
+
81
+ - `capture.source` is `stored` for counts gscdump saved on its daily run, and `live` for counts made during the request from saved inspections. Neither value means Google was inspected again.
82
+ - Quote `capturedAt` with every count. Also quote `capture.oldestVerdictAt` and the `capture.freshness` percents: `olderThan7dPercent` and `olderThan30dPercent`.
83
+ - If `capture._tag` is `empty`, gscdump holds no verdicts for the Site. Report no ladder.
84
+ - The counts are stored URL Inspection verdicts. They are not Google's live index.
85
+ They are also not the Search Console Page indexing report. The two can differ a lot, and the Page indexing report can list many more URLs.
86
+ - `olderThan7d` and `olderThan30d` count verdicts inspected more than 7 times 24 hours and 30 times 24 hours before `capturedAt`. They are not calendar days.
87
+ - A verdict can be months older than its capture. Unchanged URLs are rechecked less often over time. Say when most verdicts are old.
88
+ - Unknown and discovered verdicts do not show Google's current crawl queue. They do not prove noindex or content rejection.
89
+ - An accepted indexing request does not confirm crawling or indexing. Compare its receipt date with the saved inspection date.
90
+ - If the saved inspection predates a request, it cannot show that request's outcome. A live indexability test also cannot confirm indexing.
91
+ - A move between rungs shows in the trend. Compare two `counted` days and name the dates.
92
+ - `capture.scope` says which URLs the counts cover. `sitemap_members` means every counted URL is in a live sitemap.
93
+ `inspected_urls` means sitemap membership was not available, so the counts can include URLs outside every sitemap.
94
+ - If `unknown_to_google` URLs are already in a sitemap, the sitemap is not the gap. Do not tell the owner to add them to it.
95
+ Read the sitemap's last download with `gscdump sitemaps current --site SITE --json`. Compare that date with each saved inspection date.
96
+ A sitemap download confirms that Google read the feed. It does not confirm discovery, crawling, or indexing of every URL.
97
+ Check one URL's membership with `gscdump sitemaps membership`.
98
+ - Name no cause beyond what the evidence shows.
99
+ - Cite the command behind each number.
100
+
101
+ ## Watch URLs (hosted)
102
+
103
+ The frozen `checkpoints` array holds Watched URL inspections. These are the scheduled inspection history for watched URLs.
104
+
105
+ ```sh
106
+ gscdump indexing watch add --site example.com https://example.com/guide https://example.com/pricing
107
+ gscdump indexing watch list --site example.com --json
108
+ gscdump indexing watch remove --site example.com https://example.com/pricing
109
+ ```
110
+
111
+ - A Site can hold 50 Watched URLs. `add` and `remove` send 50 URLs per request, so a longer list works. Every URL must be absolute. `add` fails before any request if one is relative.
112
+ - If `add` skips a URL with `inspection_disabled`, the Site has URL Inspection off. gscdump never inspects it, so the URL gets no Watched URL inspection. Tell the owner to turn on URL Inspection in the Site settings.
113
+ - gscdump inspects each Watched URL every 7 days, before other scheduled URLs. It never backs off.
114
+ - Each scheduled inspection spends the Site's daily URL Inspection budget and the URL Inspections meter. Get consent before `add`.
115
+ - Each scheduled inspection stores one Watched URL inspection. `watch list --json` returns Watched URL inspections newest first, with `checkedAt`, the `coverageState` key, and Google's text.
116
+ - Compare Watched URL inspections by date to see when a URL moves up the ladder. A new Watched URL has no Watched URL inspection until its first inspection.
117
+ - `remove` deletes the URL's Watched URL inspections. The saved URL Inspection results stay.
118
+
119
+
120
+ ## Refresh hosted Google URL Inspection results
121
+
122
+ ```sh
123
+ gscdump indexing inspect https://example.com/page --site example.com --yes --json
124
+ ```
125
+
126
+ This asks Google for its current indexed-page verdict. It does not run a live-page test or request indexing.
127
+ Pass 1 to 10 unique URLs, or use `--file` or stdin. The platform enforces its shared daily quota.
128
+ The command sends one request. It never splits batches or retries automatically.
129
+ If the API returns 429, wait for the reported reset before another request.
130
+ If `errors` or `skipped` contains URLs, the command exits 1 and preserves its result on stdout.
131
+ Read `rateLimit.remaining` before deciding whether to inspect more URLs.
@@ -0,0 +1,65 @@
1
+ # Store queries
2
+
3
+ ## Query rows
4
+
5
+ ```sh
6
+ gscdump query --site example.com --dimensions page,query \
7
+ --start 2026-08-01 --end 2026-08-28 --limit 1000 --format json
8
+ ```
9
+
10
+ - Dimension names are singular: `page`, `query`, `date`, `country`, `device`.
11
+ - A page breakdown uses `--tables pages` for sync and `-d page` for query.
12
+ `-d page,query` needs `page_queries`; syncing only `pages` does not fill that table.
13
+ - Filters: `--query`, `--page`, `--country`, `--device`,
14
+ `--search-appearance`. Prefixes: bare equals, `~` contains, `!~` not
15
+ contains, `re:` regex, `!re:` not regex, `!` not equals.
16
+ - `--page` takes a path or a full URL. The Store compares paths.
17
+ - Without dates, `query` reads the 28 days ending on the newest synced day.
18
+ - `--live` bypasses the Store and needs Local mode. `--type` selects a search type. The default is `web`.
19
+ `--data-state` and `--aggregation-type` apply to live mode only.
20
+ - Metrics already include clicks, impressions, CTR, and position. There is no `--metrics` option.
21
+ - If Store coverage is missing, read the JSON error and run its `nextCommand`. It syncs only the missing dates and tables.
22
+ Do not switch dimensions to make a failed query succeed.
23
+ - `--explain` prints the request body or planned SQL without executing.
24
+
25
+ ## SQL over the Store
26
+
27
+ ```sh
28
+ gscdump query --schema --format json
29
+ gscdump query --format json --sql "SELECT search_type, SUM(clicks) AS clicks,
30
+ gsc_position(sum_position, impressions) AS position
31
+ FROM pages WHERE date >= DATE '2026-08-01' GROUP BY search_type"
32
+ ```
33
+
34
+ - `--sql` runs DuckDB SQL over one view per Store table: `pages`, `queries`,
35
+ `page_queries`, `countries`, `dates`, `hourly_pages`, and the
36
+ `search_appearance*` tables. Join views on `site`, `search_type`, `url`, and `date`.
37
+ - `--schema` lists each view, its columns, its Sites, and its date range.
38
+ - Every view has `site` (the Site URL) and `search_type`. The Store keeps
39
+ every search type, so filter or group by `search_type`. A plain `SUM` adds
40
+ web, image, and Discover rows together.
41
+ - `url` holds the page path. `page` is the same value.
42
+ - `sum_position` is the zero-based position times impressions. Use
43
+ `gsc_position(sum_position, impressions)` for the average position. It adds 1
44
+ and weights by impressions. Never average a per-row position.
45
+ - The views cover every Site in the Store. `--site` and `--type` narrow them.
46
+ - Dates return as `YYYY-MM-DD`. Integers return as numbers.
47
+ - If a query names a table with no synced data, the JSON has a `warnings` list.
48
+
49
+ ## Export the Store
50
+
51
+ ```sh
52
+ gscdump dump --site example.com --format parquet --out ./export
53
+ gscdump dump --all-sites --format sqlite --out ./export
54
+ ```
55
+
56
+ - `dump` reads only the Store. It never calls Google to fill a gap.
57
+ - Every exported row has `site` and `search_type`.
58
+ - File formats write `<site>/<search_type>/<table>.<ext>` and
59
+ `<site>/<dataset>.<ext>` for inspections, sitemaps, and Indexing API metadata.
60
+ - `csv`, `json`, and `ndjson` rows also have `position`: `sum_position / impressions + 1`.
61
+ - `sqlite` and `duckdb` write one file, `gscdump.sqlite` or `gscdump.duckdb`,
62
+ with one table per dataset for every Site and search type.
63
+ - `manifest.json` lists every dataset with its row count, plus the coverage that `sync --status --json` reports. Partial coverage is progress: daily sync fills the rest.
64
+ - `sites.json` lists each exported Site URL with its Store ID.
65
+