@gscdump/cli 3.4.4 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +217 -54
- package/bin/gscdump.mjs +6 -1
- package/dist/analysis-local.mjs +1 -1
- package/dist/auth-state.mjs +112 -0
- package/dist/auth.mjs +109 -122
- package/dist/bing-auth.mjs +200 -0
- package/dist/bing-data.mjs +160 -0
- package/dist/bing-hosted.mjs +121 -0
- package/dist/cli.d.mts +4 -4
- package/dist/cli.mjs +20 -21
- package/dist/cloud-google.mjs +92 -0
- package/dist/command-meta.mjs +13 -1
- package/dist/command-registry.mjs +5 -2
- package/dist/commands/analyze.mjs +34 -177
- package/dist/commands/auth.mjs +161 -7
- package/dist/commands/bing.mjs +337 -0
- package/dist/commands/config.mjs +36 -43
- package/dist/commands/doctor.mjs +73 -24
- package/dist/commands/dump.mjs +1 -1
- package/dist/commands/entities.mjs +4 -4
- package/dist/commands/indexing.mjs +11 -14
- package/dist/commands/init.mjs +1 -1
- package/dist/commands/inspect.mjs +3 -3
- package/dist/commands/mcp.mjs +16 -2
- package/dist/commands/papercut.mjs +76 -0
- package/dist/commands/profile-selection.mjs +2 -2
- package/dist/commands/profile.mjs +8 -3
- package/dist/commands/query.mjs +86 -40
- package/dist/commands/report.mjs +5 -3
- package/dist/commands/sitemaps.mjs +24 -18
- package/dist/commands/skill.mjs +52 -0
- package/dist/commands/stats.mjs +58 -32
- package/dist/commands/sync.mjs +58 -25
- package/dist/config.mjs +39 -4
- package/dist/context.mjs +13 -8
- package/dist/env-file.mjs +1 -1
- package/dist/local-store.mjs +2 -2
- package/dist/mcp/errors.mjs +8 -0
- package/dist/mcp/handlers/diagnostics.mjs +31 -0
- package/dist/mcp/handlers/reports.mjs +38 -11
- package/dist/mcp/server/index.mjs +9 -10
- package/dist/mcp/types.mjs +8 -3
- package/dist/package.mjs +1 -1
- package/dist/papercut.mjs +99 -0
- package/dist/render/analysis.mjs +98 -0
- package/dist/render/charts.mjs +170 -0
- package/dist/render/layout.mjs +87 -0
- package/dist/render/metrics.mjs +163 -0
- package/dist/render/query.mjs +30 -0
- package/dist/render/report.mjs +69 -0
- package/dist/render/terminal.mjs +25 -0
- package/dist/runtime.d.mts +5 -5
- package/dist/runtime.mjs +1 -1
- package/dist/skill.mjs +45 -0
- package/dist/utils.mjs +10 -38
- package/package.json +14 -12
- package/skills/gscdump/SKILL.md +287 -0
package/README.md
CHANGED
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
[](https://npm.chart.dev/@gscdump/cli)
|
|
5
5
|
[](https://github.com/harlan-zw/gscdump/blob/main/LICENSE)
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Query Google Search Console and Bing with hosted or local authentication.
|
|
8
|
+
Sync Google rows to a local Parquet Store, and run SEO Analyzers or Reports.
|
|
9
|
+
The package also provides the MCP server.
|
|
10
|
+
Use Node.js 22.13 or later in the 22 release line, or Node.js 24 or later.
|
|
8
11
|
|
|
9
12
|
## Install
|
|
10
13
|
|
|
@@ -17,20 +20,21 @@ npx @gscdump/cli
|
|
|
17
20
|
## Quick start
|
|
18
21
|
|
|
19
22
|
```bash
|
|
20
|
-
#
|
|
21
|
-
gscdump init
|
|
23
|
+
# Set up local Google OAuth
|
|
24
|
+
gscdump init --mode local
|
|
25
|
+
gscdump auth login --mode local
|
|
22
26
|
|
|
23
27
|
# List sites
|
|
24
28
|
gscdump sites
|
|
25
29
|
|
|
26
|
-
# Sync
|
|
27
|
-
gscdump sync --site
|
|
30
|
+
# Sync 90 days to the Store
|
|
31
|
+
gscdump sync --site sc-domain:example.com --days 90 --tables pages,queries,page_queries,countries,dates
|
|
28
32
|
|
|
29
|
-
# Query the
|
|
30
|
-
gscdump query --site
|
|
33
|
+
# Query the Store
|
|
34
|
+
gscdump query --site sc-domain:example.com --dimensions page,query --limit 50
|
|
31
35
|
|
|
32
|
-
# Run an
|
|
33
|
-
gscdump analyze striking-distance --site
|
|
36
|
+
# Run an Analyzer
|
|
37
|
+
gscdump analyze striking-distance --site sc-domain:example.com
|
|
34
38
|
|
|
35
39
|
# Start the MCP server
|
|
36
40
|
gscdump mcp
|
|
@@ -40,27 +44,124 @@ gscdump mcp
|
|
|
40
44
|
|
|
41
45
|
| Command | Description |
|
|
42
46
|
|---|---|
|
|
43
|
-
| `init` | Full setup (OAuth + dataDir; offers to write a `.env` for
|
|
47
|
+
| `init` | Full setup (OAuth + dataDir; offers to write a `.env` for later use) |
|
|
44
48
|
| `auth` | Manage authentication (`status`, `login`, `logout`, `refresh`) |
|
|
49
|
+
| `bing` | Connect Bing, list sites, dump datasets, inspect URLs, and check hosted verification |
|
|
45
50
|
| `config` | Manage CLI configuration (`show`, `set`, `unset`, `path`, `validate`) |
|
|
46
51
|
| `doctor` | Health checks: auth, scopes, dataDir writability, API reachability |
|
|
47
52
|
| `sites [--owner-only] [--with-sitemaps]` | List available GSC sites |
|
|
48
|
-
| `sites add <url>` / `sites delete <url> [--yes]` | Register
|
|
53
|
+
| `sites add <url>` / `sites delete <url> [--yes]` | Register or remove a Site in Search Console (add registers in unverified state) |
|
|
49
54
|
| `sites verify-token <url> [--method]` / `sites verify <url> [--method]` | Get a verification token, then trigger ownership verification (META/FILE/DNS_TXT/DNS_CNAME/ANALYTICS/TAG_MANAGER) |
|
|
50
|
-
| `sitemaps` |
|
|
51
|
-
| `inspect <url>` / `inspect batch [--concurrency]` | URL inspection (single URL or batch from file/stdin); renders
|
|
52
|
-
| `indexing` | Notify Google about URL changes (`submit`, `remove`, `status`, `batch
|
|
53
|
-
| `sync` | Sync GSC data to the local Parquet
|
|
54
|
-
| `query` | Run a search analytics query (
|
|
55
|
-
| `dump` | Export from the
|
|
56
|
-
| `analyze <tool>` | Run an SEO
|
|
55
|
+
| `sitemaps` | List, submit, or delete Google sitemaps; probe live URLs; read hosted snapshots (`current`, `history`, `membership`, `lastmod`, `export`) |
|
|
56
|
+
| `inspect <url>` / `inspect batch [--concurrency]` | URL inspection (single URL or batch from file/stdin); renders Indexing Evidence, rich results, and AMP |
|
|
57
|
+
| `indexing` | Notify Google about URL changes (`submit`, `remove`, `status`, `batch`, `batch-status`, `quota`); supports `--retries` |
|
|
58
|
+
| `sync` | Sync GSC data to the local Parquet Store; `--retry-failed`, `--dry-run` |
|
|
59
|
+
| `query` | Run a search analytics query (Store by default; `--live` hits GSC API). Filters: `--query`, `--page`, `--country`, `--device`, `--search-appearance`, `--type`, `--data-state`, `--aggregation-type`. `--explain` previews the request body; `--output -` writes to stdout. |
|
|
60
|
+
| `dump` | Export from the Store to a directory (`--format parquet\|json\|ndjson\|csv`, `--tables`, `--all-sites`) |
|
|
61
|
+
| `analyze <tool>` | Run an SEO Analyzer against the Store (`--live` for row-based against fresh API) |
|
|
57
62
|
| `entities` | Snapshot URL inspections and indexing metadata into the local entity store |
|
|
58
63
|
| `store stats` | Show row/byte counts per table and on-disk footprint |
|
|
59
|
-
| `store compact` |
|
|
64
|
+
| `store compact` | Compact older data into weekly, monthly, and quarterly tiers (`--dry-run`) |
|
|
60
65
|
| `store gc` | Delete orphaned objects past the grace window (`--dry-run`) |
|
|
61
66
|
| `store export` | Export the live store to a single `.duckdb` file |
|
|
67
|
+
| `store rm-site` / `store reset` | Delete one Site's data or reset the Store; inspect `--help` before use |
|
|
62
68
|
| `store rollups rebuild` | Rebuild post-sync rollup tables |
|
|
69
|
+
| `report <id>` / `report list` | Run or list Reports; `--explain` previews a plan |
|
|
70
|
+
| `profile` | Create, select, list, or delete credential profiles |
|
|
63
71
|
| `mcp` | Start the MCP server for AI assistants |
|
|
72
|
+
| `skill install [--agent claude\|codex] [--target <dir>]` | Copy the packaged `gscdump` agent skill (SKILL.md) into an agent skill directory |
|
|
73
|
+
| `papercut --command <cmd> --comment <text> --agent <name> [--intent bug\|improvement] --yes` | Report a CLI problem to gscdump.com; anonymous, ten per hour per network address |
|
|
74
|
+
|
|
75
|
+
## Hosted and local authentication
|
|
76
|
+
|
|
77
|
+
One authentication mode applies to both Google and Bing.
|
|
78
|
+
Credentials and the saved mode belong to the current `--profile` or `--config-dir`.
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
# Use a user API key from https://gscdump.com/app/settings
|
|
82
|
+
export GSCDUMP_API_KEY=gsd_user_...
|
|
83
|
+
gscdump auth login --mode cloud
|
|
84
|
+
gscdump auth status --json
|
|
85
|
+
|
|
86
|
+
# Google uses the connection on gscdump.com
|
|
87
|
+
gscdump sites --json
|
|
88
|
+
gscdump query --live --site sc-domain:example.com --dimensions page
|
|
89
|
+
|
|
90
|
+
# Connect Bing through gscdump.com, then read its data
|
|
91
|
+
gscdump bing sites --json
|
|
92
|
+
gscdump bing login --site s_SITE_ID
|
|
93
|
+
gscdump bing status --site s_SITE_ID --json
|
|
94
|
+
gscdump bing dump --site s_SITE_ID --out ./bing-export --format csv
|
|
95
|
+
|
|
96
|
+
# Use local credentials for one command
|
|
97
|
+
gscdump bing sites --mode local --json
|
|
98
|
+
|
|
99
|
+
# Save local mode after successful Google login
|
|
100
|
+
gscdump auth login --mode local
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`--mode cloud|local` overrides the mode for one invocation.
|
|
104
|
+
`GSCDUMP_AUTH_MODE` provides the same override.
|
|
105
|
+
A successful login saves the mode for later commands.
|
|
106
|
+
If no mode is saved, `GSCDUMP_API_KEY` selects hosted authentication.
|
|
107
|
+
With neither source, the CLI defaults to local authentication.
|
|
108
|
+
When a saved mode exists, it remains selected unless an explicit override applies.
|
|
109
|
+
`GSCDUMP_API_ROOT` defaults to `https://gscdump.com/api`.
|
|
110
|
+
If you change a saved API root, supply the API key explicitly.
|
|
111
|
+
|
|
112
|
+
| Operation | Hosted authentication | Local authentication |
|
|
113
|
+
| --- | --- | --- |
|
|
114
|
+
| Google queries, sync, sites, sitemaps, URL inspection | Uses the Google connection through gscdump.com | Calls Google with local credentials |
|
|
115
|
+
| Google Indexing API and Site Verification API | Requires local mode | Supported with the required Google scopes |
|
|
116
|
+
| Bing datasets | Reads synced datasets through the public API | Reads data currently returned by Bing |
|
|
117
|
+
| Bing connection and CNAME verification | Uses `bing login`, `bing status`, and `bing verify` | Verify sites in Bing Webmaster Tools |
|
|
118
|
+
| Hosted sitemap membership and history | Supported | Requires hosted authentication |
|
|
119
|
+
| Store queries and exports | Reads the local Store | Reads the local Store |
|
|
120
|
+
|
|
121
|
+
Hosted Bing access follows the API's plan and preview access rules.
|
|
122
|
+
`auth logout` removes the saved mode and saved Google and Bing credentials.
|
|
123
|
+
Environment credentials remain active until you unset them.
|
|
124
|
+
|
|
125
|
+
## Bing
|
|
126
|
+
|
|
127
|
+
For local Bing access, generate an API key in Bing Webmaster Tools under Settings, API Access.
|
|
128
|
+
See [Microsoft's authentication guide](https://learn.microsoft.com/en-us/bingwebmaster/getting-access).
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
export BING_API_KEY=...
|
|
132
|
+
gscdump bing login --mode local
|
|
133
|
+
gscdump bing sites --json
|
|
134
|
+
gscdump bing dump --site https://example.com/ --format json --out ./bing-export
|
|
135
|
+
gscdump bing dump --all-sites --format ndjson
|
|
136
|
+
gscdump bing inspect https://example.com/page --site https://example.com/ --json
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
`BING_ACCESS_TOKEN` also accepts an existing Bing OAuth access token.
|
|
140
|
+
Local API keys and OAuth credentials are saved separately from Google credentials.
|
|
141
|
+
`bing logout --mode local` removes only saved Bing credentials.
|
|
142
|
+
|
|
143
|
+
For local browser OAuth, register a Bing OAuth client with this exact redirect URI:
|
|
144
|
+
`http://127.0.0.1:53683/oauth/bing`.
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
export BING_CLIENT_ID=...
|
|
148
|
+
export BING_CLIENT_SECRET=...
|
|
149
|
+
gscdump bing login --mode local --oauth
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Use `--redirect-uri` or `BING_REDIRECT_URI` for another registered loopback URI.
|
|
153
|
+
`--no-browser` prints the authorization URL. Saved OAuth credentials refresh automatically.
|
|
154
|
+
|
|
155
|
+
`bing dump` exports `traffic`, `pages`, `keywords`, and `crawl` by default.
|
|
156
|
+
Use `--datasets traffic,pages` to select datasets.
|
|
157
|
+
Local mode also supports `--datasets crawl-issues`.
|
|
158
|
+
Files use JSON, NDJSON, or CSV, with one directory per Bing site and a `metadata.json` file.
|
|
159
|
+
|
|
160
|
+
In hosted mode, exports default to the last 366 days.
|
|
161
|
+
Use `--start YYYY-MM-DD --end YYYY-MM-DD` for a range up to 366 days.
|
|
162
|
+
The CLI follows every returned page and rejects unavailable datasets or a snapshot that changes during export.
|
|
163
|
+
In local mode, date options filter the rows Bing returns. They cannot recover older history.
|
|
164
|
+
Bing Indexing Evidence preserves uncertainty and does not imply an indexed verdict.
|
|
64
165
|
|
|
65
166
|
### Filter expressions
|
|
66
167
|
|
|
@@ -81,45 +182,68 @@ gscdump query --live --site sc-domain:example.com \
|
|
|
81
182
|
--page '~/blog/' --query '~brand' --dimensions page,query
|
|
82
183
|
```
|
|
83
184
|
|
|
185
|
+
`--limit` and `--format` override saved `defaultLimit` and `defaultFormat` values.
|
|
186
|
+
Without saved values, queries use 1000 rows and JSON.
|
|
187
|
+
Limits must be positive integers. Formats must be `json` or `csv`.
|
|
188
|
+
|
|
189
|
+
`--start` and `--end` work independently. An omitted date keeps its default.
|
|
190
|
+
Non-interactive queries default to 31 days ago through three days ago, using UTC dates.
|
|
191
|
+
Dates must use `YYYY-MM-DD`, and `--start` cannot follow `--end`.
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
# Export query rows with a CSV header.
|
|
195
|
+
gscdump query --live --site sc-domain:example.com \
|
|
196
|
+
--dimensions page,query --format csv --output rows.csv
|
|
197
|
+
```
|
|
198
|
+
|
|
84
199
|
### Global flags
|
|
85
200
|
|
|
86
201
|
- `--no-color` / `NO_COLOR` env: strip ANSI from stdout (stderr keeps colour for interactive use).
|
|
87
202
|
- `--config-dir <path>` / `GSCDUMP_CONFIG_DIR`: override `~/.config/gscdump`.
|
|
88
|
-
- `--profile <name>` / `GSCDUMP_PROFILE`:
|
|
89
|
-
- Most commands accept `--quiet` and `--json` for
|
|
203
|
+
- `--profile <name>` / `GSCDUMP_PROFILE`: separate tokens and config to a profile under `~/.config/gscdump/profiles/<name>` (separate Google credentials).
|
|
204
|
+
- Most commands accept `--quiet` and `--json` for scripts. The `query` command uses `--format json` instead.
|
|
90
205
|
|
|
91
|
-
|
|
206
|
+
Use `query --profile` for query timings. Use `--profile <name>` to select a credential profile.
|
|
207
|
+
Numeric flags reject fractions, negative counts, and text suffixes.
|
|
208
|
+
Saved config rejects invalid values and unknown keys. If parsing fails, fix the reported file.
|
|
92
209
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
**Core SEO:** `striking-distance`, `opportunity`, `movers`, `decay`, `zero-click`, `brand`, `cannibalization`
|
|
210
|
+
## Analyzers
|
|
96
211
|
|
|
97
|
-
|
|
212
|
+
`gscdump analyze <tool>` runs one of 29 Analyzers from `@gscdump/analysis`.
|
|
213
|
+
See the [full list](../../README.md#analyzers) and [Source support](../analysis/README.md#sources).
|
|
98
214
|
|
|
99
|
-
|
|
215
|
+
Each Analyzer accepts `--site`, `--start`, `--end`, `--limit`, and output flags.
|
|
216
|
+
`movers` and `decay` also accept `--prev-start` and `--prev-end`.
|
|
217
|
+
Use `gscdump analyze <tool> --help` for additional options.
|
|
100
218
|
|
|
101
|
-
|
|
219
|
+
The CLI requires local data unless you pass `--live`.
|
|
220
|
+
Pass `--live` to use Google explicitly.
|
|
221
|
+
SQL-only Analyzers require local data.
|
|
102
222
|
|
|
103
223
|
## Sync
|
|
104
224
|
|
|
105
225
|
```bash
|
|
106
|
-
# Default:
|
|
107
|
-
gscdump sync --site
|
|
226
|
+
# Default: three days ending three days ago; skip completed dates
|
|
227
|
+
gscdump sync --site sc-domain:example.com --tables pages,queries,page_queries,countries,dates
|
|
108
228
|
|
|
109
|
-
# Backfill
|
|
110
|
-
gscdump sync --site
|
|
229
|
+
# Backfill from 450 days ago to three days ago
|
|
230
|
+
gscdump sync --site sc-domain:example.com --full --tables pages,queries,page_queries,countries,dates
|
|
111
231
|
|
|
112
232
|
# Custom range
|
|
113
|
-
gscdump sync --site
|
|
233
|
+
gscdump sync --site sc-domain:example.com --start 2026-08-01 --end 2026-08-31 \
|
|
234
|
+
--tables pages,queries,page_queries,countries,dates
|
|
114
235
|
|
|
115
|
-
# Check
|
|
116
|
-
gscdump sync --site
|
|
236
|
+
# Check sync state and watermarks
|
|
237
|
+
gscdump sync --site sc-domain:example.com --status
|
|
117
238
|
|
|
118
|
-
#
|
|
119
|
-
gscdump sync --site
|
|
239
|
+
# Limit concurrent day requests per table
|
|
240
|
+
gscdump sync --site sc-domain:example.com --concurrency 4 \
|
|
241
|
+
--tables pages,queries,page_queries,countries,dates
|
|
120
242
|
```
|
|
121
243
|
|
|
122
|
-
Sync
|
|
244
|
+
Sync skips completed dates; `--force` refreshes them.
|
|
245
|
+
Cross-process locks coordinate `sync`, `compact`, and `gc`.
|
|
246
|
+
Pagination follows Google's 25,000-row pages, subject to [Google's data limits](https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data).
|
|
123
247
|
|
|
124
248
|
## MCP server
|
|
125
249
|
|
|
@@ -136,7 +260,7 @@ Add to your Claude / VS Code config:
|
|
|
136
260
|
"mcpServers": {
|
|
137
261
|
"gscdump": {
|
|
138
262
|
"command": "npx",
|
|
139
|
-
"args": ["@gscdump/cli", "mcp"]
|
|
263
|
+
"args": ["-y", "@gscdump/cli", "mcp"]
|
|
140
264
|
}
|
|
141
265
|
}
|
|
142
266
|
}
|
|
@@ -145,20 +269,28 @@ Add to your Claude / VS Code config:
|
|
|
145
269
|
Then ask questions like:
|
|
146
270
|
|
|
147
271
|
- "What pages lost traffic this week?"
|
|
148
|
-
- "Find
|
|
272
|
+
- "Find queries in striking distance (positions 4 to 20)."
|
|
149
273
|
- "Which queries have cannibalization issues?"
|
|
150
274
|
- "Compare this month vs last month for /blog/ pages."
|
|
151
275
|
|
|
152
276
|
## Auth
|
|
153
277
|
|
|
154
|
-
|
|
278
|
+
This section covers local Google credentials. See [shared authentication](#hosted-and-local-authentication) for cloud mode and [Bing](#bing) for Bing credentials.
|
|
279
|
+
`gscdump init --mode local` configures Google OAuth and a Store directory.
|
|
280
|
+
Credentials are saved under `~/.config/gscdump/` on XDG systems, or the platform equivalent.
|
|
281
|
+
Use `gscdump auth login --mode local` to connect Google and save local mode.
|
|
282
|
+
|
|
283
|
+
Local Google browser login uses a temporary listener on `127.0.0.1` with a random port.
|
|
284
|
+
Each attempt uses state validation and PKCE S256 to bind the authorization response to that attempt.
|
|
285
|
+
The listener closes after authorization, denial, or a five-minute timeout.
|
|
155
286
|
|
|
156
287
|
For manual setup:
|
|
157
288
|
|
|
158
289
|
1. Create a Google Cloud project.
|
|
159
|
-
2. Enable **Search Console API
|
|
290
|
+
2. Enable **Search Console API**, **Web Search Indexing API**, and **Site Verification API**.
|
|
160
291
|
3. Create OAuth2 credentials (Desktop app).
|
|
161
|
-
4. Run `gscdump init
|
|
292
|
+
4. Run `gscdump init --mode local` to configure credentials and a Store directory.
|
|
293
|
+
5. Run `gscdump auth login --mode local` to save local mode.
|
|
162
294
|
|
|
163
295
|
### BYOK (Bring Your Own Key)
|
|
164
296
|
|
|
@@ -168,39 +300,70 @@ Skip `init` entirely by setting env vars. Either path works (`GSC_*` preferred,
|
|
|
168
300
|
# Option A: raw bearer token (e.g., from gcloud or another OAuth flow)
|
|
169
301
|
export GSC_ACCESS_TOKEN=ya29...
|
|
170
302
|
|
|
171
|
-
# Option B: refresh-token flow (
|
|
303
|
+
# Option B: refresh-token flow (OAuth refresh credentials)
|
|
172
304
|
export GSC_CLIENT_ID=...
|
|
173
305
|
export GSC_CLIENT_SECRET=...
|
|
174
306
|
export GSC_REFRESH_TOKEN=...
|
|
175
307
|
```
|
|
176
308
|
|
|
177
|
-
|
|
309
|
+
`gscdump auth status` shows which credential source is active.
|
|
310
|
+
`auth login --mode local` skips OAuth when it finds BYOK credentials and saves local mode.
|
|
178
311
|
|
|
179
312
|
### Service account
|
|
180
313
|
|
|
181
|
-
For CI / headless usage, point `gscdump` at a service-account JSON key.
|
|
314
|
+
For CI / headless usage, point `gscdump` at a service-account JSON key. Grant the service account access to each Site in Search Console under Settings → Users and permissions.
|
|
182
315
|
|
|
183
316
|
```bash
|
|
184
|
-
gscdump auth login --service-account ./gsc-sa.json # smoke-test the key
|
|
185
|
-
export GOOGLE_APPLICATION_CREDENTIALS
|
|
317
|
+
gscdump auth login --mode local --service-account ./gsc-sa.json # smoke-test the key
|
|
318
|
+
export GOOGLE_APPLICATION_CREDENTIALS=/absolute/path/to/gsc-sa.json
|
|
186
319
|
gscdump sites
|
|
187
320
|
```
|
|
188
321
|
|
|
189
322
|
### Headless OAuth
|
|
190
323
|
|
|
191
|
-
|
|
324
|
+
If you want to open the authorization URL yourself, disable automatic browser opening:
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
gscdump auth login --mode local --no-browser
|
|
328
|
+
# Open the printed URL in your browser.
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
This uses the same Desktop application OAuth client and loopback flow as browser login.
|
|
332
|
+
Google's device flow does not support the required scopes.
|
|
333
|
+
|
|
334
|
+
If the CLI runs on another host, forward its printed loopback port before opening the URL.
|
|
335
|
+
Keep the login command running on that host.
|
|
336
|
+
For example, if the CLI prints port `45678`, run this command on your browser host:
|
|
192
337
|
|
|
193
338
|
```bash
|
|
194
|
-
|
|
195
|
-
# → opens a verification URL on any device; type the displayed user code
|
|
339
|
+
ssh -N -L 45678:127.0.0.1:45678 user@host
|
|
196
340
|
```
|
|
197
341
|
|
|
342
|
+
Replace `user@host` with the CLI host.
|
|
343
|
+
Keep the forwarding command running until login completes.
|
|
344
|
+
Then open the printed authorization URL in your browser.
|
|
345
|
+
For containers or WSL, forward the same port to the environment running the CLI.
|
|
346
|
+
If you cannot forward loopback traffic, use a refresh token or service account.
|
|
347
|
+
|
|
348
|
+
See Google's [native application OAuth guide](https://developers.google.com/identity/protocols/oauth2/native-app)
|
|
349
|
+
and [device flow scope limits](https://developers.google.com/identity/protocols/oauth2/limited-input-device#allowedscopes).
|
|
350
|
+
|
|
351
|
+
Indexing notifications apply only to eligible job or livestream pages.
|
|
352
|
+
See [URL inspection and indexing](../../docs/guides/url-indexing.md).
|
|
353
|
+
|
|
198
354
|
## Related
|
|
199
355
|
|
|
200
|
-
- [`gscdump`](../gscdump)
|
|
201
|
-
- [`@gscdump/engine`](../engine)
|
|
202
|
-
- [`@gscdump/analysis`](../analysis)
|
|
356
|
+
- [`gscdump`](../gscdump) : Google and Bing clients with a typed query builder.
|
|
357
|
+
- [`@gscdump/engine`](../engine) : Storage engine the CLI syncs into.
|
|
358
|
+
- [`@gscdump/analysis`](../analysis) : SEO Analyzers (row-based + DuckDB-native).
|
|
203
359
|
|
|
204
360
|
## License
|
|
205
361
|
|
|
206
362
|
[MIT](../../LICENSE)
|
|
363
|
+
|
|
364
|
+
## CLI charts
|
|
365
|
+
|
|
366
|
+
Human output shares chart styles and metric units across Analyzers, Reports, and Store stats.
|
|
367
|
+
Use `gscdump query --format table` for query charts.
|
|
368
|
+
JSON and CSV keep their existing payloads and defaults.
|
|
369
|
+
See [CLI charts](../../docs/guides/cli-charts.md) for examples and data limits.
|
package/bin/gscdump.mjs
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import process from 'node:process'
|
|
4
|
+
|
|
5
|
+
import ('../dist/cli.mjs').then(({ runCli }) => runCli()).catch((error) => {
|
|
6
|
+
console.error(error instanceof Error ? error.message : String(error))
|
|
7
|
+
process.exitCode = 1
|
|
8
|
+
})
|
package/dist/analysis-local.mjs
CHANGED
|
@@ -3,8 +3,8 @@ import { createLocalStore } from "./local-store.mjs";
|
|
|
3
3
|
import { createCommandContext } from "./context.mjs";
|
|
4
4
|
import { LocalStoreUnsupportedError } from "./error-handler.mjs";
|
|
5
5
|
import process from "node:process";
|
|
6
|
-
import { join } from "node:path";
|
|
7
6
|
import { readdir } from "node:fs/promises";
|
|
7
|
+
import { join } from "node:path";
|
|
8
8
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
9
9
|
import { createGscApiQuerySource } from "@gscdump/engine-gsc-api";
|
|
10
10
|
import { AnalyzerCapabilityError, runAnalyzerFromSource } from "@gscdump/engine/analyzer";
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { useCliRuntime } from "./runtime.mjs";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
const apiRootSchema = z.url().transform((value) => value.replace(/\/+$/, "")).refine((value) => {
|
|
7
|
+
const url = new URL(value);
|
|
8
|
+
return !url.username && !url.password && !url.search && !url.hash && (url.protocol === "https:" || url.protocol === "http:" && [
|
|
9
|
+
"localhost",
|
|
10
|
+
"127.0.0.1",
|
|
11
|
+
"[::1]"
|
|
12
|
+
].includes(url.hostname));
|
|
13
|
+
}, "Use HTTPS, or HTTP on loopback, for the API root");
|
|
14
|
+
const stateSchema = z.discriminatedUnion("_tag", [z.object({ _tag: z.literal("Local") }), z.object({
|
|
15
|
+
_tag: z.literal("Cloud"),
|
|
16
|
+
apiRoot: apiRootSchema,
|
|
17
|
+
apiKey: z.string().trim().regex(/^gsd_user_\S+$/)
|
|
18
|
+
})]);
|
|
19
|
+
function parseAuthMode(value) {
|
|
20
|
+
if (value === void 0 || value === "") return void 0;
|
|
21
|
+
if (value === "cloud" || value === "local") return value;
|
|
22
|
+
throw new Error("Authentication mode must be cloud or local.");
|
|
23
|
+
}
|
|
24
|
+
function parseAuthentication(value) {
|
|
25
|
+
const parsed = stateSchema.safeParse(value);
|
|
26
|
+
if (!parsed.success) throw new Error("Invalid authentication. Use a gscdump user API key and a trusted API root.");
|
|
27
|
+
return parsed.data;
|
|
28
|
+
}
|
|
29
|
+
async function saveAuthentication(state) {
|
|
30
|
+
const parsed = parseAuthentication(state);
|
|
31
|
+
const configDir = useCliRuntime().configDir;
|
|
32
|
+
await fs.mkdir(configDir, {
|
|
33
|
+
recursive: true,
|
|
34
|
+
mode: 448
|
|
35
|
+
});
|
|
36
|
+
const target = path.join(configDir, "authentication.json");
|
|
37
|
+
const temporary = `${target}.${randomUUID()}.tmp`;
|
|
38
|
+
try {
|
|
39
|
+
await fs.writeFile(temporary, JSON.stringify(parsed, null, 2), {
|
|
40
|
+
mode: 384,
|
|
41
|
+
flag: "wx"
|
|
42
|
+
});
|
|
43
|
+
await fs.rename(temporary, target);
|
|
44
|
+
} finally {
|
|
45
|
+
await fs.rm(temporary, { force: true });
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async function clearAuthentication() {
|
|
49
|
+
await fs.rm(path.join(useCliRuntime().configDir, "authentication.json"), { force: true });
|
|
50
|
+
}
|
|
51
|
+
async function resolveAuthentication() {
|
|
52
|
+
const runtime = useCliRuntime();
|
|
53
|
+
const env = runtime.environment;
|
|
54
|
+
const mode = runtime.authModeOverride ?? parseAuthMode(env.GSCDUMP_AUTH_MODE);
|
|
55
|
+
if (mode === "local") return { _tag: "Local" };
|
|
56
|
+
const body = await fs.readFile(path.join(runtime.configDir, "authentication.json"), "utf8").catch((error) => {
|
|
57
|
+
if (error.code === "ENOENT") return null;
|
|
58
|
+
throw error;
|
|
59
|
+
});
|
|
60
|
+
const saved = body === null ? null : stateSchema.safeParse(JSON.parse(body));
|
|
61
|
+
if (saved && !saved.success) throw new Error("Saved authentication is invalid. Run `gscdump auth login --mode cloud` or `--mode local`.");
|
|
62
|
+
const state = saved?.success ? saved.data : null;
|
|
63
|
+
if (mode !== "cloud" && state?._tag === "Local") return state;
|
|
64
|
+
if (env.GSCDUMP_API_KEY) {
|
|
65
|
+
const parsed = stateSchema.safeParse({
|
|
66
|
+
_tag: "Cloud",
|
|
67
|
+
apiKey: env.GSCDUMP_API_KEY,
|
|
68
|
+
apiRoot: env.GSCDUMP_API_ROOT ?? (state?._tag === "Cloud" ? state.apiRoot : "https://gscdump.com/api")
|
|
69
|
+
});
|
|
70
|
+
if (!parsed.success) throw new Error("Invalid hosted authentication. Set GSCDUMP_API_KEY to a gscdump user API key.");
|
|
71
|
+
return parsed.data;
|
|
72
|
+
}
|
|
73
|
+
if (state?._tag === "Cloud") {
|
|
74
|
+
if (env.GSCDUMP_API_ROOT && env.GSCDUMP_API_ROOT.replace(/\/+$/, "") !== state.apiRoot) throw new Error("The API root changed. Supply GSCDUMP_API_KEY explicitly for the new API root.");
|
|
75
|
+
return state;
|
|
76
|
+
}
|
|
77
|
+
if (mode === "cloud") throw new Error("Hosted credentials are missing. Run `gscdump auth login --mode cloud`.");
|
|
78
|
+
return { _tag: "Local" };
|
|
79
|
+
}
|
|
80
|
+
const accountSchema = z.object({
|
|
81
|
+
user: z.object({
|
|
82
|
+
publicId: z.string(),
|
|
83
|
+
email: z.string()
|
|
84
|
+
}),
|
|
85
|
+
sites: z.array(z.object({
|
|
86
|
+
siteId: z.string(),
|
|
87
|
+
siteUrl: z.string()
|
|
88
|
+
}).passthrough())
|
|
89
|
+
});
|
|
90
|
+
async function cloudRequest(state, route, options = {}) {
|
|
91
|
+
const response = await fetch(`${state.apiRoot}${route}`, {
|
|
92
|
+
...options,
|
|
93
|
+
headers: {
|
|
94
|
+
"Content-Type": "application/json",
|
|
95
|
+
"x-api-key": state.apiKey
|
|
96
|
+
},
|
|
97
|
+
signal: options.signal ?? AbortSignal.timeout(3e4),
|
|
98
|
+
redirect: "error"
|
|
99
|
+
});
|
|
100
|
+
if (!response.ok) throw Object.assign(/* @__PURE__ */ new Error(`Hosted request failed (${response.status}) for ${route.split("?")[0]}. Check \`gscdump auth status\`.`), {
|
|
101
|
+
statusCode: response.status,
|
|
102
|
+
retryAfter: response.headers.get("retry-after"),
|
|
103
|
+
response
|
|
104
|
+
});
|
|
105
|
+
return response.status === 204 ? void 0 : response.json();
|
|
106
|
+
}
|
|
107
|
+
async function getCloudAccount(state) {
|
|
108
|
+
const result = accountSchema.safeParse(await cloudRequest(state, "/cli/me"));
|
|
109
|
+
if (!result.success) throw new Error("The hosted API returned invalid account data.");
|
|
110
|
+
return result.data;
|
|
111
|
+
}
|
|
112
|
+
export { clearAuthentication, cloudRequest, getCloudAccount, parseAuthMode, parseAuthentication, resolveAuthentication, saveAuthentication };
|