@gscdump/cli 3.5.0 → 3.6.0

package/dist/skill.mjs

@@ -0,0 +1,45 @@
+import { cp, mkdir, stat } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+const SKILL_NAME = "gscdump";
+const SKILL_AGENTS = ["claude", "codex"];
+const AGENT_DIRECTORIES = {
+	claude: ".claude",
+	codex: ".codex"
+};
+function skillSourceDirectory(moduleUrl = import.meta.url) {
+	return path.join(path.dirname(fileURLToPath(moduleUrl)), "..", "skills", SKILL_NAME);
+}
+function skillDestination(homeDirectory, agent) {
+	return path.join(homeDirectory, AGENT_DIRECTORIES[agent], "skills", SKILL_NAME);
+}
+async function installSkill(options) {
+	const source = options.sourceDirectory ?? skillSourceDirectory();
+	if (!(await stat(source).catch(() => {
+		return null;
+	}))?.isDirectory()) return {
+		_tag: "Err",
+		reason: "source_missing",
+		message: `The packaged skill is missing at ${source}. Reinstall @gscdump/cli, then run the command again.`
+	};
+	const destination = options.target ? path.join(options.target, SKILL_NAME) : skillDestination(options.homeDirectory, options.agent);
+	return mkdir(path.dirname(destination), { recursive: true }).then(() => cp(source, destination, {
+		recursive: true,
+		force: true
+	})).then(() => ({
+		_tag: "Ok",
+		installation: {
+			agent: options.agent,
+			source,
+			destination
+		}
+	})).catch((cause) => {
+		const detail = cause instanceof Error ? cause.message : String(cause);
+		return {
+			_tag: "Err",
+			reason: "write_failed",
+			message: `Could not write the skill to ${destination}: ${detail}`
+		};
+	});
+}
+export { SKILL_AGENTS, SKILL_NAME, installSkill, skillDestination, skillSourceDirectory };

package/dist/utils.mjs

@@ -1,10 +1,10 @@
 import { useCliRuntime } from "./runtime.mjs";
 import { version } from "./package.mjs";
 import process from "node:process";
-import os from "node:os";
 import fs from "node:fs/promises";
-import { Buffer } from "node:buffer";
+import os from "node:os";
 import { SearchTypes } from "gscdump/query";
+import { Buffer } from "node:buffer";
 const ALL_SEARCH_TYPES = Object.values(SearchTypes);
 const VERSION = version;
 function noSubcommandSelected(parent, subNames) {
@@ -65,13 +65,17 @@ function parseSearchType(value, flag = "--search-type") {
 	}
 	return v;
 }
+function parseIntegerOption(value, flag, minimum = 1) {
+	if (value === void 0 || value === null) return void 0;
+	const text = typeof value === "number" || typeof value === "string" ? String(value).trim() : "";
+	const parsed = Number(text);
+	if (!/^\d+$/.test(text) || !Number.isSafeInteger(parsed) || parsed < minimum) throw new Error(`${flag} must be ${minimum === 0 ? "a non-negative integer" : "a positive integer"}.`);
+	return parsed;
+}
 const ANSI_RE = /\x1B\[[0-9;]*m/g;
 function setNoColor(disable) {
 	if (disable) useCliRuntime().colorEnabled = false;
 }
-function configureColor(opts) {
-	if (opts.noColor || !opts.forceColor && !opts.stderrIsTTY) setNoColor(true);
-}
 function isColorEnabled() {
 	return useCliRuntime().colorEnabled;
 }
@@ -180,36 +184,4 @@ async function readUrlList(args) {
 	}
 	return [];
 }
-function exportToCSV(output) {
-	const sections = [];
-	if (output.pages?.data) sections.push(`# Pages\n${toCSV(output.pages.data, [
-		"url",
-		"clicks",
-		"impressions",
-		"ctr",
-		"position"
-	])}`);
-	if (output.keywords?.current) sections.push(`# Keywords (Current Period)\n${toCSV(output.keywords.current, [
-		"query",
-		"clicks",
-		"impressions",
-		"ctr",
-		"position"
-	])}`);
-	if (output.countries?.current) sections.push(`# Countries (Current Period)\n${toCSV(output.countries.current, [
-		"country",
-		"clicks",
-		"impressions",
-		"ctr",
-		"position"
-	])}`);
-	if (output.devices?.current) sections.push(`# Devices (Current Period)\n${toCSV(output.devices.current, [
-		"device",
-		"clicks",
-		"impressions",
-		"ctr",
-		"position"
-	])}`);
-	return sections.join("\n\n");
-}
-export { ALL_SEARCH_TYPES, OUTPUT_ARGS, VERSION, applyOutputMode, clearLine, color, configureColor, cyan, dim, displayPath, exportToCSV, formatAge, isColorEnabled, logger, noSubcommandSelected, parseSearchType, progressBar, readUrlList, runWithConcurrency, setNoColor, setQuiet, showSplash, toCSV, withConfiguredOutput };
+export { ALL_SEARCH_TYPES, OUTPUT_ARGS, VERSION, applyOutputMode, clearLine, color, cyan, dim, displayPath, formatAge, isColorEnabled, logger, noSubcommandSelected, parseIntegerOption, parseSearchType, progressBar, readUrlList, runWithConcurrency, setNoColor, setQuiet, showSplash, toCSV, withConfiguredOutput };

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@gscdump/cli",
   "type": "module",
-  "version": "3.5.0",
-  "description": "CLI for Google Search Console - dump, query, and run MCP server",
+  "version": "3.6.0",
+  "description": "CLI for Google Search Console and Bing with hosted or local authentication, data exports, and an MCP server",
   "author": {
     "name": "Harlan Wilton",
     "email": "harlan@harlanzw.com",
@@ -35,30 +35,32 @@
   },
   "files": [
     "bin",
-    "dist"
+    "dist",
+    "skills"
   ],
   "engines": {
     "node": ">=22"
   },
   "dependencies": {
-    "@clack/prompts": "^1.7.0",
-    "@duckdb/node-api": "1.5.1-r.2",
-    "@gscdump/analysis": "^3.5.0",
-    "@gscdump/engine": "^3.5.0",
-    "@gscdump/engine-gsc-api": "^3.5.0",
-    "@gscdump/sdk": "^3.5.0",
+    "@clack/prompts": "^1.8.0",
+    "@duckdb/node-api": "1.5.5-r.4",
+    "@gscdump/analysis": "^3.6.0",
+    "@gscdump/engine": "^3.6.0",
+    "@gscdump/engine-gsc-api": "^3.6.0",
+    "@gscdump/sdk": "^3.6.0",
     "@modelcontextprotocol/sdk": "^1.30.0",
     "citty": "^0.2.2",
     "consola": "^3.4.2",
     "google-auth-library": "^11.0.2",
-    "gscdump": "^3.5.0",
+    "gscdump": "^3.6.0",
     "ofetch": "^1.5.1",
     "open": "^11.0.2",
     "sitemapd": "^0.2.2",
-    "zod": "^4.5.4"
+    "string-width": "^8.2.2",
+    "zod": "^4.6.1"
   },
   "devDependencies": {
-    "vitest": "^4.1.11"
+    "vitest": "^5.0.0"
   },
   "scripts": {
     "build": "obuild",

package/skills/gscdump/SKILL.md

@@ -0,0 +1,287 @@
+---
+name: gscdump
+description: Drive the `gscdump` CLI for Google Search Console and Bing with cloud or local authentication. Sync Google rows to a local Store, export Bing datasets, run SEO Analyzers and Reports, inspect Indexing Evidence, and manage sitemaps. Use when the user mentions gscdump, Search Console data, Bing Webmaster data, or GSC automation.
+---
+
+# gscdump CLI
+
+`gscdump` reads Google Search Console and Bing with hosted or local authentication.
+It keeps a local Parquet Store for Google rows. Every command has `--help`.
+
+## Authentication mode
+
+Check `gscdump auth status --json` before queries. Reuse the user's selected mode.
+
+| Mode | Credentials | Query path |
+| --- | --- | --- |
+| `cloud` | gscdump user API key | `https://gscdump.com/api` uses saved Search Engine connections |
+| `local` | Google OAuth/service account or Bing API key/OAuth | Calls the Search Engine directly |
+
+`--mode cloud|local` overrides one invocation. `GSCDUMP_AUTH_MODE` also overrides the saved mode.
+A successful login saves the mode per profile.
+If no mode is saved, `GSCDUMP_API_KEY` selects cloud mode.
+With neither source, the CLI defaults to local mode.
+When a saved mode exists, it remains selected unless an explicit override applies.
+Never switch modes to bypass an authentication failure.
+`GSCDUMP_API_ROOT` defaults to `https://gscdump.com/api`. Supply the API key explicitly when changing a saved API root.
+
+```sh
+# The user supplies a user API key from gscdump.com settings.
+gscdump auth login --mode cloud
+gscdump bing sites --json
+gscdump bing login --site s_SITE_ID
+gscdump bing dump --site s_SITE_ID --out ./bing-export --format json
+
+# Local Google and Bing credentials stay separate.
+gscdump auth login --mode local
+gscdump bing login --mode local
+gscdump bing dump --site https://example.com/ --out ./bing-export
+```
+
+Hosted Bing login opens the existing connection flow on gscdump.com.
+Local Bing login uses `BING_API_KEY` or a password prompt.
+Local `--oauth` uses `BING_CLIENT_ID`, `BING_CLIENT_SECRET`, and a registered loopback callback.
+The default callback is `http://127.0.0.1:53683/oauth/bing`. `BING_ACCESS_TOKEN` accepts an existing OAuth access token.
+After cloud Bing login opens a browser, use `bing status --site s_SITE_ID` to confirm the connection.
+
+`auth logout` removes the saved mode and saved Google and Bing credentials.
+`bing logout --mode local` removes only saved Bing credentials. Environment credentials remain active until unset.
+
+Hosted Bing commands use the API's plan and preview access rules.
+Hosted connection verification uses `bing verify --site s_SITE_ID`.
+Google Indexing API and Site Verification commands require local mode.
+Hosted sitemap membership and history require hosted credentials.
+
+## Data boundaries
+
+- `sync`, `query --live`, `analyze --live`, `report --live`, `sites`,
+  `sitemaps`, and `inspect` use the selected authentication mode.
+- Google Indexing API requests require local credentials. `indexing quota` only prints documented limits and needs no authentication.
+- `query`, `analyze`, `report`, `dump`, and `store` read the local Store by
+  default. If the Store has no rows for the Site, sync first or pass `--live`.
+- Google returns a 2 to 3 day data delay. Default windows end three days ago.
+- Google omits low-volume rows. Pagination cannot recover them.
+- URL Inspection is limited to 2,000 requests per Site per day.
+- `indexing submit` and `indexing remove` are only for job posting and
+  livestream pages. Google rejects other content.
+
+## Get the binary
+
+```sh
+npx -y @gscdump/cli --version    # no install
+npm install -g @gscdump/cli      # or pnpm add -g
+```
+
+Use Node 22.13 or later in the 22 release line, or Node 24 or later.
+`gscdump` alone is the library; `@gscdump/cli`
+provides the command.
+
+## Install this skill
+
+```sh
+gscdump skill install --agent claude    # Codex: --agent codex
+```
+
+After upgrading the CLI, run this command again to update the installed skill.
+The command prints where it wrote the skill. Clients without a skill
+directory can read `gscdump --help` and `gscdump <command> --help` instead.
+
+## Local Google authentication
+
+Check first. Never run `init` when credentials already work.
+
+```sh
+gscdump auth status
+gscdump doctor --json
+```
+
+If local Google credentials are missing, use one of these paths:
+
+| Path | When | Command |
+| --- | --- | --- |
+| Environment token | The user already has an OAuth access token | `export GSC_ACCESS_TOKEN=ya29...` |
+| Refresh token | CI or a headless machine with OAuth client credentials | `export GSC_CLIENT_ID=... GSC_CLIENT_SECRET=... GSC_REFRESH_TOKEN=...` |
+| Service account | CI with a service-account key that has Site access | `export GOOGLE_APPLICATION_CREDENTIALS=/abs/path/key.json` |
+| Interactive OAuth | A person is present | `gscdump init --mode local` |
+
+`init` needs a Google Cloud OAuth client of type Desktop app. Ask the user to
+run it; do not guess client credentials. Use `gscdump auth login --mode local --no-browser`
+when a browser cannot open.
+Run `gscdump auth login --mode local` to save local mode after configuring credentials.
+
+`--profile <name>` or `GSCDUMP_PROFILE` isolates the selected mode and Google, Bing, and cloud credentials.
+
+## Site identifiers
+
+For Google, use the exact value that `gscdump sites` prints.
+
+- Domain property: `sc-domain:example.com`
+- URL-prefix property: `https://example.com/` (trailing slash included)
+
+For cloud Bing commands, use a Site ID from `gscdump bing sites`, such as `s_SITE_ID`.
+For local Bing commands, use the full verified Site URL from `gscdump bing sites --mode local`.
+Bing commands require their own explicit `--site`; the Google `defaultSite` setting does not select a Bing Site.
+
+Set a default once to drop `--site` from later commands:
+
+```sh
+gscdump config set defaultSite sc-domain:example.com
+```
+
+## Output
+
+Pass `--json` on every command that supports it. `query` uses
+`--format json` and prints rows to stdout. Progress goes to stderr, so stdout
+stays parseable. `--quiet` drops progress lines.
+
+Parse JSON. Never scrape human output.
+
+## Commands
+
+| Command | Use it for |
+| --- | --- |
+| `gscdump sites` | List Google Sites and permission levels |
+| `gscdump bing login`, `status`, `logout` | Manage Bing authentication and check connections |
+| `gscdump bing sites` | List Bing Sites and connection details |
+| `gscdump bing dump` | Export Bing traffic, pages, keywords, and crawl data |
+| `gscdump bing inspect` | Read Bing Indexing Evidence for one URL |
+| `gscdump bing verify` | Check and activate a cloud Bing connection |
+| `gscdump sync` | Copy Search Console rows into the local Store |
+| `gscdump query` | Rows by page, query, date, country, or device |
+| `gscdump analyze <id>` | One Analyzer over the Store or live rows |
+| `gscdump report <id>` | A Report that composes several Analyzers |
+| `gscdump inspect <url>` | URL Inspection with Indexing Evidence |
+| `gscdump sitemaps` | List, submit, delete, and probe sitemaps |
+| `gscdump indexing` | Indexing API notifications and quota |
+| `gscdump dump` | Export Store tables to Parquet, CSV, JSON, or NDJSON |
+| `gscdump store` | Store stats, compaction, garbage collection, resets |
+| `gscdump entities` | Snapshot URL inspections into the entity store |
+| `gscdump config` | Defaults such as `defaultSite`, `dataDir`, `defaultLimit` |
+| `gscdump profile` | Separate credential and config directories |
+| `gscdump auth` | `status`, `login`, `logout`, `refresh` |
+| `gscdump doctor` | Health checks for auth, scopes, Store, and reachability |
+| `gscdump init` | Interactive first-time setup |
+| `gscdump mcp` | Start Google MCP tools with the selected authentication |
+| `gscdump skill install` | Copy this skill into an agent skill directory |
+| `gscdump papercut` | Report a CLI problem to gscdump.com |
+
+`gscdump login`, `gscdump logout`, and `gscdump status` are top-level aliases of
+the matching `auth` subcommands.
+The MCP server does not expose Bing tools. Use `gscdump bing` commands through this skill.
+
+## Sync before local analysis
+
+```sh
+gscdump sync --site sc-domain:example.com --days 90 \
+  --tables pages,queries,page_queries,countries
+gscdump sync --site sc-domain:example.com --status
+```
+
+- Pass an explicit `--tables` list. The default list has a known daily-totals
+  limitation.
+- `--full` backfills the 450 days Google keeps.
+- Sync skips completed dates. `--force` refreshes them. `--retry-failed`
+  reruns only failed dates.
+- `--dry-run` prints the planned work without calling Google.
+
+## Query rows
+
+```sh
+gscdump query --site sc-domain:example.com --dimensions page,query \
+  --start 2026-08-01 --end 2026-08-28 --limit 1000 --format json
+```
+
+- Dimension names are singular: `page`, `query`, `date`, `country`, `device`.
+- Filters: `--query`, `--page`, `--country`, `--device`,
+  `--search-appearance`. Prefixes: bare equals, `~` contains, `!~` not
+  contains, `re:` regex, `!re:` not regex, `!` not equals.
+- `--live` bypasses the Store. `--search-type`, `--data-state`, and
+  `--aggregation-type` apply to live mode only.
+- `--explain` prints the request body or planned SQL without executing.
+- `--sql` runs raw DuckDB SQL over the Store with `{{FILES}}` as the file list.
+
+## Analyze and report
+
+```sh
+gscdump report list --json
+gscdump report opportunities --site sc-domain:example.com --json
+gscdump report movers --site sc-domain:example.com --period 28d --vs prev-period --json
+gscdump analyze list --json
+gscdump analyze striking-distance --site sc-domain:example.com --json
+```
+
+- Report ids: `brand`, `growth`, `health`, `movers`, `opportunities`,
+  `pre-publish`, `risks`, `triage`.
+- `--period` takes `7d`, `28d`, `90d`, `mtd`, `ytd`, or `custom` with
+  `--start` and `--end`. `--vs` takes `none`, `prev-period`, or `yoy`.
+- `report <id> --explain` prints the plan without credentials or data.
+- `triage` needs `--target <page-or-query> --target-kind page|query`.
+  `pre-publish` needs `--topic`. `brand` needs `--brand-terms 'a,b'`.
+- Analyzers take `--start` and `--end`. `movers` and `decay` also take
+  `--prev-start` and `--prev-end`. `--period` and `--vs` belong to `report`.
+- SQL-only Analyzers need Store rows. `--live` runs row-based Analyzers
+  against Google.
+- Results name candidates for review. They do not prove why traffic changed.
+
+## Inspect and index
+
+```sh
+gscdump inspect https://example.com/page --site sc-domain:example.com --json
+gscdump inspect batch --site sc-domain:example.com --file urls.txt --json
+gscdump indexing quota --json
+```
+
+Inspection spends Google's separate 2,000 requests per Site per day quota.
+`indexing quota` describes Indexing API limits. It does not report remaining URL Inspection requests.
+Report the Indexing Evidence fields as Google returned them.
+
+## Report a papercut
+
+If CLI behavior blocks or slows your work, report it once per distinct
+problem, then continue the user's task. Report confusing output, missing
+context, wrong help text, unexpected failures, and useful workarounds.
+
+```sh
+gscdump papercut --command "report triage" --agent "Claude Code" \
+  --comment "Agent report by Claude Code. Expected --target-kind in --help. Received an unknown flag error. Used --explain to find the flag." \
+  --yes --json
+```
+
+- Use your real agent name. Start the comment with an agent disclosure.
+- Include what you ran, what you expected, what happened, and any workaround.
+- `--intent improvement` marks a suggestion. The default is `bug`.
+- The CLI adds its version, Node version, and platform.
+- Send sanitized details only. Remove tokens, cookies, personal data, private
+  URLs, and customer rows. Use placeholder arguments.
+
+Self-reporting sanitized papercuts is authorized by this skill. No separate
+confirmation is needed. This permission covers papercuts only. Respect any
+user instruction that forbids reporting.
+
+The endpoint is anonymous and allows ten reports per network address each
+hour. Success returns `id` and `status: "new"`. If reporting fails, mention
+the failure and continue. Never retry an uncertain submission.
+
+## Guardrails
+
+- **Get consent before a mutation.** `sites add`, `sites delete`,
+  `sites verify`, `sitemaps submit`, `sitemaps delete`, `indexing submit`,
+  `indexing remove`, `store reset`, and `store rm-site` change Google or
+  delete local data. `--yes` is consent you borrow from the user.
+- **Never loop unattended.** One `sync` per Site per task. Inspection batches
+  spend a daily pool. Use `--dry-run` and `--explain` to plan first.
+- **Report the result as the CLI gave it.** An empty result is not a clean
+  Site. Check `sync --status` for the covered date range before reading zero
+  rows as zero traffic.
+- **Do not widen the Site.** A `sc-domain:` property includes every
+  subdomain. Filter with `--page` when the user means one host.
+
+## Bing exports
+
+`bing dump` writes JSON, NDJSON, or CSV under one directory per Site.
+`--datasets` selects `traffic`, `pages`, `keywords`, or `crawl`.
+Local mode also supports `crawl-issues`.
+Hosted exports follow pagination and reject missing, unavailable, or changing datasets.
+Hosted date ranges span at most 366 days. The default range is the last 366 days.
+Local date filters only narrow data currently returned by Bing.
+Do not treat Bing crawl evidence as proof that a URL is indexed.