crawlmark-mcp 0.0.0-stage → 0.1.1

package/.env.example

@@ -0,0 +1,177 @@
+# Copy to .env in the config home (gitignored), or run `crawlmark-cli setup`, which writes it for you with mode 0600.
+# Relative paths below resolve against the config home, not the working directory.
+#
+# Config home: CRAWLMARK_HOME if set; else the project root for a source checkout (a .git directory, src/ next to dist/,
+# or an existing .env or config/clients.yaml there); else ~/.crawlmark. CRAWLMARK_HOME itself can't be set in this file
+# (the file is found through it): set it in your shell or the MCP client's server env. `crawlmark-cli setup migrate --to <dir>`
+# copies an existing home somewhere else.
+# CRAWLMARK_HOME=~/.crawlmark
+
+# Google (Search Console + GA4) is optional. Without it, DataForSEO, Lighthouse and axe still work.
+# How the server signs in to Google: oauth or service_account. Leave unset to auto-detect:
+#   service_account if GOOGLE_APPLICATION_CREDENTIALS is set, else oauth if both OAuth client vars are set, else no Google.
+#   oauth: a person signs in once with `crawlmark-cli auth login`; the token is stored at DATA_DIR/google-token.json.
+#   service_account: uses the JSON key at GOOGLE_APPLICATION_CREDENTIALS (also for unattended jobs).
+GOOGLE_AUTH_MODE=
+
+# OAuth client of type "Desktop app" from Google Cloud > Google Auth Platform > Clients. Required in oauth mode.
+GOOGLE_OAUTH_CLIENT_ID=
+GOOGLE_OAUTH_CLIENT_SECRET=
+
+# Path to a Google service-account JSON key. Required in service_account mode, ignored in oauth mode.
+# Setting it turns on service_account mode when GOOGLE_AUTH_MODE is unset.
+GOOGLE_APPLICATION_CREDENTIALS=
+
+# DataForSEO API credentials from https://app.dataforseo.com/api-access (the API password, not the account password).
+# Needed for paid tools. `crawlmark-cli setup --step dataforseo` checks them for free and saves them.
+DATAFORSEO_LOGIN=your-login@example.com
+DATAFORSEO_PASSWORD=your-api-password
+
+# Sandbox is free and returns fake data. Use https://api.dataforseo.com/v3 for real data.
+DATAFORSEO_BASE_URL=https://sandbox.dataforseo.com/v3
+# Alternative to the URL: DATAFORSEO_LIVE=true selects the live API when DATAFORSEO_BASE_URL is unset (the Claude Desktop bundle uses this).
+# DATAFORSEO_LIVE=false
+
+# Default provider for paid keyword, ranking, competitor and backlink data: dataforseo, ahrefs or semrush (any case).
+# Blank means dataforseo. An unknown value doesn't stop the server; setup_status and every paid call report it.
+# A client's dataProvider in clients.yaml overrides it. Live SERPs, on-page checks and crawls always use DataForSEO.
+# `crawlmark-cli setup --step provider` sets it.
+SEO_DATA_PROVIDER=
+
+# Ahrefs API v3 key (optional): keyword volumes, rankings, competitors and backlinks from Ahrefs instead of DataForSEO.
+# Needs the Lite plan or higher. Create it under Account settings > API keys (https://app.ahrefs.com/account/api-keys;
+# only workspace owners and admins can open that page). `crawlmark-cli setup --step ahrefs` checks it for free and saves it.
+AHREFS_API_KEY=
+# Ahrefs API units crawlmark may use per calendar month (UTC), across all clients, and per client by default
+# (a client's monthlyUnits.ahrefs in clients.yaml overrides the per-client value). Units are never converted to dollars.
+AHREFS_MONTHLY_UNIT_CAP=50000
+DEFAULT_CLIENT_MONTHLY_AHREFS_UNITS=10000
+
+# Semrush (optional): keyword volume (v4 Get Keyword Metrics, 20 units per keyword) and backlinks (v4 overview) from Semrush.
+# Semrush's v4 API has nothing else, so rankings, competitors and visibility come from DataForSEO or Ahrefs. The API needs
+# a plan that includes API units (50,000 come with SEO Classic Pro and Guru, and Semrush One Starter and Pro+) or a bought
+# unit package. SEMRUSH_API_KEY is a v4 key, from your profile > API keys; Semrush's older v3 keys aren't supported.
+# Semrush has no free key check, so the key is checked by the first paid call. `crawlmark-cli setup --step semrush` saves it.
+# (SEMRUSH_V4_API_KEY, the name 0.1.0 used for the v4 key, is still read and wins if both are set: in 0.1.0 SEMRUSH_API_KEY
+# held the old v3 key. `setup --step semrush` saves SEMRUSH_API_KEY and removes SEMRUSH_V4_API_KEY.)
+SEMRUSH_API_KEY=
+# Semrush API units crawlmark may use per calendar month (UTC), across all clients, and per client by default
+# (a client's monthlyUnits.semrush in clients.yaml overrides the per-client value).
+SEMRUSH_MONTHLY_UNIT_CAP=20000
+DEFAULT_CLIENT_MONTHLY_SEMRUSH_UNITS=5000
+
+# Testing only (leave unset): point the Ahrefs, Semrush, CrUX and Bing clients at a local fake server. Read from the shell
+# environment only; values in this file are ignored, and setup never writes them. https, or plain http to
+# 127.0.0.1 / localhost / [::1] only; the server and `setup` print a stderr line while one is in use.
+# AHREFS_BASE_URL=
+# SEMRUSH_BASE_URL=
+# CRUX_BASE_URL=
+# BING_BASE_URL=
+# CRAWLER_IPRANGES_BASE_URL=
+
+# PageSpeed Insights API key (optional; adds real-user field data to lighthouse).
+PSI_API_KEY=
+
+# Chrome UX Report API key (optional): Core Web Vitals history for crux_history, the cwv report section and
+# run_audit cruxHistory. A Google Cloud API key with the Chrome UX Report API enabled. Unset, PSI_API_KEY is tried
+# (it works when its project has that API enabled). `crawlmark-cli setup --step crux` checks it for free and saves it.
+CRUX_API_KEY=
+
+# Bing Webmaster Tools API key (optional, free): bing_report and the bing_overview report section. In Bing Webmaster
+# Tools: Settings > API Access > Generate API Key (one key per user, for all your verified sites). crawlmark only
+# calls read methods with it. `crawlmark-cli setup --step bing` checks it for free and saves it.
+BING_WEBMASTER_API_KEY=
+
+# AI-search visibility (optional, PAID, off by default): ai_visibility and the ai_visibility report section, from
+# DataForSEO's LLM Mentions API. $0.10 per request plus $0.001 per row (no request costs less than $0.10). DataForSEO has
+# required a $100 monthly top-up for LLM Mentions access; its 2026-07-07 update says that was removed, so check your
+# account first. The sandbox serves it free. Only "true" turns it on. Its own monthly cap across all clients (USD),
+# checked on top of the client and global caps:
+AI_VISIBILITY_ENABLED=
+AI_VISIBILITY_MONTHLY_BUDGET_USD=2
+
+# Google indexing (free): URL Inspection requests per Search Console property per Pacific-time day, for
+# inspect_index_coverage, `crawlmark-cli index-coverage` and the indexing_issues report section (gsc_inspect_url and
+# run_audit count too). Google allows 2,000 a day and 600 a minute per property; the defaults leave headroom for your
+# other tools. Max 2000 and 600.
+GSC_INSPECTION_DAILY_BUDGET=1000
+GSC_INSPECTION_PER_MINUTE=60
+
+# A stored provider snapshot (backlinks, keyword visibility, competitors) this many days old or newer is reused instead
+# of a new paid call by the backlinks, organic_visibility and competitor_overview sections and tools and seo-snapshot;
+# 0 always fetches. Max 31.
+SNAPSHOT_REUSE_DAYS=7
+
+# The client list. Discovered Search Console / GA4 domains are added to it automatically, and you can
+# add your own (`crawlmark-cli clients add <domain>`). Created when first needed. See config/clients.example.yaml.
+CLIENTS_FILE=./config/clients.yaml
+
+# Defaults for discovered clients (and clients.yaml entries that leave these out).
+# DataForSEO location code (2840 = United States) and language code.
+DEFAULT_LOCATION_CODE=2840
+DEFAULT_LANGUAGE_CODE=en
+# Per-client monthly cap for paid data, in USD.
+DEFAULT_CLIENT_MONTHLY_BUDGET_USD=5
+
+# Extra rules for the suggest_competitors denylist (on top of the built-in list of social, review, marketplace,
+# encyclopedia, video and big-box sites and .gov/.edu). A file with one rule per line (optional; gitignored by default)
+# and/or a comma-separated list. Rules: domain.example (and subdomains), *.suffix, name.* (any short suffix), !domain (exempt).
+COMPETITOR_DENYLIST_FILE=./config/competitor-denylist.txt
+COMPETITOR_DENYLIST=
+
+# Directory for the SQLite cache, spend ledger and Google OAuth token.
+DATA_DIR=./data
+
+# Report files written by export_report and `crawlmark-cli report` (<REPORTS_DIR>/<client>/<YYYY-MM>/), mode 0600.
+# Templates live in config/report-templates.yaml; `crawlmark-cli report templates init` creates it.
+REPORTS_DIR=./reports
+
+# Server-log analysis (log_analysis): the directories it may read, comma-separated absolute paths, e.g.
+# LOG_DIRS=/var/log/nginx,/var/log/apache2. The logs folder in the config home is always allowed. Unset (and no
+# logs folder), log_analysis is unavailable; `crawlmark-cli logs <file>...` needs no setting. Logs are read on this
+# machine only and never uploaded; IP addresses and user agents are never returned or stored.
+LOG_DIRS=
+
+# Search Console Pages report imports (import_index_report): the directories it may read exports from, comma-separated
+# absolute paths, e.g. IMPORT_DIRS=/Users/me/Downloads. The imports folder in the config home is always allowed.
+# `crawlmark-cli index-coverage import --client <domain> <file>` needs no setting. Files are parsed on this machine only.
+IMPORT_DIRS=
+
+# Hard monthly cap across all clients, in USD.
+GLOBAL_MONTHLY_BUDGET_USD=50
+
+# Optional: path to a Chrome/Chromium binary for Lighthouse and axe (read directly, not via config).
+# CHROME_PATH=
+
+# Chrome's sandbox is on by default. Set to true only where it cannot start (it is added automatically on Linux as root).
+# CHROME_NO_SANDBOX=
+
+# Set to true to let lighthouse, accessibility_check, render_check and audits reach loopback/private addresses
+# (local development only). Without it, hostnames that resolve to such an address are refused, and headless Chrome's
+# egress proxy blocks them for every request the page makes.
+ALLOW_PRIVATE_URLS=
+
+# Set to true to only accept URLs on a client domain (or a subdomain of one) from the client list, for lighthouse,
+# accessibility_check, render_check, run_audit `urls` and every URL and redirect the local crawler (local_crawl) requests. Resources the page loads are not restricted (they still pass
+# the private-address check). Values other than true/false are refused, so a typo never silently turns it off.
+RESTRICT_TO_CLIENT_DOMAINS=
+
+# Scheduled runs (crawlmark-cli check-traffic / audit / check-ranks): alert when non-branded clicks fall by at least this percent...
+ALERT_TRAFFIC_DROP_PCT=25
+# ...and by at least this many clicks (avoids alerts on tiny sites).
+ALERT_MIN_CLICKS_LOST=50
+# crawlmark-cli check-ranks --live: alert when a tracked keyword's live rank falls at least this many places
+# (leaving the top 10 always alerts). Search Console averages never alert.
+ALERT_RANK_DROP=5
+# Optional incoming webhook (Slack-style) that gets {"text": "..."} when there are alerts. Must be https. Keep it secret.
+ALERT_WEBHOOK_URL=
+
+# Set to 1 to turn off the "Update available" check. Without it, `init`, `setup`, `--help` and `--version` ask the npm
+# registry for crawlmark-mcp's latest version at most once a day (in a terminal only; never in CI, with --json, or from
+# the MCP server). The request carries no query, headers or client data. NO_UPDATE_NOTIFIER=1 also turns it off.
+CRAWLMARK_NO_UPDATE_CHECK=
+
+# Testing the setup wizard only (leave unset): use this Claude Desktop config file and this `claude` binary
+# instead of the real ones. Read from the shell environment only; values in this file are ignored.
+# CRAWLMARK_CLAUDE_DESKTOP_CONFIG=
+# CRAWLMARK_CLAUDE_BIN=

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Andrew Long
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.