crawlwise 1.0.0

package/README.md

@@ -0,0 +1,55 @@
+# crawlwise CLI
+
+Audit a URL from a terminal, and ask whether AI crawlers can reach it.
+
+```bash
+npx crawlwise ai-crawlers https://example.com
+npx crawlwise audit https://example.com
+npx crawlwise audit https://example.com --baseline <audit-id> --fail-on critical
+npx crawlwise tools
+```
+
+## What needs a key and what does not
+
+`ai-crawlers` and `tools` are free and need no key: they call the same endpoints the free
+web tools call. `audit` needs an API key because an audit spends a credit, and Pro or above
+can create one at <https://crawlwise.site/account/keys>.
+
+```bash
+export CRAWLWISE_API_KEY=cw_live_...
+```
+
+## Exit codes
+
+- `0` — done, and no regression at or above `--fail-on` (default `critical`)
+- `1` — a regression at or above `--fail-on`
+- `2` — usage error, or the API refused the request
+
+That makes it usable as a merge gate:
+
+```bash
+npx crawlwise audit "$PREVIEW_URL" --baseline "$LAST_KNOWN_GOOD_AUDIT" --fail-on critical
+```
+
+A check that did not run on the baseline is reported as new and is never counted as a
+regression — a methodology change is not a defect.
+
+## Options
+
+| Flag | Meaning |
+|---|---|
+| `--json` | machine-readable output |
+| `--api-key <key>` | instead of `CRAWLWISE_API_KEY` |
+| `--endpoint <origin>` | point at staging, default `https://crawlwise.site` |
+| `--fail-on <level>` | `critical`, `warning`, `never` |
+
+## Publishing
+
+This package is not on npm yet. Until it is, run it from a checkout:
+
+```bash
+node cli/bin/crawlwise.mjs ai-crawlers https://example.com
+```
+
+Publishing it is `npm publish` from this directory with `private` removed from
+`package.json` — a deliberate step, and npm now requires 2FA-capable publishing.

package/bin/crawlwise.mjs

@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+
+const args = process.argv.slice(2);
+const flag = (name, fallback = '') => {
+  const index = args.indexOf('--' + name);
+  if (index < 0) return fallback;
+  const next = args[index + 1];
+  return next && !next.startsWith('--') ? next : 'true';
+};
+const has = (name) => args.includes('--' + name);
+const positional = args.filter((arg, index) => !arg.startsWith('--') && !(index > 0 && args[index - 1].startsWith('--') && !args[index - 1].includes('=')));
+
+const endpoint = (flag('endpoint') || process.env.CRAWLWISE_ENDPOINT || 'https://crawlwise.site').replace(/\/+$/, '');
+const apiKey = flag('api-key') || process.env.CRAWLWISE_API_KEY || '';
+const asJson = has('json');
+
+function usage() {
+  console.log(`crawlwise
+
+  npx crawlwise audit <url>                 audit one page
+  npx crawlwise audit <url> --baseline <id> audit and diff against an earlier audit
+  npx crawlwise ai-crawlers <url>           can GPTBot, ClaudeBot and the rest reach it? free, no key
+  npx crawlwise tools                       the free toolkit, with what each one cannot do
+
+Options
+  --json               machine-readable output
+  --api-key <key>      or set CRAWLWISE_API_KEY (cw_live_…, Pro plan or above)
+  --endpoint <origin>  or set CRAWLWISE_ENDPOINT (default ${endpoint})
+  --fail-on <level>    critical | warning | never  (audit diffs only, default critical)
+  --help
+
+Exit codes: 0 ok · 1 regression at or above --fail-on · 2 usage or request error`);
+}
+
+function fail(message, code = 2) {
+  console.error('crawlwise: ' + message);
+  process.exit(code);
+}
+
+async function api(path, init = {}) {
+  const response = await fetch(endpoint + path, {
+    ...init,
+    headers: { 'Content-Type': 'application/json', ...(apiKey ? { Authorization: 'Bearer ' + apiKey } : {}), ...(init.headers ?? {}) },
+  });
+  const body = await response.json().catch(() => ({}));
+  if (!response.ok) {
+    const detail = body?.error?.message || body?.error || 'HTTP ' + response.status;
+    fail(detail, 2);
+  }
+  return body;
+}
+
+async function audit(url, baselineId) {
+  if (!apiKey) fail('an audit needs an API key. Set CRAWLWISE_API_KEY, or run: crawlwise ai-crawlers <url> (free, no key).');
+  const started = await api('/v1/audits', { method: 'POST', body: JSON.stringify({ url }) });
+  process.stderr.write('crawlwise: audit ' + started.id + ' started\n');
+  const deadline = Date.now() + 5 * 60_000;
+  let job;
+  while (Date.now() < deadline) {
+    await new Promise((resolve) => setTimeout(resolve, 2500));
+    job = await api('/v1/audits/' + encodeURIComponent(started.id));
+    if (job.status === 'complete' || job.status === 'failed' || job.status === 'abandoned') break;
+  }
+  if (!job || job.status !== 'complete') fail('the audit ended as ' + (job?.status ?? 'still running') + (job?.error ? ': ' + job.error : ''), 2);
+
+  if (asJson) {
+    console.log(JSON.stringify({ id: started.id, status: job.status, url: job.url, report: job.report }, null, 2));
+    return 0;
+  }
+
+  const report = job.report ?? {};
+  const scoring = report.scoring ?? {};
+  console.log('');
+  console.log((job.url || url) + '  ·  ' + (scoring.score === null || scoring.score === undefined ? 'not scored' : 'score ' + scoring.score + '/' + (scoring.applicable ?? '?') + ' applicable'));
+  const priority = report.priority ?? [];
+  const review = report.worth_considering ?? [];
+  const unmeasured = report.not_measured ?? [];
+  console.log(priority.length + ' to fix · ' + review.length + ' worth reviewing · ' + unmeasured.length + ' not measured');
+  for (const [index, action] of priority.slice(0, 10).entries()) {
+    console.log('');
+    console.log('  ' + (index + 1) + '. [' + action.priority + '] ' + action.title);
+    console.log('     ' + action.why);
+  }
+  console.log('');
+  console.log('Report: ' + endpoint + '/history');
+
+  if (!baselineId) return 0;
+  const before = await api('/v1/audits/' + encodeURIComponent(baselineId));
+  if (before.status !== 'complete') fail('the baseline audit ' + baselineId + ' is ' + before.status, 2);
+  const old = new Map((before.report?.checks ?? []).map((check) => [check.id, check]));
+  const regressions = (report.checks ?? []).filter((check) => check.status !== 'pass' && old.get(check.id)?.status === 'pass');
+  console.log('');
+  console.log(regressions.length ? regressions.length + ' regression(s) against ' + baselineId + ':' : 'No regressions against ' + baselineId + '.');
+  for (const check of regressions) console.log('  ' + check.id + ': pass → ' + check.status + '  ' + check.title);
+
+  const failOn = flag('fail-on', 'critical');
+  if (failOn === 'never') return 0;
+  const critical = regressions.filter((check) => ['index', 'ai-crawler-access'].includes(check.id));
+  if (failOn === 'critical' && critical.length) return 1;
+  if (failOn === 'warning' && regressions.length) return 1;
+  return 0;
+}
+
+async function aiCrawlers(url) {
+  const body = await api('/api/tools/ai-crawlers', { method: 'POST', body: JSON.stringify({ url }) });
+  if (asJson) {
+    console.log(JSON.stringify(body, null, 2));
+    return 0;
+  }
+  console.log('');
+  console.log(body.finalUrl || url);
+  for (const row of body.rows ?? []) {
+    console.log('  ' + row.id.padEnd(20) + row.verdict.padEnd(17) + (row.deliberate ? '(deliberate opt-out)' : row.purpose));
+  }
+  const blocked = (body.rows ?? []).filter((row) => row.verdict === 'blocked' && !row.deliberate);
+  if (blocked.length) {
+    console.log('');
+    console.log(blocked.length + ' crawler(s) blocked without an intended opt-out: ' + blocked.map((row) => row.id).join(', '));
+  }
+  if (body.shareId) console.log('\nShareable: ' + endpoint + '/tools/ai-crawler-check/r/' + body.shareId);
+  return 0;
+}
+
+async function tools() {
+  const body = await api('/api/tools/catalogue').catch(() => null);
+  const list = body?.tools ?? [];
+  if (asJson) {
+    console.log(JSON.stringify(list, null, 2));
+    return 0;
+  }
+  console.log('');
+  console.log('Free tools, no account, no credit: ' + endpoint + '/tools');
+  for (const tool of list) console.log('  ' + tool.slug.padEnd(22) + tool.title);
+  return 0;
+}
+
+const [command, target] = positional;
+if (!command || has('help') || command === 'help') {
+  usage();
+  process.exit(has('help') || command === 'help' ? 0 : 2);
+}
+
+if (command === 'audit') {
+  if (!target) fail('which URL?  crawlwise audit https://example.com');
+  process.exit(await audit(target, flag('baseline')));
+} else if (command === 'ai-crawlers') {
+  if (!target) fail('which URL?  crawlwise ai-crawlers https://example.com');
+  process.exit(await aiCrawlers(target));
+} else if (command === 'tools') {
+  process.exit(await tools());
+} else {
+  usage();
+  process.exit(2);
+}

package/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "crawlwise",
+  "version": "1.0.0",
+  "description": "Audit a URL from the command line, without installing anything.",
+  "type": "module",
+  "bin": { "crawlwise": "bin/crawlwise.mjs" },
+  "engines": { "node": ">=22" },
+  "files": ["bin", "README.md"],
+  "license": "UNLICENSED",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Adnanarodiya/crawlwise.git",
+    "directory": "cli"
+  },
+  "homepage": "https://crawlwise.site/agents",
+  "bugs": "https://github.com/Adnanarodiya/crawlwise/issues",
+  "keywords": ["seo", "audit", "ai-crawlers", "gptbot", "cli", "crawlwise"]
+}