@kirkelabs/agent-readiness-scan 0.1.0

package/src/fetcher.js

@@ -0,0 +1,89 @@
+/**
+ * fetcher.js
+ *
+ * Retrieves a URL the same way a non-JavaScript AI crawler does, and
+ * also fetches site-rooted sibling paths (robots.txt and .well-known/*).
+ * GPTBot, ClaudeBot, PerplexityBot and Google-Extended do not execute
+ * JavaScript — they read the raw HTML the server returns. This module
+ * deliberately performs a single plain HTTP GET with an AI-crawler
+ * user agent and never runs scripts.
+ */
+
+const USER_AGENTS = {
+  gptbot:
+    'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot',
+  claudebot:
+    'Mozilla/5.0 (compatible; ClaudeBot/1.0; +https://www.anthropic.com/claude-bot)',
+  perplexitybot:
+    'Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)',
+  google:
+    'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Google-Extended',
+  default:
+    'agent-readiness-scan/0.1 (+https://github.com/KirkeLabs/agent-readiness-scan)',
+};
+
+/**
+ * Fetch a resource as an AI crawler would.
+ * @param {string} url
+ * @param {object} opts
+ * @param {string} [opts.agent='gptbot']
+ * @param {number} [opts.timeoutMs=15000]
+ * @returns {Promise<{ok:boolean,status:number,html:string,headers:object,finalUrl:string,error?:string}>}
+ */
+export async function fetchAsCrawler(url, opts = {}) {
+  const agent = USER_AGENTS[opts.agent] || USER_AGENTS.gptbot;
+  const timeoutMs = opts.timeoutMs ?? 15000;
+  const controller = new AbortController();
+  const t = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      signal: controller.signal,
+      headers: {
+        'User-Agent': agent,
+        Accept: 'text/html,application/xhtml+xml,application/json,*/*',
+      },
+    });
+    const html = await res.text();
+    const headers = {};
+    res.headers.forEach((v, k) => {
+      headers[k] = v;
+    });
+    return {
+      ok: res.ok,
+      status: res.status,
+      html,
+      headers,
+      finalUrl: res.url || url,
+    };
+  } catch (err) {
+    return {
+      ok: false,
+      status: 0,
+      html: '',
+      headers: {},
+      finalUrl: url,
+      error: err.name === 'AbortError' ? `Timed out after ${timeoutMs}ms` : err.message,
+    };
+  } finally {
+    clearTimeout(t);
+  }
+}
+
+/**
+ * Fetch a sibling resource (robots.txt, /.well-known/*) at the site root.
+ * @param {string} baseUrl
+ * @param {string} path e.g. '/.well-known/mcp/server-card.json'
+ */
+export async function fetchSibling(baseUrl, path, opts = {}) {
+  let origin;
+  try {
+    origin = new URL(baseUrl).origin;
+  } catch {
+    return { ok: false, status: 0, html: '', headers: {}, finalUrl: path };
+  }
+  return fetchAsCrawler(origin + path, opts);
+}
+
+export { USER_AGENTS };

package/src/generators.js

@@ -0,0 +1,174 @@
+/**
+ * generators.js
+ *
+ * Produces the "customs declaration" — a drop-in set of files ready to
+ * paste at the site root that close most of the policy gaps the
+ * scanner identifies. The user reviews and edits before deploy;
+ * these are conservative scaffolds, not finished output.
+ *
+ * Generated artefacts:
+ *   - robots.txt                            (per-bot policy + Content-Signals)
+ *   - .well-known/security.txt              (RFC 9116)
+ *   - .well-known/mcp/server-card.json      (MCP server card scaffold)
+ *   - .well-known/acp/manifest.json         (ACP manifest scaffold)
+ */
+
+const AI_BOTS = {
+  training: [
+    'GPTBot',
+    'ClaudeBot',
+    'Google-Extended',
+    'anthropic-ai',
+    'CCBot',
+    'Bytespider',
+    'Amazonbot',
+    'Applebot-Extended',
+    'meta-externalagent',
+  ],
+  grounding: ['OAI-SearchBot', 'PerplexityBot', 'Claude-Web'],
+  userDirected: ['ChatGPT-User', 'Claude-User'],
+};
+
+function generateRobotsTxt(origin) {
+  const lines = [];
+  lines.push('# robots.txt — Customs Declaration');
+  lines.push('# Generated by @kirkelabs/agent-readiness-scan.');
+  lines.push('# Review and edit before deploying. Tighten or loosen rules to match');
+  lines.push('# your stated bargain with AI agents (training vs grounding vs user-directed).');
+  lines.push('');
+  lines.push('# --- Default allow-all baseline ---');
+  lines.push('User-agent: *');
+  lines.push('Allow: /');
+  lines.push('');
+  lines.push('# --- Training crawlers (use your content to train foundation models) ---');
+  for (const bot of AI_BOTS.training) {
+    lines.push(`User-agent: ${bot}`);
+    lines.push('Disallow:                       # TODO: set to / to disallow, blank to allow');
+    lines.push('');
+  }
+  lines.push('# --- Grounding crawlers (fetch in real time for answer-engine retrieval) ---');
+  for (const bot of AI_BOTS.grounding) {
+    lines.push(`User-agent: ${bot}`);
+    lines.push('Allow: /');
+    lines.push('');
+  }
+  lines.push('# --- User-directed crawlers (fetched on behalf of a logged-in user) ---');
+  for (const bot of AI_BOTS.userDirected) {
+    lines.push(`User-agent: ${bot}`);
+    lines.push('Allow: /');
+    lines.push('');
+  }
+  lines.push('# --- Cloudflare Content Signals (which uses you permit) ---');
+  lines.push('# See https://contentsignals.org/');
+  lines.push('Content-Signal: search=yes, ai-input=yes, ai-train=no');
+  lines.push('');
+  lines.push(`Sitemap: ${origin}/sitemap.xml`);
+  lines.push('');
+  return lines.join('\n');
+}
+
+function generateSecurityTxt() {
+  const expires = new Date();
+  expires.setFullYear(expires.getFullYear() + 1);
+  const lines = [];
+  lines.push('# /.well-known/security.txt — RFC 9116');
+  lines.push('# Generated by @kirkelabs/agent-readiness-scan. Review before deploying.');
+  lines.push('');
+  lines.push('Contact: mailto:security@your-domain.example');
+  lines.push(`Expires: ${expires.toISOString()}`);
+  lines.push('Preferred-Languages: en');
+  lines.push('# Canonical: https://your-domain.example/.well-known/security.txt');
+  lines.push('# Encryption: https://your-domain.example/pgp-key.asc');
+  lines.push('');
+  return lines.join('\n');
+}
+
+function generateMcpServerCard(origin, name) {
+  return JSON.stringify(
+    {
+      $schema: 'https://modelcontextprotocol.io/schemas/server-card/v1',
+      name: name || 'TODO: Your service name',
+      description: 'TODO: One-sentence description of what this MCP server lets agents do.',
+      version: '0.1.0',
+      url: `${origin}/mcp`,
+      contact: {
+        name: 'TODO: maintainer',
+        email: 'mcp@your-domain.example',
+      },
+      tools: [
+        {
+          name: 'TODO_tool_name',
+          description: 'TODO: what this tool does',
+          inputSchema: {
+            type: 'object',
+            properties: {},
+            required: [],
+          },
+        },
+      ],
+      authentication: {
+        type: 'oauth2',
+        authorizationEndpoint: `${origin}/oauth/authorize`,
+        tokenEndpoint: `${origin}/oauth/token`,
+        pkce: { codeChallengeMethods: ['S256'] },
+      },
+    },
+    null,
+    2,
+  );
+}
+
+function generateAcpManifest(origin, name) {
+  return JSON.stringify(
+    {
+      version: '2026-04-17',
+      merchant: {
+        name: name || 'TODO: Your merchant name',
+        url: origin,
+        contact: 'merchant@your-domain.example',
+      },
+      capabilities: {
+        checkout: { endpoint: `${origin}/acp/checkout`, supportedMethods: ['card'] },
+        productCatalog: { endpoint: `${origin}/acp/products` },
+      },
+      keys: [
+        {
+          kty: 'TODO',
+          kid: 'TODO',
+          alg: 'RS256',
+          use: 'sig',
+          n: 'TODO: base64url-encoded RSA modulus',
+          e: 'AQAB',
+        },
+      ],
+    },
+    null,
+    2,
+  );
+}
+
+function deriveName($, finalUrl) {
+  const t = ($('title').first().text() || '').trim();
+  if (t) return t.split('—')[0].split('|')[0].trim();
+  try {
+    return new URL(finalUrl).hostname.replace(/^www\./, '');
+  } catch {
+    return 'Your Brand';
+  }
+}
+
+export function generateCustomsDeclaration(ctx) {
+  let origin = ctx.finalUrl;
+  try {
+    origin = new URL(ctx.finalUrl).origin;
+  } catch {
+    /* keep */
+  }
+  const name = deriveName(ctx.$, ctx.finalUrl);
+  return {
+    'robots.txt': generateRobotsTxt(origin),
+    '.well-known/security.txt': generateSecurityTxt(),
+    '.well-known/mcp/server-card.json': generateMcpServerCard(origin, name),
+    '.well-known/acp/manifest.json': generateAcpManifest(origin, name),
+  };
+}

package/src/index.js

@@ -0,0 +1,126 @@
+/**
+ * index.js — the scan orchestrator (public API).
+ *
+ * Programmatic entry point:
+ *   import { scan } from '@kirkelabs/agent-readiness-scan';
+ *   const result = await scan('https://example.com');
+ *
+ * Pre-fetches the target page + robots.txt + 7 .well-known/* paths in
+ * parallel, then runs 8 pure check functions against the assembled
+ * context. Each check is a synchronous module that exports
+ * `meta` ({id, title, weight, why}) and `run(ctx)`.
+ */
+
+import { load } from 'cheerio';
+import { fetchAsCrawler, fetchSibling } from './fetcher.js';
+import { generateCustomsDeclaration } from './generators.js';
+
+import * as c01 from './checks/01-per-bot-policy.js';
+import * as c02 from './checks/02-declared-use-signals.js';
+import * as c03 from './checks/03-bot-auth-readiness.js';
+import * as c04 from './checks/04-mcp-exposure.js';
+import * as c05 from './checks/05-agentic-commerce.js';
+import * as c06 from './checks/06-product-offer.js';
+import * as c07 from './checks/07-identity-corroboration.js';
+import * as c08 from './checks/08-source-regulatory.js';
+
+const CHECKS = [c01, c02, c03, c04, c05, c06, c07, c08];
+
+const WELL_KNOWN_PATHS = {
+  securityTxt: '/.well-known/security.txt',
+  mcpServerCard: '/.well-known/mcp/server-card.json',
+  acpManifest: '/.well-known/acp/manifest.json',
+  ucp: '/.well-known/ucp',
+  botAuthDirectory: '/.well-known/http-message-signatures-directory',
+  oauthProtectedResource: '/.well-known/oauth-protected-resource',
+  oauthAuthorizationServer: '/.well-known/oauth-authorization-server',
+};
+
+async function fetchWellKnown(baseUrl, opts) {
+  const entries = await Promise.all(
+    Object.entries(WELL_KNOWN_PATHS).map(async ([key, path]) => {
+      const res = await fetchSibling(baseUrl, path, opts);
+      return [
+        key,
+        {
+          found: res.ok && res.html && res.html.trim().length > 0,
+          content: res.html,
+          headers: res.headers,
+          status: res.status,
+        },
+      ];
+    }),
+  );
+  return Object.fromEntries(entries);
+}
+
+export async function scan(url, opts = {}) {
+  const page = await fetchAsCrawler(url, opts);
+  if (!page.ok && !page.html) {
+    return {
+      url,
+      ok: false,
+      error: page.error || `HTTP ${page.status}`,
+      score: 0,
+      grade: 'F',
+      dimensions: [],
+    };
+  }
+
+  const $ = load(page.html);
+  const [robots, wellKnown] = await Promise.all([
+    fetchSibling(url, '/robots.txt', opts),
+    fetchWellKnown(url, opts),
+  ]);
+
+  const ctx = {
+    $,
+    html: page.html,
+    finalUrl: page.finalUrl,
+    headers: page.headers,
+    robotsTxt: robots.ok ? robots.html : null,
+    wellKnown,
+  };
+
+  const dimensions = [];
+  let weightedSum = 0;
+  let weightTotal = 0;
+
+  for (const mod of CHECKS) {
+    const res = mod.run(ctx);
+    const w = mod.meta.weight;
+    weightedSum += (res.score / res.max) * w;
+    weightTotal += w;
+    dimensions.push({
+      id: mod.meta.id,
+      title: mod.meta.title,
+      why: mod.meta.why,
+      weight: w,
+      score: res.score,
+      max: res.max,
+      findings: res.findings,
+      detail: res.detail || {},
+    });
+  }
+
+  const score = Math.round((weightedSum / weightTotal) * 100);
+  return {
+    url,
+    finalUrl: page.finalUrl,
+    ok: true,
+    status: page.status,
+    scannedAt: new Date().toISOString(),
+    score,
+    grade: grade(score),
+    dimensions,
+    generated: generateCustomsDeclaration(ctx),
+  };
+}
+
+export function grade(s) {
+  if (s >= 90) return 'A';
+  if (s >= 80) return 'B';
+  if (s >= 65) return 'C';
+  if (s >= 50) return 'D';
+  return 'F';
+}

package/src/scorecard.js

@@ -0,0 +1,87 @@
+/**
+ * scorecard.js — renders a self-contained, shareable HTML scorecard.
+ * No external assets; deployable to GitHub Pages or saved as a file.
+ */
+
+const C = {
+  bg: '#0A0E0D',
+  panel: '#121A17',
+  line: '#23302B',
+  txt: '#E6EDE9',
+  soft: '#A7B5AF',
+  mute: '#6C7C75',
+  acc: '#00D08A',
+  warn: '#E0A93C',
+  fail: '#E0623C',
+};
+
+function gradeColor(g) {
+  return g === 'A' || g === 'B' ? C.acc : g === 'C' ? C.warn : C.fail;
+}
+function lvlColor(l) {
+  return l === 'pass'
+    ? C.acc
+    : l === 'warn'
+      ? C.warn
+      : l === 'fail'
+        ? C.fail
+        : C.mute;
+}
+function esc(s) {
+  return String(s).replace(/[&<>"]/g, (m) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' })[m]);
+}
+
+export function renderScorecard(r) {
+  const gc = gradeColor(r.grade);
+  const dims = r.dimensions
+    .map((d) => {
+      const pct = Math.round((d.score / d.max) * 100);
+      const findings = d.findings
+        .map(
+          (f) =>
+            `<li style="border-left:2px solid ${lvlColor(f.level)};padding-left:10px;margin:6px 0;color:${C.soft}">
+               <span style="font-family:'JetBrains Mono',monospace;font-size:10px;text-transform:uppercase;letter-spacing:.12em;color:${lvlColor(f.level)}">${f.level}</span><br>${esc(f.msg)}</li>`,
+        )
+        .join('');
+      return `<div style="background:${C.panel};border:1px solid ${C.line};border-radius:12px;padding:20px 22px;margin:12px 0">
+        <div style="display:flex;justify-content:space-between;align-items:baseline;gap:14px;flex-wrap:wrap">
+          <h3 style="margin:0;font-size:17px;color:${C.txt}">${esc(d.title)}</h3>
+          <span style="font-family:'JetBrains Mono',monospace;font-size:15px;color:${pct >= 70 ? C.acc : pct >= 40 ? C.warn : C.fail}">${d.score}/${d.max}</span>
+        </div>
+        <div style="height:6px;background:${C.line};border-radius:4px;margin:12px 0;overflow:hidden">
+          <div style="height:100%;width:${pct}%;background:${pct >= 70 ? C.acc : pct >= 40 ? C.warn : C.fail}"></div>
+        </div>
+        <p style="margin:6px 0 10px;font-size:13px;color:${C.mute};font-style:italic">${esc(d.why)}</p>
+        <ul style="list-style:none;margin:0;padding:0;font-size:13.5px;line-height:1.5">${findings}</ul>
+      </div>`;
+    })
+    .join('');
+
+  return `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8">
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<title>Agent Readiness Scorecard — ${esc(r.url)}</title>
+<link href="https://fonts.googleapis.com/css2?family=Archivo:wght@400;600;800;900&family=JetBrains+Mono:wght@400;700&display=swap" rel="stylesheet">
+<style>body{margin:0;background:${C.bg};color:${C.txt};font-family:'Archivo',sans-serif;line-height:1.6;
+background-image:radial-gradient(800px 400px at 80% -5%,rgba(0,208,138,.06),transparent 60%)}
+.wrap{max-width:820px;margin:0 auto;padding:48px 28px}a{color:${C.acc}}</style></head><body>
+<div class="wrap">
+  <div style="font-family:'JetBrains Mono',monospace;font-size:11px;letter-spacing:.22em;text-transform:uppercase;color:${C.acc};margin-bottom:24px">Agent Readiness Scorecard · @kirkelabs/agent-readiness-scan</div>
+  <div style="display:flex;align-items:center;gap:28px;flex-wrap:wrap;border-bottom:1px solid ${C.line};padding-bottom:28px;margin-bottom:24px">
+    <div style="width:120px;height:120px;border-radius:20px;border:2px solid ${gc};display:flex;flex-direction:column;align-items:center;justify-content:center">
+      <div style="font-family:'Archivo';font-weight:900;font-size:46px;color:${gc};line-height:1">${r.grade}</div>
+      <div style="font-family:'JetBrains Mono',monospace;font-size:13px;color:${C.soft}">${r.score}/100</div>
+    </div>
+    <div style="flex:1;min-width:240px">
+      <div style="font-size:13px;color:${C.mute};font-family:'JetBrains Mono',monospace">SCANNED URL</div>
+      <div style="font-size:19px;font-weight:600;word-break:break-all;margin:4px 0 12px">${esc(r.url)}</div>
+      <div style="font-size:12px;color:${C.mute};font-family:'JetBrains Mono',monospace">${esc(r.scannedAt)}</div>
+    </div>
+  </div>
+  ${dims}
+  <div style="margin-top:34px;border-top:1px solid ${C.line};padding-top:22px;font-size:12px;color:${C.mute};font-family:'JetBrains Mono',monospace;line-height:1.8">
+    Open-source · MIT · Built by Soleman El Gelawi (CTO, Kirke Labs), with Steve Kirton — as a gift to the Algorand ecosystem.<br>
+    github.com/KirkeLabs/agent-readiness-scan · www.kirkelabs.com<br>
+    Heuristic indicators, not a guarantee of agent action. See the methodology doc in the repo.
+  </div>
+</div></body></html>`;
+}