@braedenbuilds/crawl-sim 1.0.1

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@braedenbuilds/crawl-sim",
+  "version": "1.0.1",
+  "description": "Agent-native multi-bot web crawler simulator. See your site through the eyes of Googlebot, GPTBot, ClaudeBot, and PerplexityBot.",
+  "bin": {
+    "crawl-sim": "bin/install.js"
+  },
+  "keywords": [
+    "seo",
+    "crawler",
+    "googlebot",
+    "gptbot",
+    "claudebot",
+    "perplexitybot",
+    "ai-visibility",
+    "ai-seo",
+    "geo",
+    "llms-txt",
+    "claude-code",
+    "claude-code-skill"
+  ],
+  "author": "BraedenBDev",
+  "license": "MIT",
+  "homepage": "https://github.com/BraedenBDev/crawl-sim#readme",
+  "bugs": {
+    "url": "https://github.com/BraedenBDev/crawl-sim/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/BraedenBDev/crawl-sim.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "files": [
+    "bin/",
+    "SKILL.md",
+    "profiles/",
+    "scripts/",
+    "README.md",
+    "LICENSE"
+  ]
+}

package/profiles/chatgpt-user.json

@@ -0,0 +1,28 @@
+{
+  "id": "chatgpt-user",
+  "name": "ChatGPT-User",
+  "vendor": "OpenAI",
+  "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
+  "robotsTxtToken": "ChatGPT-User",
+  "purpose": "user-initiated",
+  "rendersJavaScript": "unknown",
+  "respectsRobotsTxt": "partial",
+  "crawlDelaySupported": "unknown",
+  "ipRangesUrl": "https://openai.com/chatgpt-user.json",
+  "docs": "https://developers.openai.com/api/docs/bots",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": "unknown",
+      "level": "inferred",
+      "source": "Not documented by OpenAI"
+    },
+    "respectsRobotsTxt": {
+      "value": "partial",
+      "level": "official",
+      "source": "Official docs state: 'Because these actions are initiated by a user, robots.txt rules may not apply.'"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": ["gptbot", "oai-searchbot"],
+  "notes": "Not used for automatic crawling. Not used to determine search appearance. User-initiated fetches in ChatGPT and Custom GPTs."
+}

package/profiles/claude-searchbot.json

@@ -0,0 +1,28 @@
+{
+  "id": "claude-searchbot",
+  "name": "Claude-SearchBot",
+  "vendor": "Anthropic",
+  "userAgent": "Claude-SearchBot",
+  "robotsTxtToken": "Claude-SearchBot",
+  "purpose": "search",
+  "rendersJavaScript": "unknown",
+  "respectsRobotsTxt": true,
+  "crawlDelaySupported": "unknown",
+  "ipRangesUrl": null,
+  "docs": "https://privacy.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": "unknown",
+      "level": "inferred",
+      "source": "Not documented by Anthropic"
+    },
+    "respectsRobotsTxt": {
+      "value": true,
+      "level": "official",
+      "source": "Official docs, but notes blocking may reduce visibility and accuracy in user search results"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": ["claudebot", "claude-user"],
+  "notes": "Navigates the web to improve search result quality. Focused on search indexing, not training."
+}

package/profiles/claude-user.json

@@ -0,0 +1,28 @@
+{
+  "id": "claude-user",
+  "name": "Claude-User",
+  "vendor": "Anthropic",
+  "userAgent": "Claude-User",
+  "robotsTxtToken": "Claude-User",
+  "purpose": "user-initiated",
+  "rendersJavaScript": "unknown",
+  "respectsRobotsTxt": true,
+  "crawlDelaySupported": "unknown",
+  "ipRangesUrl": null,
+  "docs": "https://privacy.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": "unknown",
+      "level": "inferred",
+      "source": "Not documented by Anthropic"
+    },
+    "respectsRobotsTxt": {
+      "value": true,
+      "level": "official",
+      "source": "Official docs, but notes blocking may reduce visibility for user-directed web search"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": ["claudebot", "claude-searchbot"],
+  "notes": "When individuals ask questions to Claude, it may access websites. Blocking prevents Claude from retrieving content in response to user queries."
+}

package/profiles/claudebot.json

@@ -0,0 +1,28 @@
+{
+  "id": "claudebot",
+  "name": "ClaudeBot",
+  "vendor": "Anthropic",
+  "userAgent": "ClaudeBot",
+  "robotsTxtToken": "ClaudeBot",
+  "purpose": "training",
+  "rendersJavaScript": false,
+  "respectsRobotsTxt": true,
+  "crawlDelaySupported": true,
+  "ipRangesUrl": null,
+  "docs": "https://privacy.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": false,
+      "level": "observed",
+      "source": "Observational evidence consistent with no JS rendering"
+    },
+    "respectsRobotsTxt": {
+      "value": true,
+      "level": "official",
+      "source": "https://privacy.claude.com"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": ["claude-user", "claude-searchbot"],
+  "notes": "Collects web content that could potentially contribute to AI model training. Crawl-delay explicitly supported (non-standard). Blocking IP addresses will not reliably work."
+}

package/profiles/googlebot.json

@@ -0,0 +1,28 @@
+{
+  "id": "googlebot",
+  "name": "Googlebot",
+  "vendor": "Google",
+  "userAgent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "robotsTxtToken": "Googlebot",
+  "purpose": "search-indexing",
+  "rendersJavaScript": true,
+  "respectsRobotsTxt": true,
+  "crawlDelaySupported": false,
+  "ipRangesUrl": null,
+  "docs": "https://developers.google.com/search/docs/crawling-indexing/javascript/fix-search-javascript",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": true,
+      "level": "official",
+      "source": "https://developers.google.com/search/docs/crawling-indexing/javascript/fix-search-javascript"
+    },
+    "respectsRobotsTxt": {
+      "value": true,
+      "level": "official",
+      "source": "RFC 9309 compliant"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": [],
+  "notes": "Two-phase: initial fetch (HTML) then queued render (headless Chrome via WRS). Evergreen Chromium. Stateless sessions. ~5s default timeout. Mobile-first indexing."
+}

package/profiles/gptbot.json

@@ -0,0 +1,28 @@
+{
+  "id": "gptbot",
+  "name": "GPTBot",
+  "vendor": "OpenAI",
+  "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.3; +https://openai.com/gptbot",
+  "robotsTxtToken": "GPTBot",
+  "purpose": "training",
+  "rendersJavaScript": false,
+  "respectsRobotsTxt": true,
+  "crawlDelaySupported": "unknown",
+  "ipRangesUrl": "https://openai.com/gptbot.json",
+  "docs": "https://developers.openai.com/api/docs/bots",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": false,
+      "level": "observed",
+      "source": "Multiple third-party tests with JS-only pages show empty content"
+    },
+    "respectsRobotsTxt": {
+      "value": true,
+      "level": "official",
+      "source": "https://developers.openai.com/api/docs/bots"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": ["oai-searchbot", "chatgpt-user"],
+  "notes": "Disallowing GPTBot indicates a site's content should not be used in training generative AI foundation models."
+}

package/profiles/oai-searchbot.json

@@ -0,0 +1,28 @@
+{
+  "id": "oai-searchbot",
+  "name": "OAI-SearchBot",
+  "vendor": "OpenAI",
+  "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot",
+  "robotsTxtToken": "OAI-SearchBot",
+  "purpose": "search",
+  "rendersJavaScript": "unknown",
+  "respectsRobotsTxt": true,
+  "crawlDelaySupported": "unknown",
+  "ipRangesUrl": "https://openai.com/searchbot.json",
+  "docs": "https://developers.openai.com/api/docs/bots",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": "unknown",
+      "level": "inferred",
+      "source": "UA mimics Chrome 131 — may indicate rendering capability, but unconfirmed"
+    },
+    "respectsRobotsTxt": {
+      "value": true,
+      "level": "official",
+      "source": "https://developers.openai.com/api/docs/bots"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": ["gptbot", "chatgpt-user"],
+  "notes": "Sites opted out of OAI-SearchBot will not be shown in ChatGPT search answers, though can still appear as navigational links."
+}

package/profiles/perplexity-user.json

@@ -0,0 +1,28 @@
+{
+  "id": "perplexity-user",
+  "name": "Perplexity-User",
+  "vendor": "Perplexity",
+  "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
+  "robotsTxtToken": "Perplexity-User",
+  "purpose": "user-initiated",
+  "rendersJavaScript": "unknown",
+  "respectsRobotsTxt": false,
+  "crawlDelaySupported": "unknown",
+  "ipRangesUrl": "https://www.perplexity.com/perplexity-user.json",
+  "docs": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": "unknown",
+      "level": "inferred",
+      "source": "Not documented by Perplexity"
+    },
+    "respectsRobotsTxt": {
+      "value": false,
+      "level": "official",
+      "source": "Official docs state: 'Since a user requested the fetch, this fetcher generally ignores robots.txt rules.'"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": ["perplexitybot"],
+  "notes": "Supports user actions within Perplexity. Not used for web crawling or AI training. Generally ignores robots.txt since fetches are user-initiated."
+}

package/profiles/perplexitybot.json

@@ -0,0 +1,28 @@
+{
+  "id": "perplexitybot",
+  "name": "PerplexityBot",
+  "vendor": "Perplexity",
+  "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
+  "robotsTxtToken": "PerplexityBot",
+  "purpose": "search-indexing",
+  "rendersJavaScript": false,
+  "respectsRobotsTxt": true,
+  "crawlDelaySupported": "unknown",
+  "ipRangesUrl": "https://www.perplexity.com/perplexitybot.json",
+  "docs": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
+  "confidence": {
+    "rendersJavaScript": {
+      "value": false,
+      "level": "observed",
+      "source": "Most third-party reports indicate no JS rendering"
+    },
+    "respectsRobotsTxt": {
+      "value": true,
+      "level": "official",
+      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
+    }
+  },
+  "lastVerified": "2026-04-11",
+  "relatedBots": ["perplexity-user"],
+  "notes": "Designed to surface and link websites in search results on Perplexity. NOT used to crawl content for AI foundation models. Changes may take up to 24 hours to reflect."
+}

package/scripts/_lib.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# crawl-sim shared helpers. Source this from other scripts:
+#   SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+#   . "$SCRIPT_DIR/_lib.sh"
+
+# Extract "https://host" from any URL.
+origin_from_url() {
+  printf '%s' "$1" | sed -E 's#(^https?://[^/]+).*#\1#'
+}
+
+# Extract the host from a URL, stripping "www." prefix.
+host_from_url() {
+  printf '%s' "$1" | sed -E 's#^https?://##' | sed -E 's#/.*$##' | sed -E 's#^www\.##'
+}
+
+# Extract the path portion of a URL. Returns "/" if empty.
+path_from_url() {
+  local p
+  p=$(printf '%s' "$1" | sed -E 's#^https?://[^/]+##')
+  printf '%s' "${p:-/}"
+}
+
+# Extract the directory portion of a URL's path (everything up to the last /).
+# Used for resolving relative URLs against a base page URL.
+# Example: https://example.com/blog/index.html -> https://example.com/blog/
+dir_from_url() {
+  local url="$1"
+  local origin
+  origin=$(origin_from_url "$url")
+  local p
+  p=$(path_from_url "$url")
+  # If path ends with /, keep as-is; otherwise strip last segment
+  case "$p" in
+    */) printf '%s%s' "$origin" "$p" ;;
+    *)  printf '%s%s/' "$origin" "$(printf '%s' "$p" | sed -E 's#/[^/]*$##')" ;;
+  esac
+}
+
+# Count visible words in an HTML file (strips tags, counts alnum tokens).
+count_words() {
+  sed 's/<[^>]*>//g' "$1" | tr -s '[:space:]' '\n' | grep -c '[a-zA-Z0-9]' || true
+}
+
+# Fetch a URL to a local file and return the HTTP status code on stdout.
+# Usage: status=$(fetch_to_file <url> <output-file> [timeout-seconds])
+fetch_to_file() {
+  local url="$1"
+  local out="$2"
+  local timeout="${3:-15}"
+  curl -sS -L -o "$out" -w '%{http_code}' --max-time "$timeout" "$url" 2>/dev/null || echo "000"
+}

package/scripts/check-llmstxt.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+set -eu
+
+# check-llmstxt.sh — Check for llms.txt and llms-full.txt presence + structure
+# Usage: check-llmstxt.sh <url>
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+URL="${1:?Usage: check-llmstxt.sh <url>}"
+printf '[check-llmstxt] %s\n' "$URL" >&2
+ORIGIN=$(origin_from_url "$URL")
+
+TMPDIR="${TMPDIR:-/tmp}"
+LLMS_FILE=$(mktemp "$TMPDIR/crawlsim-llms.XXXXXX")
+LLMS_FULL_FILE=$(mktemp "$TMPDIR/crawlsim-llms-full.XXXXXX")
+trap 'rm -f "$LLMS_FILE" "$LLMS_FULL_FILE"' EXIT
+
+analyze_file() {
+  local file="$1"
+  local status_code="$2"
+
+  local exists=false
+  local line_count=0
+  local has_title=false
+  local title=""
+  local has_description=false
+  local url_count=0
+
+  # Treat non-200 or HTML responses as "not present"
+  if [ "$status_code" = "200" ] && [ -s "$file" ]; then
+    # Heuristic: if file starts with <!doctype or <html, site serves HTML fallback — not a real llms.txt
+    local first_bytes
+    first_bytes=$(head -c 100 "$file" | tr '[:upper:]' '[:lower:]')
+    case "$first_bytes" in
+      *"<!doctype"*|*"<html"*) ;;
+      *)
+        exists=true
+        line_count=$(wc -l < "$file" | tr -d ' ')
+        # Title: first line starting with "# "
+        if head -1 "$file" | grep -qE '^#[[:space:]]+'; then
+          has_title=true
+          title=$(head -1 "$file" | sed -E 's/^#[[:space:]]+//' | tr -d '\r')
+        fi
+        # Description: block quote or paragraph after title
+        if grep -qE '^>[[:space:]]+' "$file" || sed -n '2,5p' "$file" | grep -qE '^[A-Za-z]'; then
+          has_description=true
+        fi
+        # Count URLs (markdown links)
+        url_count=$(grep -oE '\[[^]]*\]\(https?://[^)]+\)' "$file" 2>/dev/null | wc -l | tr -d ' ' || echo 0)
+        ;;
+    esac
+  fi
+
+  # Output values via globals (bash function limitation workaround)
+  EXISTS="$exists"
+  LINE_COUNT="$line_count"
+  HAS_TITLE="$has_title"
+  TITLE="$title"
+  HAS_DESCRIPTION="$has_description"
+  URL_COUNT="$url_count"
+}
+
+LLMS_STATUS=$(fetch_to_file "${ORIGIN}/llms.txt" "$LLMS_FILE")
+analyze_file "$LLMS_FILE" "$LLMS_STATUS"
+LLMS_EXISTS=$EXISTS
+LLMS_LINES=$LINE_COUNT
+LLMS_HAS_TITLE=$HAS_TITLE
+LLMS_TITLE=$TITLE
+LLMS_HAS_DESC=$HAS_DESCRIPTION
+LLMS_URLS=$URL_COUNT
+
+LLMS_FULL_STATUS=$(fetch_to_file "${ORIGIN}/llms-full.txt" "$LLMS_FULL_FILE")
+analyze_file "$LLMS_FULL_FILE" "$LLMS_FULL_STATUS"
+LLMS_FULL_EXISTS=$EXISTS
+LLMS_FULL_LINES=$LINE_COUNT
+LLMS_FULL_HAS_TITLE=$HAS_TITLE
+LLMS_FULL_HAS_DESC=$HAS_DESCRIPTION
+LLMS_FULL_URLS=$URL_COUNT
+
+jq -n \
+  --arg url "$URL" \
+  --arg llmsUrl "${ORIGIN}/llms.txt" \
+  --arg llmsFullUrl "${ORIGIN}/llms-full.txt" \
+  --argjson llmsExists "$LLMS_EXISTS" \
+  --argjson llmsLines "$LLMS_LINES" \
+  --argjson llmsHasTitle "$LLMS_HAS_TITLE" \
+  --arg llmsTitle "$LLMS_TITLE" \
+  --argjson llmsHasDesc "$LLMS_HAS_DESC" \
+  --argjson llmsUrls "$LLMS_URLS" \
+  --argjson llmsFullExists "$LLMS_FULL_EXISTS" \
+  --argjson llmsFullLines "$LLMS_FULL_LINES" \
+  --argjson llmsFullHasTitle "$LLMS_FULL_HAS_TITLE" \
+  --argjson llmsFullHasDesc "$LLMS_FULL_HAS_DESC" \
+  --argjson llmsFullUrls "$LLMS_FULL_URLS" \
+  '{
+    url: $url,
+    llmsTxt: {
+      url: $llmsUrl,
+      exists: $llmsExists,
+      lineCount: $llmsLines,
+      hasTitle: $llmsHasTitle,
+      title: (if $llmsTitle == "" then null else $llmsTitle end),
+      hasDescription: $llmsHasDesc,
+      urlCount: $llmsUrls
+    },
+    llmsFullTxt: {
+      url: $llmsFullUrl,
+      exists: $llmsFullExists,
+      lineCount: $llmsFullLines,
+      hasTitle: $llmsFullHasTitle,
+      hasDescription: $llmsFullHasDesc,
+      urlCount: $llmsFullUrls
+    }
+  }'

package/scripts/check-robots.sh

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+set -eu
+
+# check-robots.sh — Fetch robots.txt and parse rules for a given UA token
+# Usage: check-robots.sh <url> <ua-token>
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+URL="${1:?Usage: check-robots.sh <url> <ua-token>}"
+UA_TOKEN="${2:?Usage: check-robots.sh <url> <ua-token>}"
+
+printf '[check-robots] %s for %s\n' "$URL" "$UA_TOKEN" >&2
+
+ORIGIN=$(origin_from_url "$URL")
+URL_PATH=$(path_from_url "$URL")
+ROBOTS_URL="${ORIGIN}/robots.txt"
+
+TMPDIR="${TMPDIR:-/tmp}"
+ROBOTS_FILE=$(mktemp "$TMPDIR/crawlsim-robots.XXXXXX")
+RAW_FILE=$(mktemp "$TMPDIR/crawlsim-robots-raw.XXXXXX")
+DISALLOWED_PATHS_FILE=$(mktemp "$TMPDIR/crawlsim-disallowed.XXXXXX")
+ALLOW_PATHS_FILE=$(mktemp "$TMPDIR/crawlsim-allow.XXXXXX")
+SITEMAPS_FILE=$(mktemp "$TMPDIR/crawlsim-sitemaps.XXXXXX")
+trap 'rm -f "$ROBOTS_FILE" "$RAW_FILE" "$DISALLOWED_PATHS_FILE" "$ALLOW_PATHS_FILE" "$SITEMAPS_FILE"' EXIT
+
+HTTP_STATUS=$(fetch_to_file "$ROBOTS_URL" "$ROBOTS_FILE")
+
+EXISTS=false
+if [ "$HTTP_STATUS" = "200" ] && [ -s "$ROBOTS_FILE" ]; then
+  EXISTS=true
+fi
+
+ALLOWED=true
+CRAWL_DELAY="null"
+
+if [ "$EXISTS" = "true" ]; then
+  # Extract sitemap directives
+  grep -iE '^[[:space:]]*sitemap[[:space:]]*:' "$ROBOTS_FILE" 2>/dev/null \
+    | sed -E 's/^[[:space:]]*[sS][iI][tT][eE][mM][aA][pP][[:space:]]*:[[:space:]]*//' \
+    | tr -d '\r' \
+    | sed -E 's/[[:space:]]+$//' \
+    > "$SITEMAPS_FILE" || true
+
+  # Parse User-agent blocks using portable awk
+  # State machine: track current UA group(s), emit rules tagged EXACT_ or WILD_
+  awk -v ua="$UA_TOKEN" '
+    function lower(s) { return tolower(s) }
+    function trim(s) {
+      sub(/^[ \t\r]+/, "", s)
+      sub(/[ \t\r]+$/, "", s)
+      return s
+    }
+    function parse_directive(line,    colon, key, val) {
+      colon = index(line, ":")
+      if (colon == 0) return ""
+      key = lower(trim(substr(line, 1, colon - 1)))
+      val = trim(substr(line, colon + 1))
+      return key "\t" val
+    }
+    function emit(kind, value,    i, u) {
+      for (i = 1; i <= n_uas; i++) {
+        u = uas[i]
+        if (lower(u) == lower(ua)) {
+          print "EXACT_" kind "\t" value
+        }
+        if (u == "*") {
+          print "WILD_" kind "\t" value
+        }
+      }
+    }
+    BEGIN { n_uas = 0; prev_was_rule = 0 }
+    {
+      line = $0
+      # Strip comments
+      hash = index(line, "#")
+      if (hash > 0) line = substr(line, 1, hash - 1)
+      line = trim(line)
+      if (line == "") next
+
+      parsed = parse_directive(line)
+      if (parsed == "") next
+
+      tab = index(parsed, "\t")
+      key = substr(parsed, 1, tab - 1)
+      val = substr(parsed, tab + 1)
+
+      if (key == "user-agent") {
+        if (prev_was_rule) {
+          n_uas = 0
+          prev_was_rule = 0
+        }
+        n_uas++
+        uas[n_uas] = val
+        next
+      }
+      if (key == "disallow") { prev_was_rule = 1; emit("DISALLOW", val); next }
+      if (key == "allow")    { prev_was_rule = 1; emit("ALLOW", val); next }
+      if (key == "crawl-delay") { prev_was_rule = 1; emit("DELAY", val); next }
+    }
+  ' "$ROBOTS_FILE" > "$RAW_FILE"
+
+  # Prefer exact UA rules if present, else wildcard
+  PREFIX="WILD_"
+  if grep -q '^EXACT_' "$RAW_FILE"; then
+    PREFIX="EXACT_"
+  fi
+
+  grep "^${PREFIX}DISALLOW" "$RAW_FILE" 2>/dev/null \
+    | cut -f2- \
+    | grep -v '^$' \
+    > "$DISALLOWED_PATHS_FILE" || true
+
+  grep "^${PREFIX}ALLOW" "$RAW_FILE" 2>/dev/null \
+    | cut -f2- \
+    > "$ALLOW_PATHS_FILE" || true
+
+  DELAY_LINE=$(grep "^${PREFIX}DELAY" "$RAW_FILE" 2>/dev/null | head -1 | cut -f2- || true)
+  if [ -n "$DELAY_LINE" ]; then
+    if printf '%s' "$DELAY_LINE" | grep -qE '^[0-9]+(\.[0-9]+)?$'; then
+      CRAWL_DELAY="$DELAY_LINE"
+    fi
+  fi
+
+  # Longest-match path check (allow overrides disallow at equal or longer length)
+  BEST_MATCH_LEN=-1
+  BEST_MATCH_KIND="allow"
+
+  match_pattern() {
+    # Convert robots.txt glob (* and $) to a regex prefix check
+    local pat="$1"
+    local path="$2"
+    # Escape regex special chars except * and $
+    local esc
+    esc=$(printf '%s' "$pat" | sed 's/[].[\^$()+?{|]/\\&/g' | sed 's/\*/.*/g')
+    printf '%s' "$path" | grep -qE "^${esc}"
+  }
+
+  while IFS= read -r pat; do
+    [ -z "$pat" ] && continue
+    if match_pattern "$pat" "$URL_PATH"; then
+      PAT_LEN=${#pat}
+      if [ "$PAT_LEN" -gt "$BEST_MATCH_LEN" ]; then
+        BEST_MATCH_LEN=$PAT_LEN
+        BEST_MATCH_KIND="disallow"
+      fi
+    fi
+  done < "$DISALLOWED_PATHS_FILE"
+
+  while IFS= read -r pat; do
+    [ -z "$pat" ] && continue
+    if match_pattern "$pat" "$URL_PATH"; then
+      PAT_LEN=${#pat}
+      if [ "$PAT_LEN" -ge "$BEST_MATCH_LEN" ]; then
+        BEST_MATCH_LEN=$PAT_LEN
+        BEST_MATCH_KIND="allow"
+      fi
+    fi
+  done < "$ALLOW_PATHS_FILE"
+
+  if [ "$BEST_MATCH_KIND" = "disallow" ]; then
+    ALLOWED=false
+  fi
+fi
+
+# Build JSON arrays
+DISALLOWED_JSON="[]"
+if [ -s "$DISALLOWED_PATHS_FILE" ]; then
+  DISALLOWED_JSON=$(head -100 "$DISALLOWED_PATHS_FILE" | jq -R . | jq -s .)
+fi
+
+SITEMAPS_JSON="[]"
+if [ -s "$SITEMAPS_FILE" ]; then
+  SITEMAPS_JSON=$(jq -R . < "$SITEMAPS_FILE" | jq -s .)
+fi
+
+jq -n \
+  --arg url "$URL" \
+  --arg uaToken "$UA_TOKEN" \
+  --arg robotsUrl "$ROBOTS_URL" \
+  --argjson exists "$EXISTS" \
+  --argjson allowed "$ALLOWED" \
+  --argjson crawlDelay "$CRAWL_DELAY" \
+  --argjson disallowedPaths "$DISALLOWED_JSON" \
+  --argjson sitemaps "$SITEMAPS_JSON" \
+  '{
+    url: $url,
+    uaToken: $uaToken,
+    robotsUrl: $robotsUrl,
+    exists: $exists,
+    allowed: $allowed,
+    crawlDelay: $crawlDelay,
+    disallowedPaths: $disallowedPaths,
+    sitemaps: $sitemaps
+  }'