@braedenbuilds/crawl-sim 1.1.0 → 1.3.0

package/.claude-plugin/marketplace.json

@@ -0,0 +1,15 @@
+{
+  "name": "crawl-sim",
+  "owner": {
+    "name": "BraedenBDev",
+    "url": "https://github.com/BraedenBDev"
+  },
+  "plugins": [
+    {
+      "name": "crawl-sim",
+      "source": "./",
+      "description": "Multi-bot web crawler simulator — audit how Googlebot, GPTBot, ClaudeBot, and PerplexityBot see your site",
+      "version": "1.3.0"
+    }
+  ]
+}

package/.claude-plugin/plugin.json

@@ -0,0 +1,13 @@
+{
+  "name": "crawl-sim",
+  "version": "1.3.0",
+  "description": "Multi-bot web crawler simulator — audit how Googlebot, GPTBot, ClaudeBot, and PerplexityBot see your site",
+  "author": {
+    "name": "BraedenBDev",
+    "url": "https://github.com/BraedenBDev"
+  },
+  "homepage": "https://github.com/BraedenBDev/crawl-sim#readme",
+  "repository": "https://github.com/BraedenBDev/crawl-sim",
+  "license": "MIT",
+  "keywords": ["seo", "crawler", "ai-visibility", "claude-code-skill", "googlebot", "gptbot", "claudebot", "perplexitybot"]
+}

package/README.md

@@ -44,15 +44,20 @@ The concept was validated manually: a curl-as-GPTBot + Claude analysis caught a
 
 ## Quick start
 
-### In Claude Code (recommended)
+### As a Claude Code plugin (recommended)
+
+```
+/plugin install BraedenBDev/crawl-sim@github
+```
+
+Or add as a marketplace for easy updates:
 
-```bash
-npm install -g @braedenbuilds/crawl-sim
-crawl-sim install              # → ~/.claude/skills/crawl-sim/
-crawl-sim install --project    # → .claude/skills/crawl-sim/
+```
+/plugin marketplace add BraedenBDev/crawl-sim
+/plugin install crawl-sim@crawl-sim
 ```
 
-Then in Claude Code:
+Then invoke:
 
 ```
 /crawl-sim https://yoursite.com
@@ -60,6 +65,14 @@ Then in Claude Code:
 
 Claude runs the full pipeline, interprets the results, and returns a score card plus prioritized findings.
 
+### Via npm (alternative)
+
+```bash
+npm install -g @braedenbuilds/crawl-sim
+crawl-sim install              # → ~/.claude/skills/crawl-sim/
+crawl-sim install --project    # → .claude/skills/crawl-sim/
+```
+
 > **Why `npm install -g` instead of `npx`?** Recent versions of npx have a known issue linking bins for scoped single-bin packages in ephemeral installs. A persistent global install avoids the problem entirely. The git clone path below is the zero-npm fallback.
 
 ### As a standalone CLI

package/bin/install.js

@@ -14,6 +14,7 @@ const os = require('os');
 const { execFileSync } = require('child_process');
 
 const SOURCE_DIR = path.resolve(__dirname, '..');
+const SKILL_ROOT = path.resolve(SOURCE_DIR, 'skills', 'crawl-sim');
 const SKILL_FILES = ['SKILL.md'];
 const SKILL_DIRS = ['profiles', 'scripts'];
 
@@ -80,7 +81,9 @@ function install(target) {
   fs.mkdirSync(target, { recursive: true });
 
   for (const file of SKILL_FILES) {
-    const src = path.join(SOURCE_DIR, file);
+    // Look in skills/crawl-sim/ first (canonical), fallback to root (symlink)
+    let src = path.join(SKILL_ROOT, file);
+    if (!fs.existsSync(src)) src = path.join(SOURCE_DIR, file);
     const dest = path.join(target, file);
     if (fs.existsSync(src)) {
       fs.copyFileSync(src, dest);
@@ -92,7 +95,8 @@ function install(target) {
   }
 
   for (const dir of SKILL_DIRS) {
-    const src = path.join(SOURCE_DIR, dir);
+    let src = path.join(SKILL_ROOT, dir);
+    if (!fs.existsSync(src)) src = path.join(SOURCE_DIR, dir);
     const dest = path.join(target, dir);
     if (fs.existsSync(src)) {
       if (fs.existsSync(dest)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@braedenbuilds/crawl-sim",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "description": "Agent-native multi-bot web crawler simulator. See your site through the eyes of Googlebot, GPTBot, ClaudeBot, and PerplexityBot.",
   "bin": {
     "crawl-sim": "bin/install.js"
@@ -40,9 +40,11 @@
   },
   "files": [
     "bin/",
+    "skills/",
+    ".claude-plugin/",
     "SKILL.md",
-    "profiles/",
-    "scripts/",
+    "profiles",
+    "scripts",
     "README.md",
     "LICENSE"
   ]

package/{SKILL.md → skills/crawl-sim/SKILL.md}

@@ -40,7 +40,10 @@ command -v curl >/dev/null 2>&1 || { echo "ERROR: curl is required"; exit 1; }
 command -v jq   >/dev/null 2>&1 || { echo "ERROR: jq is required (brew install jq)"; exit 1; }
 ```
 
-Locate the skill directory: typically `~/.claude/skills/crawl-sim/` or `.claude/skills/crawl-sim/`. Use `$CLAUDE_PLUGIN_ROOT` if set, otherwise find the directory containing this `SKILL.md`.
+Locate the skill directory. Check in this order:
+1. `$CLAUDE_PLUGIN_ROOT/skills/crawl-sim` (plugin install)
+2. `~/.claude/skills/crawl-sim/` (global npm install)
+3. `.claude/skills/crawl-sim/` (project-level install)
 
 ## Orchestration — five narrated stages
 
@@ -48,14 +51,32 @@ Split the work into **five Bash invocations**, each with a clear `description` f
 
 ### Stage 1 — Fetch
 
-Tell the user: "Fetching as Googlebot, GPTBot, ClaudeBot, and PerplexityBot..."
+Tell the user: "Fetching as Googlebot, GPTBot, ClaudeBot, and PerplexityBot in parallel..."
 
 ```bash
-SKILL_DIR="$HOME/.claude/skills/crawl-sim"  # or wherever this SKILL.md lives
+# Resolve skill directory
+if [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -d "$CLAUDE_PLUGIN_ROOT/skills/crawl-sim" ]; then
+  SKILL_DIR="$CLAUDE_PLUGIN_ROOT/skills/crawl-sim"
+elif [ -d "$HOME/.claude/skills/crawl-sim" ]; then
+  SKILL_DIR="$HOME/.claude/skills/crawl-sim"
+elif [ -d ".claude/skills/crawl-sim" ]; then
+  SKILL_DIR=".claude/skills/crawl-sim"
+else
+  echo "ERROR: cannot find crawl-sim skill directory" >&2; exit 1
+fi
 RUN_DIR=$(mktemp -d -t crawl-sim.XXXXXX)
 URL="<user-provided-url>"
 for bot in googlebot gptbot claudebot perplexitybot; do
-  "$SKILL_DIR/scripts/fetch-as-bot.sh" "$URL" "$SKILL_DIR/profiles/${bot}.json" > "$RUN_DIR/fetch-${bot}.json"
+  "$SKILL_DIR/scripts/fetch-as-bot.sh" "$URL" "$SKILL_DIR/profiles/${bot}.json" > "$RUN_DIR/fetch-${bot}.json" &
+done
+wait
+
+# Verify no empty fetch files (guard against silent parallel failures)
+for bot in googlebot gptbot claudebot perplexitybot; do
+  if [ ! -s "$RUN_DIR/fetch-${bot}.json" ]; then
+    echo "WARNING: fetch-${bot}.json is empty — retrying serially" >&2
+    "$SKILL_DIR/scripts/fetch-as-bot.sh" "$URL" "$SKILL_DIR/profiles/${bot}.json" > "$RUN_DIR/fetch-${bot}.json"
+  fi
 done
 ```
 
@@ -112,7 +133,7 @@ Tell the user: "Computing per-bot scores and finalizing the report..."
 
 ```bash
 "$SKILL_DIR/scripts/compute-score.sh" "$RUN_DIR" > "$RUN_DIR/score.json"
-cp "$RUN_DIR/score.json" ./crawl-sim-report.json
+"$SKILL_DIR/scripts/build-report.sh" "$RUN_DIR" > ./crawl-sim-report.json
 ```
 
 **Page-type awareness.** `compute-score.sh` derives a page type from the target URL (`root` / `detail` / `archive` / `faq` / `about` / `contact` / `generic`) and picks a schema rubric accordingly. Root pages are expected to ship `Organization` + `WebSite` — penalizing them for missing `BreadcrumbList` or `FAQPage` would be wrong, so the scorer doesn't. If the URL heuristic picks the wrong type (e.g., a homepage at `/en/` that URL-parses as generic), pass `--page-type <type>`:
@@ -150,6 +171,18 @@ Print a boxed score card to the terminal:
 
 Progress bars are 20 chars wide using `█` and `░` (each char = 5%).
 
+**Parity-aware display.** When `parity.score >= 95` AND all per-bot composite scores are within 5 points of each other, collapse the four bot rows into one:
+
+```
+║  All 4 bots     98  A   ███████████████████░  (parity: content identical)  ║
+```
+
+Only show individual bot rows when scores diverge — that's when per-bot detail adds information. Always show the parity line in the category breakdown:
+
+```
+║  Content Parity   100  A   (all bots see the same content)                 ║
+```
+
 ## Output Layer 2 — Narrative Audit
 
 Lead with a **Bot differentiation summary** — state up front whether the bots scored the same or differently, and why. If they scored the same, explicitly say so:

package/skills/crawl-sim/scripts/build-report.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -eu
+
+# build-report.sh — Consolidate all crawl-sim outputs into a single JSON report
+# Usage: build-report.sh <results-dir>
+# Output: JSON to stdout
+
+RESULTS_DIR="${1:?Usage: build-report.sh <results-dir>}"
+
+if [ ! -f "$RESULTS_DIR/score.json" ]; then
+  echo "Error: score.json not found in $RESULTS_DIR — run compute-score.sh first" >&2
+  exit 1
+fi
+
+SCORE=$(cat "$RESULTS_DIR/score.json")
+
+# Collect per-bot raw data
+PER_BOT="{}"
+for f in "$RESULTS_DIR"/fetch-*.json; do
+  [ -f "$f" ] || continue
+  bot_id=$(basename "$f" .json | sed 's/^fetch-//')
+
+  BOT_RAW=$(jq -n \
+    --argjson fetch "$(jq '{status, timing, size, wordCount, redirectCount, finalUrl, redirectChain, fetchFailed, error}' "$f" 2>/dev/null || echo '{}')" \
+    --argjson meta "$(jq '.' "$RESULTS_DIR/meta-$bot_id.json" 2>/dev/null || echo '{}')" \
+    --argjson jsonld "$(jq '{blockCount, types, blocks}' "$RESULTS_DIR/jsonld-$bot_id.json" 2>/dev/null || echo '{}')" \
+    --argjson links "$(jq '.' "$RESULTS_DIR/links-$bot_id.json" 2>/dev/null || echo '{}')" \
+    --argjson robots "$(jq '.' "$RESULTS_DIR/robots-$bot_id.json" 2>/dev/null || echo '{}')" \
+    '{fetch: $fetch, meta: $meta, jsonld: $jsonld, links: $links, robots: $robots}')
+
+  PER_BOT=$(printf '%s' "$PER_BOT" | jq --argjson raw "$BOT_RAW" --arg id "$bot_id" '.[$id] = $raw')
+done
+
+# Collect independent (non-per-bot) data
+INDEPENDENT=$(jq -n \
+  --argjson sitemap "$(jq '.' "$RESULTS_DIR/sitemap.json" 2>/dev/null || echo '{}')" \
+  --argjson llmstxt "$(jq '.' "$RESULTS_DIR/llmstxt.json" 2>/dev/null || echo '{}')" \
+  --argjson diffRender "$(jq '.' "$RESULTS_DIR/diff-render.json" 2>/dev/null || echo '{"skipped":true,"reason":"not_found"}')" \
+  '{sitemap: $sitemap, llmstxt: $llmstxt, diffRender: $diffRender}')
+
+# Merge score + raw data
+printf '%s' "$SCORE" | jq \
+  --argjson perBot "$PER_BOT" \
+  --argjson independent "$INDEPENDENT" \
+  '. + {raw: {perBot: $perBot, independent: $independent}}'

package/{scripts → skills/crawl-sim/scripts}/check-llmstxt.sh

@@ -79,8 +79,12 @@ LLMS_FULL_HAS_TITLE=$HAS_TITLE
 LLMS_FULL_HAS_DESC=$HAS_DESCRIPTION
 LLMS_FULL_URLS=$URL_COUNT
 
+TOP_EXISTS=false
+[ "$LLMS_EXISTS" = "true" ] || [ "$LLMS_FULL_EXISTS" = "true" ] && TOP_EXISTS=true
+
 jq -n \
   --arg url "$URL" \
+  --argjson topExists "$TOP_EXISTS" \
   --arg llmsUrl "${ORIGIN}/llms.txt" \
   --arg llmsFullUrl "${ORIGIN}/llms-full.txt" \
   --argjson llmsExists "$LLMS_EXISTS" \
@@ -96,6 +100,7 @@ jq -n \
   --argjson llmsFullUrls "$LLMS_FULL_URLS" \
   '{
     url: $url,
+    exists: $topExists,
     llmsTxt: {
       url: $llmsUrl,
       exists: $llmsExists,

package/{scripts → skills/crawl-sim/scripts}/check-sitemap.sh

@@ -25,6 +25,7 @@ CONTAINS_TARGET=false
 HAS_LASTMOD=false
 IS_INDEX=false
 CHILD_SITEMAP_COUNT=0
+SAMPLE_URLS="[]"
 
 if [ "$HTTP_STATUS" = "200" ] && [ -s "$SITEMAP_FILE" ]; then
   # Check if content looks like XML (not HTML fallback)
@@ -43,6 +44,12 @@ if [ "$HTTP_STATUS" = "200" ] && [ -s "$SITEMAP_FILE" ]; then
       # Count <loc> tags (URLs, or child sitemaps in an index)
       URL_COUNT=$(grep -oE '<loc>' "$SITEMAP_FILE" | wc -l | tr -d ' ')
 
+      # Extract first 10 <loc> URLs as sample
+      SAMPLE_URLS=$(grep -oE '<loc>[^<]+</loc>' "$SITEMAP_FILE" \
+        | sed -E 's/<\/?loc>//g' \
+        | head -10 \
+        | jq -R . | jq -s .)
+
       # Check if target URL appears anywhere in the sitemap
       # Match both with and without trailing slash
       URL_NO_TRAILING=$(printf '%s' "$URL" | sed -E 's#/$##')
@@ -67,6 +74,7 @@ jq -n \
   --argjson childSitemapCount "$CHILD_SITEMAP_COUNT" \
   --argjson containsTarget "$CONTAINS_TARGET" \
   --argjson hasLastmod "$HAS_LASTMOD" \
+  --argjson sampleUrls "$SAMPLE_URLS" \
   '{
     url: $url,
     sitemapUrl: $sitemapUrl,
@@ -75,5 +83,6 @@ jq -n \
     urlCount: $urlCount,
     childSitemapCount: $childSitemapCount,
     containsTarget: $containsTarget,
-    hasLastmod: $hasLastmod
+    hasLastmod: $hasLastmod,
+    sampleUrls: $sampleUrls
   }'

package/{scripts → skills/crawl-sim/scripts}/compute-score.sh

@@ -21,6 +21,8 @@ set -eu
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=_lib.sh
 . "$SCRIPT_DIR/_lib.sh"
+# shellcheck source=schema-fields.sh
+. "$SCRIPT_DIR/schema-fields.sh"
 
 PAGE_TYPE_OVERRIDE=""
 while [ $# -gt 0 ]; do
@@ -276,12 +278,50 @@ for bot_id in $BOTS; do
   ROBOTS="$RESULTS_DIR/robots-$bot_id.json"
 
   BOT_NAME=$(jget "$FETCH" '.bot.name' "$bot_id")
-  STATUS=$(jget_num "$FETCH" '.status')
-  TOTAL_TIME=$(jget_num "$FETCH" '.timing.total')
-  SERVER_WORD_COUNT=$(jget_num "$FETCH" '.wordCount')
-  RENDERS_JS=$(jq -r '.bot.rendersJavaScript | if . == null then "unknown" else tostring end' "$FETCH" 2>/dev/null || echo "unknown")
 
-  ROBOTS_ALLOWED=$(jget_bool "$ROBOTS" '.allowed')
+  # Check for fetch failure — skip scoring, emit F grade (AC-A3)
+  FETCH_FAILED=$(jget_bool "$FETCH" '.fetchFailed')
+  if [ "$FETCH_FAILED" = "true" ]; then
+    FETCH_ERROR=$(jget "$FETCH" '.error' "unknown error")
+    RENDERS_JS=$(jq -r '.bot.rendersJavaScript | if . == null then "unknown" else tostring end' "$FETCH" 2>/dev/null || echo "unknown")
+    BOT_OBJ=$(jq -n \
+      --arg id "$bot_id" \
+      --arg name "$BOT_NAME" \
+      --arg rendersJs "$RENDERS_JS" \
+      --arg error "$FETCH_ERROR" \
+      '{
+        id: $id,
+        name: $name,
+        rendersJavaScript: (if $rendersJs == "true" then true elif $rendersJs == "false" then false else $rendersJs end),
+        fetchFailed: true,
+        error: $error,
+        score: 0,
+        grade: "F",
+        visibility: { serverWords: 0, effectiveWords: 0, missedWordsVsRendered: 0, hydrationPenaltyPts: 0 },
+        categories: {
+          accessibility:     { score: 0, grade: "F" },
+          contentVisibility: { score: 0, grade: "F" },
+          structuredData:    { score: 0, grade: "F", pageType: "unknown", expected: [], optional: [], forbidden: [], present: [], missing: [], extras: [], violations: [{ kind: "fetch_failed", impact: -100 }], calculation: "fetch failed — no data to score", notes: ("Fetch failed: " + $error) },
+          technicalSignals:  { score: 0, grade: "F" },
+          aiReadiness:       { score: 0, grade: "F" }
+        }
+      }')
+    BOTS_JSON=$(printf '%s' "$BOTS_JSON" | jq --argjson bot "$BOT_OBJ" --arg id "$bot_id" '.[$id] = $bot')
+    printf '[compute-score] %s: fetch failed, scoring as F\n' "$bot_id" >&2
+    CAT_N=$((CAT_N + 1))
+    continue
+  fi
+
+  # Batch-read fields from fetch file (1 jq call instead of 4)
+  read -r STATUS TOTAL_TIME SERVER_WORD_COUNT RENDERS_JS <<< \
+    "$(jq -r '[
+      (.status // 0),
+      (.timing.total // 0),
+      (.wordCount // 0),
+      (.bot.rendersJavaScript | if . == null then "unknown" else tostring end)
+    ] | @tsv' "$FETCH" 2>/dev/null || echo "0	0	0	unknown")"
+
+  ROBOTS_ALLOWED=$(jq -r '.allowed // false | tostring' "$ROBOTS" 2>/dev/null || echo "false")
 
   EFFECTIVE_WORD_COUNT=$SERVER_WORD_COUNT
   HYDRATION_PENALTY=0
@@ -305,10 +345,15 @@ for bot_id in $BOTS; do
 
   # --- Category 1: Accessibility (0-100) ---
   ACC=0
-  [ "$ROBOTS_ALLOWED" = "true" ] && ACC=$((ACC + 40))
-  [ "$STATUS" = "200" ] && ACC=$((ACC + 40))
-  TIME_SCORE=$(awk -v t="$TOTAL_TIME" 'BEGIN { if (t < 2) print 20; else if (t < 5) print 10; else print 0 }')
-  ACC=$((ACC + TIME_SCORE))
+  if [ "$ROBOTS_ALLOWED" != "true" ]; then
+    # R4 critical-fail: robots blocking overrides accessibility to 0/F
+    ACC=0
+  else
+    ACC=$((ACC + 40))
+    [ "$STATUS" = "200" ] && ACC=$((ACC + 40))
+    TIME_SCORE=$(awk -v t="$TOTAL_TIME" 'BEGIN { if (t < 2) print 20; else if (t < 5) print 10; else print 0 }')
+    ACC=$((ACC + TIME_SCORE))
+  fi
 
   # --- Category 2: Content Visibility (0-100) ---
   CONTENT=0
@@ -317,18 +362,23 @@ for bot_id in $BOTS; do
   elif [ "$EFFECTIVE_WORD_COUNT" -ge 50 ]; then CONTENT=$((CONTENT + 10))
   fi
 
-  H1_COUNT=$(jget_num "$META" '.headings.h1.count')
-  H2_COUNT=$(jget_num "$META" '.headings.h2.count')
+  # Batch-read fields from meta + links (1 jq call instead of 4 + 1)
+  read -r H1_COUNT H2_COUNT IMG_TOTAL IMG_WITH_ALT <<< \
+    "$(jq -r '[
+      (.headings.h1.count // 0),
+      (.headings.h2.count // 0),
+      (.images.total // 0),
+      (.images.withAlt // 0)
+    ] | @tsv' "$META" 2>/dev/null || echo "0	0	0	0")"
+
+  INTERNAL_LINKS=$(jq -r 'if (.internal | type) == "number" then .internal else .counts.internal // 0 end' "$LINKS" 2>/dev/null || echo "0")
+
   [ "$H1_COUNT" -ge 1 ] && CONTENT=$((CONTENT + 20))
   [ "$H2_COUNT" -ge 1 ] && CONTENT=$((CONTENT + 15))
 
-  INTERNAL_LINKS=$(jget_num "$LINKS" '.counts.internal')
   if [ "$INTERNAL_LINKS" -ge 5 ]; then CONTENT=$((CONTENT + 20))
   elif [ "$INTERNAL_LINKS" -ge 1 ]; then CONTENT=$((CONTENT + 10))
   fi
-
-  IMG_TOTAL=$(jget_num "$META" '.images.total')
-  IMG_WITH_ALT=$(jget_num "$META" '.images.withAlt')
   if [ "$IMG_TOTAL" -eq 0 ]; then
     CONTENT=$((CONTENT + 15))
   else
@@ -376,15 +426,42 @@ for bot_id in $BOTS; do
     [ $VALID_PENALTY -gt 20 ] && VALID_PENALTY=20
   fi
 
-  STRUCTURED=$((BASE + BONUS - FORBID_PENALTY - VALID_PENALTY))
+  # Field-level validation (C3): check required fields per schema type
+  FIELD_PENALTY=0
+  FIELD_VIOLATIONS_JSON="[]"
+  BLOCK_COUNT_FOR_FIELDS=0
+  if [ -f "$JSONLD" ]; then
+    BLOCK_COUNT_FOR_FIELDS=$(jq 'if has("blocks") then .blocks | length else 0 end' "$JSONLD" 2>/dev/null || echo "0")
+  fi
+  if [ "$BLOCK_COUNT_FOR_FIELDS" -gt 0 ]; then
+    i=0
+    while [ "$i" -lt "$BLOCK_COUNT_FOR_FIELDS" ]; do
+      BLOCK_TYPE=$(jq -r ".blocks[$i].type" "$JSONLD" 2>/dev/null || echo "")
+      BLOCK_FIELDS=$(jq -r ".blocks[$i].fields[]?" "$JSONLD" 2>/dev/null | tr '\n' ' ')
+      REQUIRED=$(required_fields_for "$BLOCK_TYPE")
+      for field in $REQUIRED; do
+        # shellcheck disable=SC2086
+        if ! list_contains "$field" $BLOCK_FIELDS; then
+          FIELD_VIOLATIONS_JSON=$(printf '%s' "$FIELD_VIOLATIONS_JSON" | jq \
+            --arg schema "$BLOCK_TYPE" --arg field "$field" \
+            '. + [{kind: "missing_required_field", schema: $schema, field: $field, impact: -5, confidence: "high"}]')
+          FIELD_PENALTY=$((FIELD_PENALTY + 5))
+        fi
+      done
+      i=$((i + 1))
+    done
+  fi
+  [ $FIELD_PENALTY -gt 30 ] && FIELD_PENALTY=30
+
+  STRUCTURED=$((BASE + BONUS - FORBID_PENALTY - VALID_PENALTY - FIELD_PENALTY))
   [ $STRUCTURED -gt 100 ] && STRUCTURED=100
   [ $STRUCTURED -lt 0 ] && STRUCTURED=0
 
-  CALCULATION=$(printf 'base: %d/%d expected present = %d; +%d optional bonus; -%d forbidden penalty; -%d validity penalty; clamp [0,100] = %d' \
+  CALCULATION=$(printf 'base: %d/%d expected present = %d; +%d optional bonus; -%d forbidden penalty; -%d validity penalty; -%d field penalty; clamp [0,100] = %d' \
     "$PRESENT_EXPECTED_COUNT" "$EXPECTED_COUNT" "$BASE" \
-    "$BONUS" "$FORBID_PENALTY" "$VALID_PENALTY" "$STRUCTURED")
+    "$BONUS" "$FORBID_PENALTY" "$VALID_PENALTY" "$FIELD_PENALTY" "$STRUCTURED")
 
-  if [ "$STRUCTURED" -ge 100 ] && [ -z "$PRESENT_FORBIDDEN" ] && [ "$VALID_PENALTY" -eq 0 ]; then
+  if [ "$STRUCTURED" -ge 100 ] && [ -z "$PRESENT_FORBIDDEN" ] && [ "$VALID_PENALTY" -eq 0 ] && [ "$FIELD_PENALTY" -eq 0 ]; then
     NOTES="All expected schemas for pageType=$PAGE_TYPE are present. No structured-data action needed."
   elif [ -n "$MISSING_EXPECTED" ] && [ -z "$PRESENT_FORBIDDEN" ]; then
     NOTES="Missing expected schemas for pageType=$PAGE_TYPE: $MISSING_EXPECTED. Add these to raise the score."
@@ -392,8 +469,12 @@ for bot_id in $BOTS; do
     NOTES="Forbidden schemas present for pageType=$PAGE_TYPE: $PRESENT_FORBIDDEN. Remove these (or re-classify the page type with --page-type)."
   elif [ -n "$PRESENT_FORBIDDEN" ] && [ -n "$MISSING_EXPECTED" ]; then
     NOTES="Mixed: missing $MISSING_EXPECTED and forbidden present $PRESENT_FORBIDDEN for pageType=$PAGE_TYPE."
-  else
+  elif [ "$FIELD_PENALTY" -gt 0 ]; then
+    NOTES="Schemas for pageType=$PAGE_TYPE are present but missing required fields. See violations for details."
+  elif [ "$VALID_PENALTY" -gt 0 ]; then
     NOTES="Score reduced by $VALID_PENALTY pts due to invalid JSON-LD blocks."
+  else
+    NOTES="Structured data scored for pageType=$PAGE_TYPE."
   fi
 
   STRUCTURED_GRADE=$(grade_for "$STRUCTURED")
@@ -410,6 +491,7 @@ for bot_id in $BOTS; do
     --arg forbiddenPresent "$PRESENT_FORBIDDEN" \
     --argjson invalidCount "$JSONLD_INVALID" \
     --argjson validPenalty "$VALID_PENALTY" \
+    --argjson fieldViolations "$FIELD_VIOLATIONS_JSON" \
     --arg calculation "$CALCULATION" \
     --arg notes "$NOTES" \
     '
@@ -425,11 +507,12 @@ for bot_id in $BOTS; do
       missing:    ($missingList    | to_arr),
       extras:     ($extrasList     | to_arr),
       violations: (
-        ($forbiddenPresent | to_arr | map({kind: "forbidden_schema", schema: ., impact: -10}))
+        ($forbiddenPresent | to_arr | map({kind: "forbidden_schema", schema: ., impact: -10, confidence: "high"}))
         + (if $validPenalty > 0
-             then [{kind: "invalid_jsonld", count: $invalidCount, impact: (0 - $validPenalty)}]
+             then [{kind: "invalid_jsonld", count: $invalidCount, impact: (0 - $validPenalty), confidence: "high"}]
              else []
            end)
+        + $fieldViolations
       ),
       calculation: $calculation,
       notes: $notes
@@ -438,20 +521,24 @@ for bot_id in $BOTS; do
 
   # --- Category 4: Technical Signals (0-100) ---
   TECHNICAL=0
-  TITLE=$(jget "$META" '.title' "")
-  DESCRIPTION=$(jget "$META" '.description' "")
-  CANONICAL=$(jget "$META" '.canonical' "")
-  OG_TITLE=$(jget "$META" '.og.title' "")
-  OG_DESC=$(jget "$META" '.og.description' "")
-
-  [ -n "$TITLE" ] && [ "$TITLE" != "null" ] && TECHNICAL=$((TECHNICAL + 25))
-  [ -n "$DESCRIPTION" ] && [ "$DESCRIPTION" != "null" ] && TECHNICAL=$((TECHNICAL + 25))
-  [ -n "$CANONICAL" ] && [ "$CANONICAL" != "null" ] && TECHNICAL=$((TECHNICAL + 20))
-  if [ -n "$OG_TITLE" ] && [ "$OG_TITLE" != "null" ]; then TECHNICAL=$((TECHNICAL + 8)); fi
-  if [ -n "$OG_DESC" ] && [ "$OG_DESC" != "null" ]; then TECHNICAL=$((TECHNICAL + 7)); fi
-
-  SITEMAP_EXISTS=$(jget_bool "$SITEMAP_FILE" '.exists')
-  SITEMAP_CONTAINS=$(jget_bool "$SITEMAP_FILE" '.containsTarget')
+  # Batch-read meta fields for technical scoring (1 jq call instead of 5)
+  IFS=$'\t' read -r TITLE DESCRIPTION CANONICAL OG_TITLE OG_DESC <<< \
+    "$(jq -r '[
+      (.title // "" | gsub("\t"; " ")),
+      (.description // "" | gsub("\t"; " ")),
+      (.canonical // "" | gsub("\t"; " ")),
+      (.og.title // "" | gsub("\t"; " ")),
+      (.og.description // "" | gsub("\t"; " "))
+    ] | @tsv' "$META" 2>/dev/null || printf '\t\t\t\t')"
+
+  [ -n "$TITLE" ] && TECHNICAL=$((TECHNICAL + 25))
+  [ -n "$DESCRIPTION" ] && TECHNICAL=$((TECHNICAL + 25))
+  [ -n "$CANONICAL" ] && TECHNICAL=$((TECHNICAL + 20))
+  [ -n "$OG_TITLE" ] && TECHNICAL=$((TECHNICAL + 8))
+  [ -n "$OG_DESC" ] && TECHNICAL=$((TECHNICAL + 7))
+
+  SITEMAP_EXISTS=$(jq -r '.exists // false | tostring' "$SITEMAP_FILE" 2>/dev/null || echo "false")
+  SITEMAP_CONTAINS=$(jq -r '.containsTarget // false | tostring' "$SITEMAP_FILE" 2>/dev/null || echo "false")
   if [ "$SITEMAP_EXISTS" = "true" ] && [ "$SITEMAP_CONTAINS" = "true" ]; then
     TECHNICAL=$((TECHNICAL + 15))
   elif [ "$SITEMAP_EXISTS" = "true" ]; then
@@ -460,10 +547,14 @@ for bot_id in $BOTS; do
 
   # --- Category 5: AI Readiness (0-100) ---
   AI=0
-  LLMS_EXISTS=$(jget_bool "$LLMSTXT_FILE" '.llmsTxt.exists')
-  LLMS_HAS_TITLE=$(jget_bool "$LLMSTXT_FILE" '.llmsTxt.hasTitle')
-  LLMS_HAS_DESC=$(jget_bool "$LLMSTXT_FILE" '.llmsTxt.hasDescription')
-  LLMS_URLS=$(jget_num "$LLMSTXT_FILE" '.llmsTxt.urlCount')
+  # Batch-read llmstxt fields (1 jq call instead of 4)
+  read -r LLMS_EXISTS LLMS_HAS_TITLE LLMS_HAS_DESC LLMS_URLS <<< \
+    "$(jq -r '[
+      (.llmsTxt.exists // false | tostring),
+      (.llmsTxt.hasTitle // false | tostring),
+      (.llmsTxt.hasDescription // false | tostring),
+      (.llmsTxt.urlCount // 0)
+    ] | @tsv' "$LLMSTXT_FILE" 2>/dev/null || echo "false	false	false	0")"
 
   if [ "$LLMS_EXISTS" = "true" ]; then
     AI=$((AI + 40))
@@ -472,7 +563,7 @@ for bot_id in $BOTS; do
     [ "$LLMS_URLS" -ge 1 ] && AI=$((AI + 6))
   fi
   [ "$EFFECTIVE_WORD_COUNT" -ge 200 ] && AI=$((AI + 20))
-  if [ "$H1_COUNT" -ge 1 ] && [ -n "$DESCRIPTION" ] && [ "$DESCRIPTION" != "null" ]; then
+  if [ "$H1_COUNT" -ge 1 ] && [ -n "$DESCRIPTION" ]; then
     AI=$((AI + 20))
   fi
 
@@ -566,6 +657,60 @@ CAT_STRUCTURED_GRADE=$(grade_for "$CAT_STRUCTURED_AVG")
 CAT_TECHNICAL_GRADE=$(grade_for "$CAT_TECHNICAL_AVG")
 CAT_AI_GRADE=$(grade_for "$CAT_AI_AVG")
 
+# --- Cross-bot content parity (C4) ---
+PARITY_MIN_WORDS=999999999
+PARITY_MAX_WORDS=0
+PARITY_BOT_COUNT=0
+for bot_id in $BOTS; do
+  FETCH="$RESULTS_DIR/fetch-$bot_id.json"
+  P_FETCH_FAILED=$(jget_bool "$FETCH" '.fetchFailed')
+  [ "$P_FETCH_FAILED" = "true" ] && continue
+  WC=$(jget_num "$FETCH" '.wordCount')
+  [ "$WC" -lt "$PARITY_MIN_WORDS" ] && PARITY_MIN_WORDS=$WC
+  [ "$WC" -gt "$PARITY_MAX_WORDS" ] && PARITY_MAX_WORDS=$WC
+  PARITY_BOT_COUNT=$((PARITY_BOT_COUNT + 1))
+done
+
+if [ "$PARITY_BOT_COUNT" -le 1 ]; then
+  PARITY_SCORE=100
+  PARITY_MAX_DELTA=0
+elif [ "$PARITY_MAX_WORDS" -gt 0 ]; then
+  PARITY_SCORE=$(awk -v min="$PARITY_MIN_WORDS" -v max="$PARITY_MAX_WORDS" \
+    'BEGIN { printf "%d", (min / max) * 100 + 0.5 }')
+  PARITY_MAX_DELTA=$(awk -v min="$PARITY_MIN_WORDS" -v max="$PARITY_MAX_WORDS" \
+    'BEGIN { printf "%d", ((max - min) / max) * 100 + 0.5 }')
+else
+  PARITY_SCORE=100
+  PARITY_MAX_DELTA=0
+fi
+
+[ "$PARITY_SCORE" -gt 100 ] && PARITY_SCORE=100
+PARITY_GRADE=$(grade_for "$PARITY_SCORE")
+
+if [ "$PARITY_SCORE" -ge 95 ]; then
+  PARITY_INTERP="Content is consistent across all bots."
+elif [ "$PARITY_SCORE" -ge 50 ]; then
+  PARITY_INTERP="Moderate content divergence between bots — likely partial client-side rendering hydration."
+else
+  PARITY_INTERP="Severe content divergence — site likely relies on client-side rendering. AI bots see significantly less content than Googlebot."
+fi
+
+# --- Warnings (H2) ---
+WARNINGS="[]"
+if [ "$DIFF_AVAILABLE" != "true" ]; then
+  DIFF_REASON="not_found"
+  if [ -f "$DIFF_RENDER_FILE" ]; then
+    DIFF_REASON=$(jq -r '.reason // "skipped"' "$DIFF_RENDER_FILE" 2>/dev/null || echo "skipped")
+  fi
+  WARNINGS=$(printf '%s' "$WARNINGS" | jq --arg reason "$DIFF_REASON" \
+    '. + [{
+      code: "diff_render_unavailable",
+      severity: "high",
+      message: "JS rendering comparison was skipped. If this site uses CSR, non-JS bot scores may be inaccurate.",
+      reason: $reason
+    }]')
+fi
+
 TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
 
 jq -n \
@@ -587,6 +732,13 @@ jq -n \
   --arg catTechnicalGrade "$CAT_TECHNICAL_GRADE" \
   --argjson catAi "$CAT_AI_AVG" \
   --arg catAiGrade "$CAT_AI_GRADE" \
+  --argjson warnings "$WARNINGS" \
+  --argjson parityScore "$PARITY_SCORE" \
+  --arg parityGrade "$PARITY_GRADE" \
+  --argjson parityMinWords "$PARITY_MIN_WORDS" \
+  --argjson parityMaxWords "$PARITY_MAX_WORDS" \
+  --argjson parityMaxDelta "$PARITY_MAX_DELTA" \
+  --arg parityInterp "$PARITY_INTERP" \
   '{
     url: $url,
     timestamp: $timestamp,
@@ -594,6 +746,15 @@ jq -n \
     pageType: $pageType,
     pageTypeOverridden: ($pageTypeOverride | length > 0),
     overall: { score: $overallScore, grade: $overallGrade },
+    parity: {
+      score: $parityScore,
+      grade: $parityGrade,
+      minWords: (if $parityMinWords >= 999999999 then 0 else $parityMinWords end),
+      maxWords: $parityMaxWords,
+      maxDeltaPct: $parityMaxDelta,
+      interpretation: $parityInterp
+    },
+    warnings: $warnings,
     bots: $bots,
     categories: {
       accessibility:     { score: $catAcc,        grade: $catAccGrade },

package/{scripts → skills/crawl-sim/scripts}/extract-jsonld.sh

@@ -62,6 +62,7 @@ fi
 
 VALID_COUNT=0
 INVALID_COUNT=0
+BLOCKS_JSON="[]"
 
 if [ "$BLOCK_COUNT" -gt 0 ]; then
   while IFS= read -r block; do
@@ -79,6 +80,15 @@ if [ "$BLOCK_COUNT" -gt 0 ]; then
           else empty end;
         collect_types
       ' 2>/dev/null >> "$TYPES_FILE" || true
+
+      # Extract per-block type + top-level field names for field validation (AC-B1)
+      BLOCK_INFO=$(printf '%s' "$block" | jq -c '
+        {
+          type: (if has("@type") then (.["@type"] | if type == "array" then .[0] else . end) else "unknown" end),
+          fields: (keys | map(select(startswith("@") | not)))
+        }
+      ' 2>/dev/null || echo '{"type":"unknown","fields":[]}')
+      BLOCKS_JSON=$(printf '%s' "$BLOCKS_JSON" | jq --argjson b "$BLOCK_INFO" '. + [$b]')
     else
       INVALID_COUNT=$((INVALID_COUNT + 1))
     fi
@@ -109,6 +119,7 @@ jq -n \
   --argjson valid "$VALID_COUNT" \
   --argjson invalid "$INVALID_COUNT" \
   --argjson types "$TYPES_JSON" \
+  --argjson blocks "$BLOCKS_JSON" \
   --argjson hasOrg "$HAS_ORG" \
   --argjson hasBreadcrumb "$HAS_BREADCRUMB" \
   --argjson hasWebsite "$HAS_WEBSITE" \
@@ -121,6 +132,7 @@ jq -n \
     validCount: $valid,
     invalidCount: $invalid,
     types: $types,
+    blocks: $blocks,
     flags: {
       hasOrganization: $hasOrg,
       hasBreadcrumbList: $hasBreadcrumb,

package/{scripts → skills/crawl-sim/scripts}/extract-links.sh

@@ -93,11 +93,9 @@ jq -n \
   --argjson internalSample "$INTERNAL_SAMPLE" \
   --argjson externalSample "$EXTERNAL_SAMPLE" \
   '{
-    counts: {
-      internal: $internalCount,
-      external: $externalCount,
-      total: ($internalCount + $externalCount)
-    },
-    internal: $internalSample,
-    external: $externalSample
+    total: ($internalCount + $externalCount),
+    internal: $internalCount,
+    external: $externalCount,
+    internalUrls: $internalSample,
+    externalUrls: $externalSample
   }'

package/skills/crawl-sim/scripts/fetch-as-bot.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# fetch-as-bot.sh — Fetch a URL as a specific bot User-Agent
+# Usage: fetch-as-bot.sh <url> <profile.json>
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+URL="${1:?Usage: fetch-as-bot.sh <url> <profile.json>}"
+PROFILE="${2:?Usage: fetch-as-bot.sh <url> <profile.json>}"
+
+BOT_ID=$(jq -r '.id' "$PROFILE")
+BOT_NAME=$(jq -r '.name' "$PROFILE")
+UA=$(jq -r '.userAgent' "$PROFILE")
+RENDERS_JS=$(jq -r '.rendersJavaScript' "$PROFILE")
+
+TMPDIR="${TMPDIR:-/tmp}"
+HEADERS_FILE=$(mktemp "$TMPDIR/crawlsim-headers.XXXXXX")
+BODY_FILE=$(mktemp "$TMPDIR/crawlsim-body.XXXXXX")
+CURL_STDERR_FILE=$(mktemp "$TMPDIR/crawlsim-stderr.XXXXXX")
+trap 'rm -f "$HEADERS_FILE" "$BODY_FILE" "$CURL_STDERR_FILE"' EXIT
+
+printf '[%s] fetching %s\n' "$BOT_ID" "$URL" >&2
+
+set +e
+TIMING=$(curl -sS -L \
+  -H "User-Agent: $UA" \
+  -D "$HEADERS_FILE" \
+  -o "$BODY_FILE" \
+  -w '{"total":%{time_total},"ttfb":%{time_starttransfer},"connect":%{time_connect},"statusCode":%{http_code},"sizeDownload":%{size_download},"redirectCount":%{num_redirects},"finalUrl":"%{url_effective}"}' \
+  --max-time 30 \
+  "$URL" 2>"$CURL_STDERR_FILE")
+CURL_EXIT=$?
+set -e
+
+CURL_ERR=""
+if [ -s "$CURL_STDERR_FILE" ]; then
+  CURL_ERR=$(cat "$CURL_STDERR_FILE")
+fi
+
+if [ "$CURL_EXIT" -ne 0 ]; then
+  printf '[%s] FAILED: curl exit %d — %s\n' "$BOT_ID" "$CURL_EXIT" "$CURL_ERR" >&2
+  jq -n \
+    --arg url "$URL" \
+    --arg botId "$BOT_ID" \
+    --arg botName "$BOT_NAME" \
+    --arg ua "$UA" \
+    --arg rendersJs "$RENDERS_JS" \
+    --arg error "$CURL_ERR" \
+    --argjson exitCode "$CURL_EXIT" \
+    '{
+      url: $url,
+      bot: {
+        id: $botId,
+        name: $botName,
+        userAgent: $ua,
+        rendersJavaScript: (if $rendersJs == "true" then true elif $rendersJs == "false" then false else $rendersJs end)
+      },
+      fetchFailed: true,
+      error: $error,
+      curlExitCode: $exitCode,
+      status: 0,
+      timing: { total: 0, ttfb: 0 },
+      size: 0,
+      wordCount: 0,
+      headers: {},
+      bodyBase64: ""
+    }'
+  exit 0
+fi
+
+read -r STATUS TOTAL_TIME TTFB SIZE REDIRECT_COUNT FINAL_URL <<< \
+  "$(echo "$TIMING" | jq -r '[.statusCode, .total, .ttfb, .sizeDownload, .redirectCount, .finalUrl] | @tsv')"
+
+# Parse response headers into a JSON object using jq for safe escaping.
+# curl -L writes multiple blocks on redirect; jq keeps the last definition
+# of each header since `add` overwrites left-to-right.
+HEADERS_JSON=$(tr -d '\r' < "$HEADERS_FILE" \
+  | grep -E '^[A-Za-z][A-Za-z0-9-]*:[[:space:]]' \
+  | jq -Rs '
+      split("\n")
+      | map(select(length > 0))
+      | map(capture("^(?<k>[^:]+):[[:space:]]*(?<v>.*)$"))
+      | map({(.k): .v})
+      | add // {}
+    ')
+
+# Parse redirect chain from headers dump.
+# curl -D writes multiple HTTP response blocks on redirect — each starts with HTTP/.
+REDIRECT_CHAIN="[]"
+if [ "$REDIRECT_COUNT" -gt 0 ]; then
+  REDIRECT_CHAIN=$(tr -d '\r' < "$HEADERS_FILE" | awk '
+    /^HTTP\// { status=$2; url="" }
+    /^[Ll]ocation:/ { url=$2 }
+    /^$/ && status && url { printf "%s %s\n", status, url; status=""; url="" }
+  ' | jq -Rs '
+    split("\n") | map(select(length > 0)) |
+    to_entries | map({
+      hop: .key,
+      status: (.value | split(" ")[0] | tonumber),
+      location: (.value | split(" ")[1:] | join(" "))
+    })
+  ')
+fi
+
+WORD_COUNT=$(count_words "$BODY_FILE")
+[ -z "$WORD_COUNT" ] && WORD_COUNT=0
+
+BODY_B64=""
+if [ -s "$BODY_FILE" ]; then
+  BODY_B64=$(base64 < "$BODY_FILE")
+fi
+
+printf '[%s] ok: status=%s size=%s words=%s time=%ss\n' "$BOT_ID" "$STATUS" "$SIZE" "$WORD_COUNT" "$TOTAL_TIME" >&2
+
+jq -n \
+  --arg url "$URL" \
+  --arg botId "$BOT_ID" \
+  --arg botName "$BOT_NAME" \
+  --arg ua "$UA" \
+  --arg rendersJs "$RENDERS_JS" \
+  --argjson status "$STATUS" \
+  --argjson totalTime "$TOTAL_TIME" \
+  --argjson ttfb "$TTFB" \
+  --argjson size "$SIZE" \
+  --argjson wordCount "$WORD_COUNT" \
+  --argjson headers "$HEADERS_JSON" \
+  --argjson redirectCount "$REDIRECT_COUNT" \
+  --arg finalUrl "$FINAL_URL" \
+  --argjson redirectChain "$REDIRECT_CHAIN" \
+  --arg bodyBase64 "$BODY_B64" \
+  '{
+    url: $url,
+    bot: {
+      id: $botId,
+      name: $botName,
+      userAgent: $ua,
+      rendersJavaScript: (if $rendersJs == "true" then true elif $rendersJs == "false" then false else $rendersJs end)
+    },
+    status: $status,
+    timing: { total: $totalTime, ttfb: $ttfb },
+    size: $size,
+    wordCount: $wordCount,
+    redirectCount: $redirectCount,
+    finalUrl: $finalUrl,
+    redirectChain: $redirectChain,
+    headers: $headers,
+    bodyBase64: $bodyBase64
+  }'

package/skills/crawl-sim/scripts/schema-fields.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# schema-fields.sh — Required field definitions per schema.org type.
+# Source this file, then call required_fields_for <SchemaType>.
+
+required_fields_for() {
+  case "$1" in
+    Organization)        echo "name url" ;;
+    WebSite)             echo "name url" ;;
+    Article)             echo "headline author datePublished" ;;
+    NewsArticle)         echo "headline author datePublished" ;;
+    FAQPage)             echo "mainEntity" ;;
+    BreadcrumbList)      echo "itemListElement" ;;
+    CollectionPage)      echo "name" ;;
+    ItemList)            echo "itemListElement" ;;
+    AboutPage)           echo "name" ;;
+    ContactPage)         echo "name" ;;
+    Product)             echo "name" ;;
+    LocalBusiness)       echo "name address" ;;
+    ProfessionalService) echo "name" ;;
+    Person)              echo "name" ;;
+    ImageObject)         echo "contentUrl" ;;
+    PostalAddress)       echo "streetAddress" ;;
+    *)                   echo "" ;;
+  esac
+}

package/scripts/fetch-as-bot.sh

@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# fetch-as-bot.sh — Fetch a URL as a specific bot User-Agent
-# Usage: fetch-as-bot.sh <url> <profile.json>
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=_lib.sh
-. "$SCRIPT_DIR/_lib.sh"
-
-URL="${1:?Usage: fetch-as-bot.sh <url> <profile.json>}"
-PROFILE="${2:?Usage: fetch-as-bot.sh <url> <profile.json>}"
-
-BOT_ID=$(jq -r '.id' "$PROFILE")
-BOT_NAME=$(jq -r '.name' "$PROFILE")
-UA=$(jq -r '.userAgent' "$PROFILE")
-RENDERS_JS=$(jq -r '.rendersJavaScript' "$PROFILE")
-
-printf '[fetch-as-bot] %s <- %s\n' "$BOT_NAME" "$URL" >&2
-
-TMPDIR="${TMPDIR:-/tmp}"
-HEADERS_FILE=$(mktemp "$TMPDIR/crawlsim-headers.XXXXXX")
-BODY_FILE=$(mktemp "$TMPDIR/crawlsim-body.XXXXXX")
-trap 'rm -f "$HEADERS_FILE" "$BODY_FILE"' EXIT
-
-TIMING=$(curl -sS -L \
-  -H "User-Agent: $UA" \
-  -D "$HEADERS_FILE" \
-  -o "$BODY_FILE" \
-  -w '{"total":%{time_total},"ttfb":%{time_starttransfer},"connect":%{time_connect},"statusCode":%{http_code},"sizeDownload":%{size_download}}' \
-  --max-time 30 \
-  "$URL" 2>/dev/null || echo '{"total":0,"ttfb":0,"connect":0,"statusCode":0,"sizeDownload":0}')
-
-STATUS=$(echo "$TIMING" | jq -r '.statusCode')
-TOTAL_TIME=$(echo "$TIMING" | jq -r '.total')
-TTFB=$(echo "$TIMING" | jq -r '.ttfb')
-SIZE=$(echo "$TIMING" | jq -r '.sizeDownload')
-
-# Parse response headers into a JSON object using jq for safe escaping.
-# curl -L writes multiple blocks on redirect; jq keeps the last definition
-# of each header since `add` overwrites left-to-right.
-HEADERS_JSON=$(tr -d '\r' < "$HEADERS_FILE" \
-  | grep -E '^[A-Za-z][A-Za-z0-9-]*:[[:space:]]' \
-  | jq -Rs '
-      split("\n")
-      | map(select(length > 0))
-      | map(capture("^(?<k>[^:]+):[[:space:]]*(?<v>.*)$"))
-      | map({(.k): .v})
-      | add // {}
-    ')
-
-WORD_COUNT=$(count_words "$BODY_FILE")
-[ -z "$WORD_COUNT" ] && WORD_COUNT=0
-
-BODY_B64=""
-if [ -s "$BODY_FILE" ]; then
-  BODY_B64=$(base64 < "$BODY_FILE")
-fi
-
-jq -n \
-  --arg url "$URL" \
-  --arg botId "$BOT_ID" \
-  --arg botName "$BOT_NAME" \
-  --arg ua "$UA" \
-  --arg rendersJs "$RENDERS_JS" \
-  --argjson status "$STATUS" \
-  --argjson totalTime "$TOTAL_TIME" \
-  --argjson ttfb "$TTFB" \
-  --argjson size "$SIZE" \
-  --argjson wordCount "$WORD_COUNT" \
-  --argjson headers "$HEADERS_JSON" \
-  --arg bodyBase64 "$BODY_B64" \
-  '{
-    url: $url,
-    bot: {
-      id: $botId,
-      name: $botName,
-      userAgent: $ua,
-      rendersJavaScript: (if $rendersJs == "true" then true elif $rendersJs == "false" then false else $rendersJs end)
-    },
-    status: $status,
-    timing: { total: $totalTime, ttfb: $ttfb },
-    size: $size,
-    wordCount: $wordCount,
-    headers: $headers,
-    bodyBase64: $bodyBase64
-  }'