@braedenbuilds/crawl-sim 1.0.1

package/scripts/extract-jsonld.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+set -eu
+
+# extract-jsonld.sh — Extract JSON-LD structured data from HTML
+# Usage: extract-jsonld.sh [file] | extract-jsonld.sh < html
+# Output: JSON to stdout with count, types, and flags
+
+# Read HTML from file or stdin
+if [ $# -ge 1 ] && [ -f "$1" ]; then
+  HTML=$(cat "$1")
+  printf '[extract-jsonld] %s\n' "$1" >&2
+else
+  HTML=$(cat)
+  printf '[extract-jsonld] (stdin)\n' >&2
+fi
+
+# Extract JSON-LD blocks into a temp file (one block per line, flattened)
+# Match <script type="application/ld+json">...</script> across lines
+TMPDIR="${TMPDIR:-/tmp}"
+BLOCKS_FILE=$(mktemp "$TMPDIR/crawlsim-jsonld.XXXXXX")
+TYPES_FILE=$(mktemp "$TMPDIR/crawlsim-types.XXXXXX")
+VALID_FILE=$(mktemp "$TMPDIR/crawlsim-valid.XXXXXX")
+trap 'rm -f "$BLOCKS_FILE" "$TYPES_FILE" "$VALID_FILE"' EXIT
+
+# Use awk to extract script blocks across lines
+printf '%s' "$HTML" | awk '
+BEGIN { in_block = 0; block = "" }
+{
+  line = $0
+  while (length(line) > 0) {
+    if (in_block == 0) {
+      # Look for opening script tag (case-insensitive)
+      idx = match(tolower(line), /<script[^>]*type=["'\''"]application\/ld\+json["'\''"][^>]*>/)
+      if (idx == 0) break
+      # Skip past the opening tag
+      end_of_open = idx + RLENGTH - 1
+      line = substr(line, end_of_open + 1)
+      in_block = 1
+      block = ""
+    } else {
+      # Look for closing tag
+      idx = match(tolower(line), /<\/script>/)
+      if (idx == 0) {
+        block = block line " "
+        break
+      }
+      block = block substr(line, 1, idx - 1)
+      print block
+      line = substr(line, idx + RLENGTH)
+      in_block = 0
+    }
+  }
+}
+' > "$BLOCKS_FILE"
+
+# Count blocks
+BLOCK_COUNT=$(wc -l < "$BLOCKS_FILE" | tr -d ' ')
+# Handle empty file (wc returns 0 for empty)
+if [ ! -s "$BLOCKS_FILE" ]; then
+  BLOCK_COUNT=0
+fi
+
+VALID_COUNT=0
+INVALID_COUNT=0
+
+if [ "$BLOCK_COUNT" -gt 0 ]; then
+  while IFS= read -r block; do
+    [ -z "$block" ] && continue
+    # Try to parse as JSON
+    if printf '%s' "$block" | jq -e . >/dev/null 2>&1; then
+      VALID_COUNT=$((VALID_COUNT + 1))
+      # Extract @type values (may be single string, array, or nested under @graph)
+      printf '%s' "$block" | jq -r '
+        def collect_types:
+          if type == "object" then
+            (if has("@type") then (.["@type"] | if type == "array" then .[] else . end) else empty end),
+            (if has("@graph") then (.["@graph"][] | collect_types) else empty end)
+          elif type == "array" then .[] | collect_types
+          else empty end;
+        collect_types
+      ' 2>/dev/null >> "$TYPES_FILE" || true
+    else
+      INVALID_COUNT=$((INVALID_COUNT + 1))
+    fi
+  done < "$BLOCKS_FILE"
+fi
+
+# Deduplicate and sort types
+TYPES_JSON="[]"
+if [ -s "$TYPES_FILE" ]; then
+  TYPES_JSON=$(sort -u "$TYPES_FILE" | jq -R . | jq -s .)
+fi
+
+# Boolean flags for common types
+has_type() {
+  printf '%s' "$TYPES_JSON" | jq -e --arg t "$1" 'any(. == $t)' >/dev/null 2>&1 && echo true || echo false
+}
+
+HAS_ORG=$(has_type "Organization")
+HAS_BREADCRUMB=$(has_type "BreadcrumbList")
+HAS_WEBSITE=$(has_type "WebSite")
+HAS_ARTICLE=$(has_type "Article")
+HAS_FAQ=$(has_type "FAQPage")
+HAS_PRODUCT=$(has_type "Product")
+HAS_PROFESSIONAL_SERVICE=$(has_type "ProfessionalService")
+
+jq -n \
+  --argjson count "$BLOCK_COUNT" \
+  --argjson valid "$VALID_COUNT" \
+  --argjson invalid "$INVALID_COUNT" \
+  --argjson types "$TYPES_JSON" \
+  --argjson hasOrg "$HAS_ORG" \
+  --argjson hasBreadcrumb "$HAS_BREADCRUMB" \
+  --argjson hasWebsite "$HAS_WEBSITE" \
+  --argjson hasArticle "$HAS_ARTICLE" \
+  --argjson hasFaq "$HAS_FAQ" \
+  --argjson hasProduct "$HAS_PRODUCT" \
+  --argjson hasProfService "$HAS_PROFESSIONAL_SERVICE" \
+  '{
+    blockCount: $count,
+    validCount: $valid,
+    invalidCount: $invalid,
+    types: $types,
+    flags: {
+      hasOrganization: $hasOrg,
+      hasBreadcrumbList: $hasBreadcrumb,
+      hasWebSite: $hasWebsite,
+      hasArticle: $hasArticle,
+      hasFAQPage: $hasFaq,
+      hasProduct: $hasProduct,
+      hasProfessionalService: $hasProfService
+    }
+  }'

package/scripts/extract-links.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -eu
+
+# extract-links.sh — Extract and classify internal/external links from HTML
+# Usage: extract-links.sh <base-url> [file] | extract-links.sh <base-url> < html
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+BASE_URL="${1:?Usage: extract-links.sh <base-url> [file]}"
+shift || true
+
+if [ $# -ge 1 ] && [ -f "$1" ]; then
+  HTML=$(cat "$1")
+  printf '[extract-links] %s (base: %s)\n' "$1" "$BASE_URL" >&2
+else
+  HTML=$(cat)
+  printf '[extract-links] (stdin) (base: %s)\n' "$BASE_URL" >&2
+fi
+
+BASE_HOST=$(host_from_url "$BASE_URL")
+BASE_ORIGIN=$(origin_from_url "$BASE_URL")
+BASE_DIR=$(dir_from_url "$BASE_URL")
+
+HTML_FLAT=$(printf '%s' "$HTML" | tr '\n' ' ')
+
+TMPDIR="${TMPDIR:-/tmp}"
+HREFS_FILE=$(mktemp "$TMPDIR/crawlsim-hrefs.XXXXXX")
+INTERNAL_FILE=$(mktemp "$TMPDIR/crawlsim-internal.XXXXXX")
+EXTERNAL_FILE=$(mktemp "$TMPDIR/crawlsim-external.XXXXXX")
+trap 'rm -f "$HREFS_FILE" "$INTERNAL_FILE" "$EXTERNAL_FILE"' EXIT
+
+# Extract hrefs from <a> tags — handle double and single quoting separately.
+{
+  printf '%s' "$HTML_FLAT" \
+    | grep -oiE '<a[[:space:]][^>]*href="[^"]*"' \
+    | sed -E 's/.*href="([^"]*)".*/\1/' || true
+  printf '%s' "$HTML_FLAT" \
+    | grep -oiE "<a[[:space:]][^>]*href='[^']*'" \
+    | sed -E "s/.*href='([^']*)'.*/\\1/" || true
+} > "$HREFS_FILE"
+
+while IFS= read -r href; do
+  [ -z "$href" ] && continue
+  case "$href" in
+    mailto:*|tel:*|javascript:*|"#"*) continue ;;
+  esac
+
+  if printf '%s' "$href" | grep -qE '^https?://'; then
+    HREF_HOST=$(host_from_url "$href")
+    if [ "$HREF_HOST" = "$BASE_HOST" ]; then
+      echo "$href" >> "$INTERNAL_FILE"
+    else
+      echo "$href" >> "$EXTERNAL_FILE"
+    fi
+  elif printf '%s' "$href" | grep -qE '^//'; then
+    # Protocol-relative — inherit base scheme
+    scheme=$(printf '%s' "$BASE_URL" | sed -E 's#^(https?):.*#\1#')
+    abs="${scheme}:${href}"
+    HREF_HOST=$(host_from_url "$abs")
+    if [ "$HREF_HOST" = "$BASE_HOST" ]; then
+      echo "$abs" >> "$INTERNAL_FILE"
+    else
+      echo "$abs" >> "$EXTERNAL_FILE"
+    fi
+  elif printf '%s' "$href" | grep -qE '^/'; then
+    # Root-relative — attach to origin
+    echo "${BASE_ORIGIN}${href}" >> "$INTERNAL_FILE"
+  else
+    # Document-relative — attach to base directory
+    echo "${BASE_DIR}${href}" >> "$INTERNAL_FILE"
+  fi
+done < "$HREFS_FILE"
+
+INTERNAL_COUNT=0
+EXTERNAL_COUNT=0
+[ -s "$INTERNAL_FILE" ] && INTERNAL_COUNT=$(wc -l < "$INTERNAL_FILE" | tr -d ' ')
+[ -s "$EXTERNAL_FILE" ] && EXTERNAL_COUNT=$(wc -l < "$EXTERNAL_FILE" | tr -d ' ')
+
+INTERNAL_SAMPLE="[]"
+EXTERNAL_SAMPLE="[]"
+if [ -s "$INTERNAL_FILE" ]; then
+  INTERNAL_SAMPLE=$(head -50 "$INTERNAL_FILE" | jq -R . | jq -s .)
+fi
+if [ -s "$EXTERNAL_FILE" ]; then
+  EXTERNAL_SAMPLE=$(head -50 "$EXTERNAL_FILE" | jq -R . | jq -s .)
+fi
+
+jq -n \
+  --argjson internalCount "$INTERNAL_COUNT" \
+  --argjson externalCount "$EXTERNAL_COUNT" \
+  --argjson internalSample "$INTERNAL_SAMPLE" \
+  --argjson externalSample "$EXTERNAL_SAMPLE" \
+  '{
+    counts: {
+      internal: $internalCount,
+      external: $externalCount,
+      total: ($internalCount + $externalCount)
+    },
+    internal: $internalSample,
+    external: $externalSample
+  }'

package/scripts/extract-meta.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -eu
+
+# extract-meta.sh — Extract title, meta, OG, headings, images from HTML
+# Usage: extract-meta.sh [file] | extract-meta.sh < html
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+if [ $# -ge 1 ] && [ -f "$1" ]; then
+  HTML=$(cat "$1")
+  printf '[extract-meta] %s\n' "$1" >&2
+else
+  HTML=$(cat)
+  printf '[extract-meta] (stdin)\n' >&2
+fi
+
+HTML_FLAT=$(printf '%s' "$HTML" | tr '\n' ' ' | tr -s ' ')
+
+# Match a tag by regex, then pull a named attribute value that respects the
+# actual opening quote char. Works for both "…" and '…' quoting.
+# $1 = grep regex to find the tag
+# $2 = attribute name (e.g. "content", "href")
+get_attr() {
+  local tag_regex="$1"
+  local attr="$2"
+  local tag
+  tag=$(printf '%s' "$HTML_FLAT" | grep -oiE "$tag_regex" | head -1 || true)
+  [ -z "$tag" ] && return 0
+  # Try double-quoted first, then single-quoted
+  local val
+  val=$(printf '%s' "$tag" | sed -n -E "s/.*${attr}=\"([^\"]*)\".*/\\1/p" | head -1)
+  if [ -z "$val" ]; then
+    val=$(printf '%s' "$tag" | sed -n -E "s/.*${attr}='([^']*)'.*/\\1/p" | head -1)
+  fi
+  printf '%s' "$val"
+}
+
+count_pattern() {
+  local n
+  n=$(printf '%s' "$HTML_FLAT" | grep -oiE "$1" | wc -l | tr -d ' ' || true)
+  printf '%s' "${n:-0}"
+}
+
+TITLE_TAG=$(printf '%s' "$HTML_FLAT" | grep -oiE '<title[^>]*>[^<]*</title>' | head -1 || true)
+TITLE=""
+if [ -n "$TITLE_TAG" ]; then
+  TITLE=$(printf '%s' "$TITLE_TAG" | sed -E 's/<title[^>]*>(.*)<\/title>/\1/I')
+fi
+
+DESCRIPTION=$(get_attr '<meta[^>]*name=["'\''"]description["'\''"][^>]*>' 'content')
+OG_TITLE=$(get_attr '<meta[^>]*property=["'\''"]og:title["'\''"][^>]*>' 'content')
+OG_DESCRIPTION=$(get_attr '<meta[^>]*property=["'\''"]og:description["'\''"][^>]*>' 'content')
+OG_IMAGE=$(get_attr '<meta[^>]*property=["'\''"]og:image["'\''"][^>]*>' 'content')
+OG_TYPE=$(get_attr '<meta[^>]*property=["'\''"]og:type["'\''"][^>]*>' 'content')
+TWITTER_CARD=$(get_attr '<meta[^>]*name=["'\''"]twitter:card["'\''"][^>]*>' 'content')
+VIEWPORT=$(get_attr '<meta[^>]*name=["'\''"]viewport["'\''"][^>]*>' 'content')
+CANONICAL=$(get_attr '<link[^>]*rel=["'\''"]canonical["'\''"][^>]*>' 'href')
+LANG_VAL=$(get_attr '<html[^>]*>' 'lang')
+
+H1_COUNT=$(count_pattern '<h1[^>]*>')
+H2_COUNT=$(count_pattern '<h2[^>]*>')
+H3_COUNT=$(count_pattern '<h3[^>]*>')
+
+H1_TAG=$(printf '%s' "$HTML_FLAT" | grep -oiE '<h1[^>]*>[^<]*</h1>' | head -1 || true)
+H1_TEXT=""
+if [ -n "$H1_TAG" ]; then
+  H1_TEXT=$(printf '%s' "$H1_TAG" | sed -E 's/<h1[^>]*>(.*)<\/h1>/\1/I')
+fi
+
+IMG_TOTAL=$(count_pattern '<img[^>]*>')
+IMG_WITH_ALT=$(count_pattern '<img[^>]*alt=("[^"]*"|'\''[^'\'']*'\'')[^>]*>')
+
+jq -n \
+  --arg title "$TITLE" \
+  --arg description "$DESCRIPTION" \
+  --arg canonical "$CANONICAL" \
+  --arg ogTitle "$OG_TITLE" \
+  --arg ogDescription "$OG_DESCRIPTION" \
+  --arg ogImage "$OG_IMAGE" \
+  --arg ogType "$OG_TYPE" \
+  --arg twitterCard "$TWITTER_CARD" \
+  --arg h1Text "$H1_TEXT" \
+  --arg lang "$LANG_VAL" \
+  --arg viewport "$VIEWPORT" \
+  --argjson h1Count "$H1_COUNT" \
+  --argjson h2Count "$H2_COUNT" \
+  --argjson h3Count "$H3_COUNT" \
+  --argjson imgTotal "$IMG_TOTAL" \
+  --argjson imgWithAlt "$IMG_WITH_ALT" \
+  '{
+    title: (if $title == "" then null else $title end),
+    description: (if $description == "" then null else $description end),
+    canonical: (if $canonical == "" then null else $canonical end),
+    lang: (if $lang == "" then null else $lang end),
+    viewport: (if $viewport == "" then null else $viewport end),
+    og: {
+      title: (if $ogTitle == "" then null else $ogTitle end),
+      description: (if $ogDescription == "" then null else $ogDescription end),
+      image: (if $ogImage == "" then null else $ogImage end),
+      type: (if $ogType == "" then null else $ogType end)
+    },
+    twitter: {
+      card: (if $twitterCard == "" then null else $twitterCard end)
+    },
+    headings: {
+      h1: { count: $h1Count, firstText: (if $h1Text == "" then null else $h1Text end) },
+      h2: { count: $h2Count },
+      h3: { count: $h3Count }
+    },
+    images: {
+      total: $imgTotal,
+      withAlt: $imgWithAlt,
+      missingAlt: ($imgTotal - $imgWithAlt)
+    }
+  }'

package/scripts/fetch-as-bot.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# fetch-as-bot.sh — Fetch a URL as a specific bot User-Agent
+# Usage: fetch-as-bot.sh <url> <profile.json>
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+URL="${1:?Usage: fetch-as-bot.sh <url> <profile.json>}"
+PROFILE="${2:?Usage: fetch-as-bot.sh <url> <profile.json>}"
+
+BOT_ID=$(jq -r '.id' "$PROFILE")
+BOT_NAME=$(jq -r '.name' "$PROFILE")
+UA=$(jq -r '.userAgent' "$PROFILE")
+RENDERS_JS=$(jq -r '.rendersJavaScript' "$PROFILE")
+
+printf '[fetch-as-bot] %s <- %s\n' "$BOT_NAME" "$URL" >&2
+
+TMPDIR="${TMPDIR:-/tmp}"
+HEADERS_FILE=$(mktemp "$TMPDIR/crawlsim-headers.XXXXXX")
+BODY_FILE=$(mktemp "$TMPDIR/crawlsim-body.XXXXXX")
+trap 'rm -f "$HEADERS_FILE" "$BODY_FILE"' EXIT
+
+TIMING=$(curl -sS -L \
+  -H "User-Agent: $UA" \
+  -D "$HEADERS_FILE" \
+  -o "$BODY_FILE" \
+  -w '{"total":%{time_total},"ttfb":%{time_starttransfer},"connect":%{time_connect},"statusCode":%{http_code},"sizeDownload":%{size_download}}' \
+  --max-time 30 \
+  "$URL" 2>/dev/null || echo '{"total":0,"ttfb":0,"connect":0,"statusCode":0,"sizeDownload":0}')
+
+STATUS=$(echo "$TIMING" | jq -r '.statusCode')
+TOTAL_TIME=$(echo "$TIMING" | jq -r '.total')
+TTFB=$(echo "$TIMING" | jq -r '.ttfb')
+SIZE=$(echo "$TIMING" | jq -r '.sizeDownload')
+
+# Parse response headers into a JSON object using jq for safe escaping.
+# curl -L writes multiple blocks on redirect; jq keeps the last definition
+# of each header since `add` overwrites left-to-right.
+HEADERS_JSON=$(tr -d '\r' < "$HEADERS_FILE" \
+  | grep -E '^[A-Za-z][A-Za-z0-9-]*:[[:space:]]' \
+  | jq -Rs '
+      split("\n")
+      | map(select(length > 0))
+      | map(capture("^(?<k>[^:]+):[[:space:]]*(?<v>.*)$"))
+      | map({(.k): .v})
+      | add // {}
+    ')
+
+WORD_COUNT=$(count_words "$BODY_FILE")
+[ -z "$WORD_COUNT" ] && WORD_COUNT=0
+
+BODY_B64=""
+if [ -s "$BODY_FILE" ]; then
+  BODY_B64=$(base64 < "$BODY_FILE")
+fi
+
+jq -n \
+  --arg url "$URL" \
+  --arg botId "$BOT_ID" \
+  --arg botName "$BOT_NAME" \
+  --arg ua "$UA" \
+  --arg rendersJs "$RENDERS_JS" \
+  --argjson status "$STATUS" \
+  --argjson totalTime "$TOTAL_TIME" \
+  --argjson ttfb "$TTFB" \
+  --argjson size "$SIZE" \
+  --argjson wordCount "$WORD_COUNT" \
+  --argjson headers "$HEADERS_JSON" \
+  --arg bodyBase64 "$BODY_B64" \
+  '{
+    url: $url,
+    bot: {
+      id: $botId,
+      name: $botName,
+      userAgent: $ua,
+      rendersJavaScript: (if $rendersJs == "true" then true elif $rendersJs == "false" then false else $rendersJs end)
+    },
+    status: $status,
+    timing: { total: $totalTime, ttfb: $ttfb },
+    size: $size,
+    wordCount: $wordCount,
+    headers: $headers,
+    bodyBase64: $bodyBase64
+  }'