crawlscope 0.1.0

data/lib/crawlscope/cli.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Crawlscope
+  class Cli
+    def self.start(argv, out: $stdout, err: $stderr, **options)
+      new(argv, out: out, err: err, **options).call
+    end
+
+    def initialize(argv, out:, err:, configuration: Configuration.new, task: nil)
+      @argv = Array(argv).dup
+      @out = out
+      @err = err
+      @configuration = configuration
+      @configuration.output = out
+      @task = task
+    end
+
+    def call
+      command = @argv.shift.to_s
+
+      case command
+      when "help", ""
+        @out.puts(general_usage)
+        0
+      when "validate"
+        run_validate
+      when "ldjson"
+        run_ldjson
+      when "version", "--version", "-v"
+        @out.puts(Crawlscope::VERSION)
+        0
+      else
+        @err.puts("Unknown command: #{command}")
+        @err.puts("")
+        @err.puts(general_usage)
+        1
+      end
+    rescue OptionParser::InvalidOption, OptionParser::MissingArgument, ConfigurationError, ArgumentError => error
+      @err.puts(error.message)
+      @err.puts("")
+      @err.puts(general_usage)
+      1
+    end
+
+    private
+
+    def general_usage
+      <<~TEXT
+        Usage:
+          crawlscope validate --base-url https://example.com [options]
+          crawlscope ldjson --url https://example.com/page [options]
+          crawlscope version
+
+        Commands:
+          validate    Audit sitemap URLs for metadata, structured data, uniqueness, and links
+          ldjson      Validate structured data on one or more URLs
+          version     Print the gem version
+      TEXT
+    end
+
+    def run_ldjson
+      options = {
+        debug: env_enabled?("DEBUG"),
+        renderer: resolved_renderer,
+        report_path: normalized_string(ENV["REPORT_PATH"]),
+        summary: env_enabled?("SUMMARY"),
+        timeout_seconds: resolved_integer("TIMEOUT", default: @configuration.timeout_seconds, minimum: 1),
+        urls: resolved_urls_from_env
+      }
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: crawlscope ldjson --url https://example.com/page [options]"
+
+        opts.on("--url URL", "Validate one URL (repeatable)") do |value|
+          options[:urls] << value.strip
+        end
+
+        opts.on("--debug", "Print detected structured data") do
+          options[:debug] = true
+        end
+
+        opts.on("--summary", "Print grouped summary output") do
+          options[:summary] = true
+        end
+
+        opts.on("--report-path PATH", "Write a JSON report to PATH") do |value|
+          options[:report_path] = value
+        end
+
+        opts.on("--renderer NAME", "Use http or browser rendering") do |value|
+          options[:renderer] = value.to_sym
+        end
+
+        opts.on("--timeout SECONDS", Integer, "Set request timeout") do |value|
+          options[:timeout_seconds] = integer_option(value, minimum: 1, name: "timeout")
+        end
+
+        opts.on("--network-idle-timeout SECONDS", Integer, "Set browser network idle timeout") do |value|
+          @configuration.network_idle_timeout_seconds = integer_option(value, minimum: 1, name: "network-idle-timeout")
+        end
+      end
+
+      parser.parse!(@argv)
+
+      urls = options[:urls].map(&:strip).reject(&:empty?)
+      raise ConfigurationError, "Crawlscope URL is not configured" if urls.empty?
+
+      configure_renderer(options[:renderer])
+
+      result = task.validate_ldjson(
+        urls: urls,
+        debug: options[:debug],
+        renderer: options[:renderer],
+        report_path: options[:report_path],
+        summary: options[:summary],
+        timeout_seconds: options[:timeout_seconds]
+      )
+
+      result.ok? ? 0 : 1
+    end
+
+    def run_validate
+      options = {
+        base_url: normalized_string(ENV["BASE_URL"]),
+        rule_names: normalized_string(ENV["RULES"]),
+        sitemap_path: normalized_string(ENV["SITEMAP"])
+      }
+
+      configure_renderer(resolved_renderer)
+      @configuration.concurrency = resolved_concurrency
+      @configuration.network_idle_timeout_seconds = resolved_integer("NETWORK_IDLE_TIMEOUT", default: @configuration.network_idle_timeout_seconds, minimum: 1)
+      @configuration.timeout_seconds = resolved_integer("TIMEOUT", default: @configuration.timeout_seconds, minimum: 1)
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: crawlscope validate --base-url https://example.com [options]"
+
+        opts.on("--base-url URL", "Set the site base URL") do |value|
+          options[:base_url] = value
+        end
+
+        opts.on("--sitemap PATH_OR_URL", "Set the sitemap path or URL") do |value|
+          options[:sitemap_path] = value
+        end
+
+        opts.on("--rules CSV", "Run a subset of rules, for example metadata,links") do |value|
+          options[:rule_names] = value
+        end
+
+        opts.on("--renderer NAME", "Use http or browser rendering") do |value|
+          configure_renderer(value.to_sym)
+        end
+
+        opts.on("--timeout SECONDS", Integer, "Set request timeout") do |value|
+          @configuration.timeout_seconds = integer_option(value, minimum: 1, name: "timeout")
+        end
+
+        opts.on("--network-idle-timeout SECONDS", Integer, "Set browser network idle timeout") do |value|
+          @configuration.network_idle_timeout_seconds = integer_option(value, minimum: 1, name: "network-idle-timeout")
+        end
+
+        opts.on("--concurrency COUNT", Integer, "Set crawl concurrency") do |value|
+          @configuration.concurrency = integer_option(value, minimum: 1, name: "concurrency")
+        end
+      end
+
+      parser.parse!(@argv)
+
+      result = task.validate(
+        base_url: options[:base_url],
+        sitemap_path: options[:sitemap_path],
+        rule_names: options[:rule_names]
+      )
+
+      result.ok? ? 0 : 1
+    end
+
+    def configure_renderer(renderer)
+      @configuration.renderer = renderer
+    end
+
+    def env_enabled?(name)
+      ENV[name].to_s == "1"
+    end
+
+    def integer_option(value, minimum:, name:)
+      integer = value.is_a?(Integer) ? value : Integer(value, 10)
+      raise ArgumentError, "#{name} must be >= #{minimum}" if integer < minimum
+
+      integer
+    rescue ArgumentError => error
+      raise error if error.message == "#{name} must be >= #{minimum}"
+
+      raise ArgumentError, "#{name} must be an integer >= #{minimum}"
+    end
+
+    def normalized_string(value)
+      normalized = value.to_s.strip
+      normalized.empty? ? nil : normalized
+    end
+
+    def resolved_concurrency
+      configured_concurrency = resolved_integer("CONCURRENCY", default: @configuration.concurrency, minimum: 1)
+
+      if @configuration.renderer == :browser && normalized_string(ENV["CONCURRENCY"]).nil?
+        browser_concurrency = @configuration.browser_concurrency
+
+        if configured_concurrency > browser_concurrency
+          @configuration.output.puts("Default JS concurrency capped at #{browser_concurrency}. Set CONCURRENCY to override.")
+          browser_concurrency
+        else
+          configured_concurrency
+        end
+      else
+        configured_concurrency
+      end
+    end
+
+    def resolved_integer(name, default:, minimum:)
+      raw_value = normalized_string(ENV[name])
+      return default if raw_value.nil?
+
+      integer_option(raw_value, minimum: minimum, name: name.downcase.tr("_", "-"))
+    end
+
+    def resolved_renderer
+      renderer = normalized_string(ENV["RENDERER"])
+      return renderer.to_sym if renderer
+
+      env_enabled?("JS") ? :browser : :http
+    end
+
+    def resolved_urls_from_env
+      raw_urls = normalized_string(ENV["URL"])
+      return [] if raw_urls.nil?
+
+      raw_urls.split(";").map(&:strip).reject(&:empty?)
+    end
+
+    def task
+      @task ||= Task.new(configuration: @configuration, reporter: Reporter.new(io: @out))
+    end
+  end
+end

data/lib/crawlscope/configuration.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class Configuration
+    DEFAULT_ALLOWED_STATUSES = [200, 301, 302].freeze
+    DEFAULT_BROWSER_CONCURRENCY = 4
+    DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS = 5
+    DEFAULT_BROWSER_SCROLL_PAGE = true
+    DEFAULT_CONCURRENCY = 10
+    DEFAULT_TIMEOUT_SECONDS = 20
+
+    attr_writer :allowed_statuses, :base_url, :browser_factory, :concurrency, :network_idle_timeout_seconds, :output, :renderer, :rule_registry, :schema_registry, :scroll_page, :site_name, :sitemap_path, :timeout_seconds
+
+    def allowed_statuses
+      value = resolve(@allowed_statuses)
+      Array(value.nil? ? DEFAULT_ALLOWED_STATUSES : value).map(&:to_i)
+    end
+
+    def base_url
+      resolve(@base_url)
+    end
+
+    def browser_factory
+      resolve(@browser_factory)
+    end
+
+    def concurrency
+      value = resolve(@concurrency)
+      value.nil? ? DEFAULT_CONCURRENCY : value.to_i
+    end
+
+    def browser_concurrency
+      value = concurrency
+      default_value = DEFAULT_BROWSER_CONCURRENCY
+
+      if value > default_value
+        default_value
+      else
+        value
+      end
+    end
+
+    def network_idle_timeout_seconds
+      value = resolve(@network_idle_timeout_seconds)
+      value.nil? ? DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS : value.to_i
+    end
+
+    def output
+      value = resolve(@output)
+      value.nil? ? $stdout : value
+    end
+
+    def renderer
+      value = resolve(@renderer)
+      normalized_value = value.to_s.strip
+      normalized_value = "http" if normalized_value.empty?
+
+      normalized_value.to_sym
+    end
+
+    def rule_registry
+      value = resolve(@rule_registry)
+      return value unless value.nil?
+
+      RuleRegistry.default(site_name: site_name)
+    end
+
+    def audit(base_url: self.base_url, sitemap_path: self.sitemap_path, rule_names: nil)
+      if base_url.to_s.strip.empty?
+        raise ConfigurationError, "Crawlscope base_url is not configured"
+      end
+
+      if sitemap_path.to_s.strip.empty?
+        raise ConfigurationError, "Crawlscope sitemap_path is not configured"
+      end
+
+      Audit.new(
+        base_url: base_url,
+        sitemap_path: sitemap_path,
+        browser_factory: browser_factory,
+        concurrency: concurrency,
+        network_idle_timeout_seconds: network_idle_timeout_seconds,
+        renderer: renderer,
+        timeout_seconds: timeout_seconds,
+        allowed_statuses: allowed_statuses,
+        rules: rule_registry.rules_for(rule_names),
+        schema_registry: schema_registry,
+        scroll_page: scroll_page?
+      )
+    end
+
+    def schema_registry
+      value = resolve(@schema_registry)
+      return value unless value.nil?
+
+      SchemaRegistry.default
+    end
+
+    def site_name
+      resolve(@site_name)
+    end
+
+    def scroll_page?
+      value = resolve(@scroll_page)
+      value.nil? ? DEFAULT_BROWSER_SCROLL_PAGE : value
+    end
+
+    def sitemap_path
+      resolve(@sitemap_path)
+    end
+
+    def timeout_seconds
+      value = resolve(@timeout_seconds)
+      value.nil? ? DEFAULT_TIMEOUT_SECONDS : value.to_i
+    end
+
+    private
+
+    def resolve(value)
+      value.respond_to?(:call) ? value.call : value
+    end
+  end
+end

data/lib/crawlscope/crawler.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Crawlscope
+  class Crawler
+    def initialize(page_fetcher:, concurrency:)
+      @page_fetcher = page_fetcher
+      @concurrency = concurrency
+    end
+
+    def call(urls)
+      pages = Concurrent::Array.new
+      pool = Concurrent::FixedThreadPool.new(@concurrency)
+
+      urls.each do |url|
+        pool.post do
+          pages << @page_fetcher.fetch(url)
+        end
+      end
+
+      pool.shutdown
+      pool.wait_for_termination
+
+      pages.to_a
+    end
+  end
+end

data/lib/crawlscope/http.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "concurrent"
+require "faraday"
+require "faraday/follow_redirects"
+require "nokogiri"
+
+module Crawlscope
+  class Http
+    MAX_REDIRECTS = 5
+    USER_AGENT = "Mozilla/5.0 (compatible; Crawlscope/1.0)"
+
+    def initialize(base_url:, timeout_seconds:)
+      @base_url = base_url
+      @timeout_seconds = timeout_seconds
+      @connections_by_thread = Concurrent::Map.new
+    end
+
+    def close
+      @connections_by_thread.clear
+    end
+
+    def fetch(url)
+      response = connection.get(url) do |request|
+        request.headers["User-Agent"] = USER_AGENT
+      end
+
+      final_url = response.env.url.to_s
+      final_url = url if final_url.empty?
+      headers = response.headers.to_h
+      body = response.body.to_s
+      doc = if response.status == 200 && html_response?(headers)
+        Nokogiri::HTML(body)
+      end
+
+      Page.new(
+        url: url,
+        normalized_url: Url.normalize(url, base_url: @base_url),
+        final_url: final_url,
+        normalized_final_url: Url.normalize(final_url, base_url: @base_url),
+        status: response.status,
+        headers: headers,
+        body: body,
+        doc: doc
+      )
+    rescue => error
+      Page.new(
+        url: url,
+        normalized_url: Url.normalize(url, base_url: @base_url),
+        final_url: url,
+        normalized_final_url: Url.normalize(url, base_url: @base_url),
+        status: nil,
+        headers: {},
+        body: nil,
+        doc: nil,
+        error: "#{error.class}: #{error.message}"
+      )
+    end
+
+    private
+
+    def connection
+      @connections_by_thread.compute_if_absent(Thread.current.object_id) do
+        Faraday.new do |faraday|
+          faraday.response :follow_redirects, limit: MAX_REDIRECTS
+          faraday.options.timeout = @timeout_seconds
+          faraday.options.open_timeout = @timeout_seconds
+        end
+      end
+    end
+
+    def html_response?(headers)
+      content_type = headers["content-type"] || headers.find { |key, _value| key.to_s.casecmp("content-type").zero? }&.last.to_s
+      content_type.empty? || content_type.include?("text/html")
+    end
+  end
+end

data/lib/crawlscope/issue.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  Issue = Data.define(:code, :severity, :category, :url, :message, :details) do
+    def error?
+      severity == :error
+    end
+
+    def warning?
+      severity == :warning
+    end
+
+    def notice?
+      severity == :notice
+    end
+  end
+end

data/lib/crawlscope/issue_collection.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class IssueCollection
+    include Enumerable
+
+    def initialize(issues = [])
+      @issues = issues.dup
+    end
+
+    def add(issue = nil, **attributes)
+      issue ||= Issue.new(**attributes)
+      @issues << issue
+      issue
+    end
+
+    def any?
+      @issues.any?
+    end
+
+    def each(&block)
+      @issues.each(&block)
+    end
+
+    def size
+      @issues.size
+    end
+
+    def to_a
+      @issues.dup
+    end
+
+    def by_category
+      @issues.group_by(&:category)
+    end
+
+    def by_severity
+      @issues.group_by(&:severity)
+    end
+  end
+end

data/lib/crawlscope/page.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class Page
+    attr_reader :body, :doc, :error, :final_url, :headers, :normalized_final_url, :normalized_url, :status, :url
+
+    def initialize(url:, normalized_url:, final_url:, normalized_final_url:, status:, headers:, body:, doc:, error: nil)
+      @url = url
+      @normalized_url = normalized_url
+      @final_url = final_url
+      @normalized_final_url = normalized_final_url
+      @status = status
+      @headers = headers || {}
+      @body = body
+      @doc = doc
+      @error = error
+    end
+
+    def html?
+      !doc.nil?
+    end
+  end
+end

data/lib/crawlscope/railtie.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path("../tasks/crawlscope_tasks.rake", __dir__)
+    end
+  end
+end

data/lib/crawlscope/reporter.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class Reporter
+    def initialize(io:)
+      @io = io
+    end
+
+    def report(result)
+      @io.puts("Crawlscope validation")
+      @io.puts("Base URL: #{result.base_url}")
+      @io.puts("Sitemap: #{result.sitemap_path}")
+      @io.puts("URLs: #{result.urls.size}")
+      @io.puts("Pages: #{result.pages.size}")
+
+      if result.ok?
+        @io.puts("Status: OK")
+        return
+      end
+
+      @io.puts("Status: FAILED")
+      @io.puts("Issues: #{result.issues.size}")
+
+      result.issues.by_severity.sort_by { |severity, _issues| severity.to_s }.each do |severity, issues|
+        @io.puts("#{severity}: #{issues.size}")
+      end
+
+      result.issues.each do |issue|
+        @io.puts("- [#{issue.severity}] #{issue.url} #{issue.message}")
+      end
+    end
+  end
+end

data/lib/crawlscope/result.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  Result = Data.define(:base_url, :sitemap_path, :urls, :pages, :issues) do
+    def ok?
+      issues.none?(&:error?) && issues.none?(&:warning?) && issues.none?(&:notice?)
+    end
+  end
+end

data/lib/crawlscope/rule_registry.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class RuleRegistry
+    attr_reader :default_codes, :rules
+
+    def initialize(rules:, default_codes: nil)
+      @rules = Array(rules)
+      @default_codes = Array(default_codes).map(&:to_sym)
+    end
+
+    def self.default(site_name: nil)
+      new(
+        rules: [
+          Rules::Metadata.new(site_name: site_name),
+          Rules::StructuredData.new,
+          Rules::Uniqueness.new,
+          Rules::Links.new
+        ],
+        default_codes: %i[metadata structured_data uniqueness links]
+      )
+    end
+
+    def codes
+      @rules.map(&:code)
+    end
+
+    def rules_for(names)
+      normalized_names = Array(names).flat_map { |value| value.to_s.split(",") }.map(&:strip).reject(&:empty?)
+      normalized_names = @default_codes.map(&:to_s) if normalized_names.empty?
+
+      selected_rules = @rules.select { |rule| normalized_names.include?(rule.code.to_s) }
+      missing_rules = normalized_names - selected_rules.map { |rule| rule.code.to_s }
+      return selected_rules if missing_rules.empty?
+
+      raise ConfigurationError, "Unknown Crawlscope rules: #{missing_rules.join(", ")}"
+    end
+  end
+end