crawlscope 0.1.0

data/lib/crawlscope/schema_registry.rb

@@ -0,0 +1,431 @@
+# frozen_string_literal: true
+
+require "json-schema"
+
+module Crawlscope
+  class SchemaRegistry
+    FAQ_PAGE = {
+      "type" => "object",
+      "required" => ["@context", "@type", "mainEntity"],
+      "properties" => {
+        "@context" => {"const" => "https://schema.org"},
+        "@type" => {"const" => "FAQPage"},
+        "mainEntity" => {
+          "type" => "array",
+          "minItems" => 1,
+          "items" => {"$ref" => "#/definitions/Question"}
+        }
+      },
+      "definitions" => {
+        "Question" => {
+          "type" => "object",
+          "required" => ["@type", "name", "acceptedAnswer"],
+          "properties" => {
+            "@type" => {"const" => "Question"},
+            "name" => {"type" => "string"},
+            "acceptedAnswer" => {"$ref" => "#/definitions/Answer"}
+          }
+        },
+        "Answer" => {
+          "type" => "object",
+          "required" => ["@type", "text"],
+          "properties" => {
+            "@type" => {"const" => "Answer"},
+            "text" => {"type" => "string"}
+          }
+        }
+      }
+    }.freeze
+
+    ARTICLE = {
+      type: "object",
+      required: ["@type", "headline"],
+      properties: {
+        "@type" => {enum: ["Article", "NewsArticle", "BlogPosting"]},
+        :headline => {type: "string", maxLength: 110},
+        :image => {type: "string", format: "uri"},
+        :datePublished => {type: "string", format: "date-time"},
+        :dateModified => {type: "string", format: "date-time"},
+        :author => {type: "object"},
+        :publisher => {type: "object"}
+      }
+    }.freeze
+
+    ORGANIZATION = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "Organization"},
+        :name => {type: "string"},
+        :url => {type: "string", format: "uri"},
+        :logo => {
+          anyOf: [
+            {type: "string", format: "uri"},
+            {
+              type: "object",
+              required: ["@type", "url"],
+              properties: {
+                "@type" => {const: "ImageObject"},
+                :url => {type: "string", format: "uri"}
+              }
+            }
+          ]
+        },
+        :description => {type: "string"}
+      }
+    }.freeze
+
+    IMAGE_OBJECT = {
+      type: "object",
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "ImageObject"},
+        :url => {type: "string", format: "uri"},
+        :contentUrl => {type: "string", format: "uri"},
+        :thumbnail => {type: ["string", "object"]}
+      }
+    }.freeze
+
+    OFFER = {
+      type: "object",
+      additionalProperties: true,
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "Offer"},
+        :name => {type: ["string", "null"]},
+        :price => {type: ["string", "number"]},
+        :priceCurrency => {type: ["string", "null"]},
+        :priceSpecification => {type: ["object", "null"]},
+        :availability => {type: "string"},
+        :shippingDetails => {type: "object"},
+        :hasMerchantReturnPolicy => {type: "boolean"},
+        :merchantReturnPolicy => {type: "object"},
+        :url => {type: "string", format: "uri"},
+        :eligibleQuantity => {type: "object"},
+        :additionalProperty => {type: "array", items: {type: "object"}}
+      }
+    }.freeze
+
+    RATING = {
+      type: "object",
+      required: ["@type", "ratingValue"],
+      properties: {
+        "@type" => {const: "Rating"},
+        :ratingValue => {type: ["string", "number"]},
+        :bestRating => {type: ["string", "number"]},
+        :worstRating => {type: ["string", "number"]}
+      }
+    }.freeze
+
+    REVIEW = {
+      type: "object",
+      required: ["@type", "itemReviewed"],
+      properties: {
+        "@type" => {const: "Review"},
+        :itemReviewed => {type: "object"},
+        :reviewRating => RATING,
+        :author => {type: ["object", "string"]},
+        :datePublished => {type: "string", format: "date-time"},
+        :reviewBody => {type: "string"}
+      }
+    }.freeze
+
+    REVIEW_SNIPPET = {
+      type: "object",
+      required: ["@type", "reviewRating"],
+      properties: {
+        "@type" => {const: "Review"},
+        :reviewRating => RATING,
+        :author => {type: ["object", "string"]},
+        :reviewBody => {type: "string"},
+        :datePublished => {type: "string", format: "date-time"}
+      }
+    }.freeze
+
+    AGGREGATE_RATING = {
+      type: "object",
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "AggregateRating"},
+        :ratingValue => {type: ["string", "number"]},
+        :ratingCount => {type: "integer"},
+        :reviewCount => {type: "integer"},
+        :bestRating => {type: ["string", "number"]},
+        :worstRating => {type: ["string", "number"]}
+      }
+    }.freeze
+
+    SOFTWARE_APPLICATION = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "SoftwareApplication"},
+        :name => {type: "string"},
+        :applicationCategory => {type: "string"},
+        :description => {type: "string"},
+        :offers => {
+          anyOf: [
+            OFFER,
+            {type: "array", items: OFFER}
+          ]
+        },
+        :featureList => {type: ["string", "array"]},
+        :aggregateRating => AGGREGATE_RATING,
+        :review => REVIEW_SNIPPET
+      }
+    }.freeze
+
+    WEB_APPLICATION = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "WebApplication"},
+        :name => {type: "string"},
+        :applicationCategory => {type: "string"},
+        :description => {type: "string"},
+        :operatingSystem => {type: "string"},
+        :url => {type: "string", format: "uri"},
+        :offers => {
+          anyOf: [
+            OFFER,
+            {type: "array", items: OFFER}
+          ]
+        },
+        :featureList => {type: ["string", "array"]},
+        :aggregateRating => AGGREGATE_RATING,
+        :review => REVIEW_SNIPPET
+      }
+    }.freeze
+
+    HOW_TO = {
+      type: "object",
+      required: ["@type", "name", "step"],
+      properties: {
+        "@type" => {const: "HowTo"},
+        :name => {type: "string"},
+        :description => {type: "string"},
+        :step => {
+          type: "array",
+          minItems: 1,
+          items: {
+            type: "object",
+            required: ["@type", "name", "text"],
+            properties: {
+              "@type" => {const: "HowToStep"},
+              :name => {type: "string"},
+              :text => {type: "string"},
+              :position => {type: "integer", minimum: 1}
+            }
+          }
+        }
+      }
+    }.freeze
+
+    CONTACT_PAGE = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "ContactPage"},
+        :name => {type: "string"},
+        :description => {type: "string"},
+        :url => {type: "string", format: "uri"}
+      }
+    }.freeze
+
+    PRODUCT = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "Product"},
+        :name => {type: "string"},
+        :image => {
+          anyOf: [
+            {type: "string", format: "uri"},
+            IMAGE_OBJECT,
+            {type: "array", items: {anyOf: [{type: "string", format: "uri"}, IMAGE_OBJECT]}}
+          ]
+        },
+        :description => {type: "string"},
+        :offers => {
+          anyOf: [
+            OFFER,
+            {type: "array", items: OFFER}
+          ]
+        }
+      }
+    }.freeze
+
+    RECIPE = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "Recipe"},
+        :name => {type: "string"},
+        :image => {type: ["string", "array"]},
+        :recipeIngredient => {type: "array", items: {type: "string"}},
+        :recipeInstructions => {type: ["string", "array"]}
+      }
+    }.freeze
+
+    EVENT = {
+      type: "object",
+      required: ["@type", "name", "startDate"],
+      properties: {
+        "@type" => {const: "Event"},
+        :name => {type: "string"},
+        :startDate => {type: "string", format: "date-time"},
+        :endDate => {type: "string", format: "date-time"},
+        :location => {type: "object"}
+      }
+    }.freeze
+
+    VIDEO_OBJECT = {
+      type: "object",
+      required: ["@type", "name", "description"],
+      properties: {
+        "@type" => {const: "VideoObject"},
+        :name => {type: "string"},
+        :description => {type: "string"},
+        :thumbnailUrl => {type: "string", format: "uri"},
+        :uploadDate => {type: "string", format: "date-time"}
+      }
+    }.freeze
+
+    WEBSITE = {
+      type: "object",
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "WebSite"},
+        :name => {type: "string"},
+        :url => {type: "string", format: "uri"},
+        :potentialAction => {type: "object"}
+      }
+    }.freeze
+
+    BREADCRUMB_LIST = {
+      type: "object",
+      required: ["@type", "itemListElement"],
+      properties: {
+        "@type" => {const: "BreadcrumbList"},
+        :itemListElement => {
+          type: "array",
+          minItems: 1,
+          items: {
+            type: "object",
+            required: ["@type", "position", "name", "item"],
+            properties: {
+              "@type" => {const: "ListItem"},
+              :position => {type: "integer", minimum: 1},
+              :name => {type: "string"},
+              :item => {type: "string", format: "uri"}
+            }
+          }
+        }
+      }
+    }.freeze
+
+    WEB_PAGE = {
+      type: "object",
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "WebPage"}
+      }
+    }.freeze
+
+    def initialize(schemas: {})
+      @schemas = schemas.transform_keys(&:to_s).dup
+    end
+
+    def self.default
+      new(
+        schemas: {
+          "FAQPage" => FAQ_PAGE,
+          "Article" => ARTICLE,
+          "NewsArticle" => ARTICLE,
+          "BlogPosting" => ARTICLE,
+          "Organization" => ORGANIZATION,
+          "SoftwareApplication" => SOFTWARE_APPLICATION,
+          "WebApplication" => WEB_APPLICATION,
+          "HowTo" => HOW_TO,
+          "ContactPage" => CONTACT_PAGE,
+          "Product" => PRODUCT,
+          "Review" => REVIEW,
+          "WebSite" => WEBSITE,
+          "BreadcrumbList" => BREADCRUMB_LIST,
+          "Recipe" => RECIPE,
+          "Event" => EVENT,
+          "VideoObject" => VIDEO_OBJECT,
+          "WebPage" => WEB_PAGE
+        }
+      )
+    end
+
+    def dup
+      self.class.new(schemas: deep_copy(@schemas))
+    end
+
+    def fetch(type)
+      @schemas.fetch(type.to_s)
+    end
+
+    def register(type, schema)
+      @schemas[type.to_s] = schema
+      self
+    end
+
+    def registered?(type)
+      @schemas.key?(type.to_s)
+    end
+
+    def validate(item)
+      if item.is_a?(Array)
+        return item.flat_map { |entry| validate(entry) }
+      end
+
+      errors = []
+
+      if item.is_a?(Hash) && item["@graph"].is_a?(Array)
+        item["@graph"].each do |graph_item|
+          errors.concat(validate(graph_item))
+        end
+      end
+
+      type = item.is_a?(Hash) ? item["@type"] : nil
+      return errors if type.nil?
+
+      schema = @schemas[type.to_s]
+      return errors if schema.nil?
+
+      JSON::Validator.fully_validate(schema, item, errors_as_objects: true).each do |error|
+        errors << {
+          field: error[:fragment].to_s.sub("#/", ""),
+          issue: error[:message],
+          type: type
+        }
+      end
+
+      errors
+    rescue JSON::Schema::ValidationError => error
+      [{field: "unknown", issue: error.message, type: type}]
+    end
+
+    def to_h
+      @schemas.dup
+    end
+
+    private
+
+    def deep_copy(value)
+      case value
+      when Hash
+        value.each_with_object({}) do |(key, entry), copy|
+          copy[key] = deep_copy(entry)
+        end
+      when Array
+        value.map { |entry| deep_copy(entry) }
+      else
+        value
+      end
+    end
+  end
+end

data/lib/crawlscope/sitemap.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "faraday/follow_redirects"
+require "nokogiri"
+require "uri"
+
+module Crawlscope
+  class Sitemap
+    SITEMAP_NAMESPACE = {"xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9"}.freeze
+
+    def initialize(path:)
+      @path = path
+    end
+
+    def urls(base_url:)
+      collect_urls(@path, base_url: base_url, visited: Set.new).uniq
+    end
+
+    private
+
+    def collect_urls(source, base_url:, visited:)
+      return [] if visited.include?(source)
+
+      visited.add(source)
+      document = Nokogiri::XML(read(source))
+      root_name = document.root&.name
+
+      if root_name == "sitemapindex"
+        document.xpath("//xmlns:sitemap/xmlns:loc", SITEMAP_NAMESPACE).flat_map do |node|
+          child_source = resolve_child_source(source, node.text.to_s.strip)
+          collect_urls(child_source, base_url: base_url, visited: visited)
+        end
+      else
+        document.xpath("//xmlns:url/xmlns:loc", SITEMAP_NAMESPACE).map do |node|
+          Url.normalize(node.text.to_s.strip, base_url: base_url)
+        end
+      end
+    end
+
+    def read(source)
+      if Url.remote?(source)
+        connection.get(source).body
+      else
+        File.read(source)
+      end
+    end
+
+    def resolve_child_source(parent_source, child_loc)
+      return child_loc if Url.remote?(child_loc)
+
+      if Url.remote?(parent_source)
+        URI.join(parent_source, child_loc).to_s
+      else
+        File.expand_path(child_loc, File.dirname(parent_source))
+      end
+    end
+
+    def connection
+      @connection ||= Faraday.new do |faraday|
+        faraday.response :follow_redirects, limit: Http::MAX_REDIRECTS
+        faraday.options.timeout = 20
+        faraday.options.open_timeout = 20
+      end
+    end
+  end
+end

data/lib/crawlscope/structured_data/audit.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  module StructuredData
+    class Audit
+      Entry = Data.define(:url, :status, :structured_items, :errors, :fetch_error, :content_type, :skipped_reason) do
+        def json_ld_count
+          structured_items.count { |item| item[:source] == "json-ld" }
+        end
+
+        def microdata_count
+          structured_items.count { |item| item[:source] == "microdata" }
+        end
+
+        def ok?
+          fetch_error.nil? && errors.empty?
+        end
+
+        def structured_data_found?
+          skipped_reason == "non-html" || structured_items.any?
+        end
+      end
+
+      Result = Data.define(:entries) do
+        def ok?
+          entries.all?(&:ok?)
+        end
+      end
+
+      def initialize(schema_registry:, renderer:, timeout_seconds:, browser_factory: nil, network_idle_timeout_seconds: Configuration::DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS, scroll_page: Configuration::DEFAULT_BROWSER_SCROLL_PAGE)
+        @schema_registry = schema_registry
+        @renderer = renderer.to_sym
+        @timeout_seconds = timeout_seconds
+        @browser_factory = browser_factory
+        @network_idle_timeout_seconds = network_idle_timeout_seconds
+        @scroll_page = scroll_page
+      end
+
+      def call(urls:)
+        fetcher = build_fetcher(urls)
+        entries = urls.map { |url| validate_url(url, fetcher) }
+
+        Result.new(entries: entries)
+      ensure
+        fetcher&.close
+      end
+
+      private
+
+      def build_browser(base_url)
+        browser_factory = @browser_factory
+
+        if browser_factory
+          browser_factory.call
+        else
+          Crawlscope::Browser.new(
+            base_url: base_url,
+            timeout_seconds: @timeout_seconds,
+            network_idle_timeout_seconds: @network_idle_timeout_seconds,
+            scroll_page: @scroll_page
+          )
+        end
+      rescue LoadError => error
+        raise ConfigurationError, "Browser rendering requires the ferrum gem (#{error.message})"
+      end
+
+      def build_fetcher(urls)
+        first_url = urls.first.to_s
+        base_url = first_url.empty? ? "http://localhost:3000" : first_url
+
+        if @renderer == :browser
+          build_browser(base_url)
+        else
+          Crawlscope::Http.new(base_url: base_url, timeout_seconds: @timeout_seconds)
+        end
+      end
+
+      def build_validation_errors(page)
+        document = Document.new(html: page.body)
+        structured_items = document.items.map do |item|
+          {
+            data: item.data,
+            source: item.source
+          }
+        end
+
+        errors = structured_items.filter_map do |item|
+          if item[:data].is_a?(Hash) && item[:data][:error]
+            {
+              errors: [{field: "parse", issue: item[:data][:message]}],
+              source: item[:source],
+              type: item[:source]
+            }
+          else
+            schema_errors = @schema_registry.validate(item[:data])
+            next if schema_errors.empty?
+
+            {
+              errors: schema_errors,
+              source: item[:source],
+              type: item[:data]["@type"] || item[:source]
+            }
+          end
+        end
+
+        [structured_items, errors]
+      end
+
+      def validate_url(url, fetcher)
+        page = fetcher.fetch(url)
+        content_type = page.headers["content-type"].to_s
+
+        if page.error
+          Entry.new(url: url, status: page.status, structured_items: [], errors: [], fetch_error: page.error, content_type: content_type, skipped_reason: nil)
+        elsif page.status && !(200..299).cover?(page.status.to_i)
+          Entry.new(
+            url: url,
+            status: page.status,
+            structured_items: [],
+            errors: [],
+            fetch_error: "Non-success status",
+            content_type: content_type,
+            skipped_reason: nil
+          )
+        elsif !content_type.empty? && !content_type.include?("text/html")
+          Entry.new(
+            url: url,
+            status: page.status,
+            structured_items: [],
+            errors: [],
+            fetch_error: nil,
+            content_type: content_type,
+            skipped_reason: "non-html"
+          )
+        else
+          structured_items, errors = build_validation_errors(page)
+          Entry.new(
+            url: url,
+            status: page.status,
+            structured_items: structured_items,
+            errors: errors,
+            fetch_error: nil,
+            content_type: content_type,
+            skipped_reason: nil
+          )
+        end
+      end
+    end
+  end
+end