crawlscope 0.1.0 → 0.2.0

data/lib/crawlscope/schemas.rb

@@ -0,0 +1,355 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class Schemas
+    FAQ_PAGE = {
+      "type" => "object",
+      "required" => ["@context", "@type", "mainEntity"],
+      "properties" => {
+        "@context" => {"const" => "https://schema.org"},
+        "@type" => {"const" => "FAQPage"},
+        "mainEntity" => {
+          "type" => "array",
+          "minItems" => 1,
+          "items" => {"$ref" => "#/definitions/Question"}
+        }
+      },
+      "definitions" => {
+        "Question" => {
+          "type" => "object",
+          "required" => ["@type", "name", "acceptedAnswer"],
+          "properties" => {
+            "@type" => {"const" => "Question"},
+            "name" => {"type" => "string"},
+            "acceptedAnswer" => {"$ref" => "#/definitions/Answer"}
+          }
+        },
+        "Answer" => {
+          "type" => "object",
+          "required" => ["@type", "text"],
+          "properties" => {
+            "@type" => {"const" => "Answer"},
+            "text" => {"type" => "string"}
+          }
+        }
+      }
+    }.freeze
+
+    ARTICLE = {
+      type: "object",
+      required: ["@type", "headline"],
+      properties: {
+        "@type" => {enum: ["Article", "NewsArticle", "BlogPosting"]},
+        :headline => {type: "string", maxLength: 110},
+        :image => {type: "string", format: "uri"},
+        :datePublished => {type: "string", format: "date-time"},
+        :dateModified => {type: "string", format: "date-time"},
+        :author => {type: "object"},
+        :publisher => {type: "object"}
+      }
+    }.freeze
+
+    ORGANIZATION = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "Organization"},
+        :name => {type: "string"},
+        :url => {type: "string", format: "uri"},
+        :logo => {
+          anyOf: [
+            {type: "string", format: "uri"},
+            {
+              type: "object",
+              required: ["@type", "url"],
+              properties: {
+                "@type" => {const: "ImageObject"},
+                :url => {type: "string", format: "uri"}
+              }
+            }
+          ]
+        },
+        :description => {type: "string"}
+      }
+    }.freeze
+
+    IMAGE_OBJECT = {
+      type: "object",
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "ImageObject"},
+        :url => {type: "string", format: "uri"},
+        :contentUrl => {type: "string", format: "uri"},
+        :thumbnail => {type: ["string", "object"]}
+      }
+    }.freeze
+
+    OFFER = {
+      type: "object",
+      additionalProperties: true,
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "Offer"},
+        :name => {type: ["string", "null"]},
+        :price => {type: ["string", "number"]},
+        :priceCurrency => {type: ["string", "null"]},
+        :priceSpecification => {type: ["object", "null"]},
+        :availability => {type: "string"},
+        :shippingDetails => {type: "object"},
+        :hasMerchantReturnPolicy => {type: "boolean"},
+        :merchantReturnPolicy => {type: "object"},
+        :url => {type: "string", format: "uri"},
+        :eligibleQuantity => {type: "object"},
+        :additionalProperty => {type: "array", items: {type: "object"}}
+      }
+    }.freeze
+
+    RATING = {
+      type: "object",
+      required: ["@type", "ratingValue"],
+      properties: {
+        "@type" => {const: "Rating"},
+        :ratingValue => {type: ["string", "number"]},
+        :bestRating => {type: ["string", "number"]},
+        :worstRating => {type: ["string", "number"]}
+      }
+    }.freeze
+
+    REVIEW = {
+      type: "object",
+      required: ["@type", "itemReviewed"],
+      properties: {
+        "@type" => {const: "Review"},
+        :itemReviewed => {type: "object"},
+        :reviewRating => RATING,
+        :author => {type: ["object", "string"]},
+        :datePublished => {type: "string", format: "date-time"},
+        :reviewBody => {type: "string"}
+      }
+    }.freeze
+
+    REVIEW_SNIPPET = {
+      type: "object",
+      required: ["@type", "reviewRating"],
+      properties: {
+        "@type" => {const: "Review"},
+        :reviewRating => RATING,
+        :author => {type: ["object", "string"]},
+        :reviewBody => {type: "string"},
+        :datePublished => {type: "string", format: "date-time"}
+      }
+    }.freeze
+
+    AGGREGATE_RATING = {
+      type: "object",
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "AggregateRating"},
+        :ratingValue => {type: ["string", "number"]},
+        :ratingCount => {type: "integer"},
+        :reviewCount => {type: "integer"},
+        :bestRating => {type: ["string", "number"]},
+        :worstRating => {type: ["string", "number"]}
+      }
+    }.freeze
+
+    SOFTWARE_APPLICATION = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "SoftwareApplication"},
+        :name => {type: "string"},
+        :applicationCategory => {type: "string"},
+        :description => {type: "string"},
+        :offers => {
+          anyOf: [
+            OFFER,
+            {type: "array", items: OFFER}
+          ]
+        },
+        :featureList => {type: ["string", "array"]},
+        :aggregateRating => AGGREGATE_RATING,
+        :review => REVIEW_SNIPPET
+      }
+    }.freeze
+
+    WEB_APPLICATION = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "WebApplication"},
+        :name => {type: "string"},
+        :applicationCategory => {type: "string"},
+        :description => {type: "string"},
+        :operatingSystem => {type: "string"},
+        :url => {type: "string", format: "uri"},
+        :offers => {
+          anyOf: [
+            OFFER,
+            {type: "array", items: OFFER}
+          ]
+        },
+        :featureList => {type: ["string", "array"]},
+        :aggregateRating => AGGREGATE_RATING,
+        :review => REVIEW_SNIPPET
+      }
+    }.freeze
+
+    HOW_TO = {
+      type: "object",
+      required: ["@type", "name", "step"],
+      properties: {
+        "@type" => {const: "HowTo"},
+        :name => {type: "string"},
+        :description => {type: "string"},
+        :step => {
+          type: "array",
+          minItems: 1,
+          items: {
+            type: "object",
+            required: ["@type", "name", "text"],
+            properties: {
+              "@type" => {const: "HowToStep"},
+              :name => {type: "string"},
+              :text => {type: "string"},
+              :position => {type: "integer", minimum: 1}
+            }
+          }
+        }
+      }
+    }.freeze
+
+    CONTACT_PAGE = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "ContactPage"},
+        :name => {type: "string"},
+        :description => {type: "string"},
+        :url => {type: "string", format: "uri"}
+      }
+    }.freeze
+
+    PRODUCT = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "Product"},
+        :name => {type: "string"},
+        :image => {
+          anyOf: [
+            {type: "string", format: "uri"},
+            IMAGE_OBJECT,
+            {type: "array", items: {anyOf: [{type: "string", format: "uri"}, IMAGE_OBJECT]}}
+          ]
+        },
+        :description => {type: "string"},
+        :offers => {
+          anyOf: [
+            OFFER,
+            {type: "array", items: OFFER}
+          ]
+        }
+      }
+    }.freeze
+
+    RECIPE = {
+      type: "object",
+      required: ["@type", "name"],
+      properties: {
+        "@type" => {const: "Recipe"},
+        :name => {type: "string"},
+        :image => {type: ["string", "array"]},
+        :recipeIngredient => {type: "array", items: {type: "string"}},
+        :recipeInstructions => {type: ["string", "array"]}
+      }
+    }.freeze
+
+    EVENT = {
+      type: "object",
+      required: ["@type", "name", "startDate"],
+      properties: {
+        "@type" => {const: "Event"},
+        :name => {type: "string"},
+        :startDate => {type: "string", format: "date-time"},
+        :endDate => {type: "string", format: "date-time"},
+        :location => {type: "object"}
+      }
+    }.freeze
+
+    VIDEO_OBJECT = {
+      type: "object",
+      required: ["@type", "name", "description"],
+      properties: {
+        "@type" => {const: "VideoObject"},
+        :name => {type: "string"},
+        :description => {type: "string"},
+        :thumbnailUrl => {type: "string", format: "uri"},
+        :uploadDate => {type: "string", format: "date-time"}
+      }
+    }.freeze
+
+    WEBSITE = {
+      type: "object",
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "WebSite"},
+        :name => {type: "string"},
+        :url => {type: "string", format: "uri"},
+        :potentialAction => {type: "object"}
+      }
+    }.freeze
+
+    BREADCRUMB_LIST = {
+      type: "object",
+      required: ["@type", "itemListElement"],
+      properties: {
+        "@type" => {const: "BreadcrumbList"},
+        :itemListElement => {
+          type: "array",
+          minItems: 1,
+          items: {
+            type: "object",
+            required: ["@type", "position", "name", "item"],
+            properties: {
+              "@type" => {const: "ListItem"},
+              :position => {type: "integer", minimum: 1},
+              :name => {type: "string"},
+              :item => {type: "string", format: "uri"}
+            }
+          }
+        }
+      }
+    }.freeze
+
+    WEB_PAGE = {
+      type: "object",
+      required: ["@type"],
+      properties: {
+        "@type" => {const: "WebPage"}
+      }
+    }.freeze
+
+    def self.schemas
+      {
+        "FAQPage" => FAQ_PAGE,
+        "Article" => ARTICLE,
+        "NewsArticle" => ARTICLE,
+        "BlogPosting" => ARTICLE,
+        "Organization" => ORGANIZATION,
+        "SoftwareApplication" => SOFTWARE_APPLICATION,
+        "WebApplication" => WEB_APPLICATION,
+        "HowTo" => HOW_TO,
+        "ContactPage" => CONTACT_PAGE,
+        "Product" => PRODUCT,
+        "Review" => REVIEW,
+        "WebSite" => WEBSITE,
+        "BreadcrumbList" => BREADCRUMB_LIST,
+        "Recipe" => RECIPE,
+        "Event" => EVENT,
+        "VideoObject" => VIDEO_OBJECT,
+        "WebPage" => WEB_PAGE
+      }
+    end
+  end
+end

data/lib/crawlscope/sitemap.rb

@@ -28,12 +28,12 @@ module Crawlscope
 
       if root_name == "sitemapindex"
         document.xpath("//xmlns:sitemap/xmlns:loc", SITEMAP_NAMESPACE).flat_map do |node|
-          child_source = resolve_child_source(source, node.text.to_s.strip)
+          child_source = resolve_child_source(source, node.text.to_s.strip, base_url: base_url)
           collect_urls(child_source, base_url: base_url, visited: visited)
         end
       else
         document.xpath("//xmlns:url/xmlns:loc", SITEMAP_NAMESPACE).map do |node|
-          Url.normalize(node.text.to_s.strip, base_url: base_url)
+          Url.normalize_for_base(node.text.to_s.strip, base_url: base_url)
         end
       end
     end
@@ -46,16 +46,28 @@ module Crawlscope
       end
     end
 
-    def resolve_child_source(parent_source, child_loc)
-      return child_loc if Url.remote?(child_loc)
-
+    def resolve_child_source(parent_source, child_loc, base_url:)
       if Url.remote?(parent_source)
-        URI.join(parent_source, child_loc).to_s
+        Url.normalize_for_base(URI.join(parent_source, child_loc).to_s, base_url: base_url)
+      elsif (local_child_path = local_child_path(parent_source, child_loc))
+        local_child_path
+      elsif Url.remote?(child_loc)
+        child_loc
       else
         File.expand_path(child_loc, File.dirname(parent_source))
       end
     end
 
+    def local_child_path(parent_source, child_loc)
+      basename = File.basename(URI.parse(child_loc).path.to_s)
+      return if basename.empty?
+
+      path = File.expand_path(basename, File.dirname(parent_source))
+      path if File.file?(path)
+    rescue URI::InvalidURIError
+      nil
+    end
+
     def connection
       @connection ||= Faraday.new do |faraday|
         faraday.response :follow_redirects, limit: Http::MAX_REDIRECTS

data/lib/crawlscope/structured_data/audit.rb

@@ -3,7 +3,7 @@
 module Crawlscope
   module StructuredData
     class Audit
-      Entry = Data.define(:url, :status, :structured_items, :errors, :fetch_error, :content_type, :skipped_reason) do
+      Page = Data.define(:url, :status, :structured_items, :errors, :fetch_error, :content_type, :skipped_reason) do
         def json_ld_count
           structured_items.count { |item| item[:source] == "json-ld" }
         end
@@ -21,7 +21,7 @@ module Crawlscope
         end
       end
 
-      Result = Data.define(:entries) do
+      Outcome = Data.define(:entries) do
         def ok?
           entries.all?(&:ok?)
         end
@@ -40,7 +40,7 @@ module Crawlscope
         fetcher = build_fetcher(urls)
         entries = urls.map { |url| validate_url(url, fetcher) }
 
-        Result.new(entries: entries)
+        Outcome.new(entries: entries)
       ensure
         fetcher&.close
       end
@@ -111,9 +111,9 @@ module Crawlscope
         content_type = page.headers["content-type"].to_s
 
         if page.error
-          Entry.new(url: url, status: page.status, structured_items: [], errors: [], fetch_error: page.error, content_type: content_type, skipped_reason: nil)
+          Page.new(url: url, status: page.status, structured_items: [], errors: [], fetch_error: page.error, content_type: content_type, skipped_reason: nil)
         elsif page.status && !(200..299).cover?(page.status.to_i)
-          Entry.new(
+          Page.new(
             url: url,
             status: page.status,
             structured_items: [],
@@ -123,7 +123,7 @@ module Crawlscope
             skipped_reason: nil
           )
         elsif !content_type.empty? && !content_type.include?("text/html")
-          Entry.new(
+          Page.new(
             url: url,
             status: page.status,
             structured_items: [],
@@ -134,7 +134,7 @@ module Crawlscope
           )
         else
           structured_items, errors = build_validation_errors(page)
-          Entry.new(
+          Page.new(
             url: url,
             status: page.status,
             structured_items: structured_items,

data/lib/crawlscope/structured_data/check.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  module StructuredData
+    class Check
+      def initialize(configuration:)
+        @configuration = configuration
+      end
+
+      def call(urls:, debug: false, renderer: @configuration.renderer, timeout_seconds: @configuration.timeout_seconds, report_path: nil, summary: false)
+        result = audit(renderer: renderer, timeout_seconds: timeout_seconds).call(urls: urls)
+        reporter = Reporter.new(io: @configuration.output, report_path: report_path)
+
+        reporter.details(result, debug: debug, renderer: renderer)
+        Writer.new(path: report_path).write(result) if report_path
+        reporter.report(result) if summary
+
+        result
+      end
+
+      private
+
+      def audit(renderer:, timeout_seconds:)
+        Audit.new(
+          browser_factory: @configuration.browser_factory,
+          network_idle_timeout_seconds: @configuration.network_idle_timeout_seconds,
+          renderer: renderer,
+          schema_registry: @configuration.schema_registry,
+          scroll_page: @configuration.scroll_page?,
+          timeout_seconds: timeout_seconds
+        )
+      end
+    end
+  end
+end

data/lib/crawlscope/structured_data/reporter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "uri"
+require "json"
 
 module Crawlscope
   module StructuredData
@@ -21,6 +22,18 @@ module Crawlscope
         end
       end
 
+      def details(result, debug:, renderer:)
+        @io.puts("JavaScript mode enabled (Ferrum)") if renderer == :browser
+        @io.puts("Validating JSON-LD on #{result.entries.size} URL(s)")
+        @io.puts("")
+
+        result.entries.each do |entry|
+          report_entry(entry, debug: debug)
+        end
+
+        @io.puts("STATUS: #{result.ok? ? "OK" : "FAILED"}")
+      end
+
       private
 
       def extract_path(url)
@@ -68,6 +81,62 @@ module Crawlscope
 
         @io.puts("#{report.failure_count} of #{report.total} URLs failed validation.")
       end
+
+      def report_entry(entry, debug:)
+        @io.puts("=" * 80)
+        @io.puts("URL: #{entry.url}")
+        @io.puts("=" * 80)
+
+        if entry.fetch_error
+          @io.puts("Error: #{entry.fetch_error}")
+          @io.puts("")
+          return
+        end
+
+        report_status(entry)
+        @io.puts("Structured data found: #{entry.structured_items.size} (JSON-LD: #{entry.json_ld_count}, Microdata: #{entry.microdata_count})")
+        report_debug(entry) if debug && entry.structured_items.any?
+        report_validation(entry)
+        @io.puts("")
+      end
+
+      def report_status(entry)
+        if entry.status
+          @io.puts("Status: #{entry.status}")
+        else
+          @io.puts("Status: JS runtime fetch")
+        end
+      end
+
+      def report_debug(entry)
+        @io.puts("")
+        @io.puts("--- Detected Structured Data ---")
+
+        entry.structured_items.each_with_index do |item, index|
+          @io.puts("")
+          @io.puts("## Item #{index + 1} [#{item[:source]}]")
+          @io.puts(JSON.pretty_generate(item[:data]))
+        end
+
+        @io.puts("")
+        @io.puts("--- End ---")
+      end
+
+      def report_validation(entry)
+        @io.puts("")
+        @io.puts("Validation results:")
+
+        if entry.errors.empty?
+          @io.puts("  All valid!")
+        else
+          entry.errors.each do |error|
+            @io.puts("  #{error[:type]}: INVALID [#{error[:source]}]")
+            error[:errors].each do |validation_error|
+              @io.puts("    - field: #{validation_error[:field]}, issue: #{validation_error[:issue]}")
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/crawlscope/url.rb

@@ -23,6 +23,20 @@ module Crawlscope
       url.to_s
     end
 
+    def normalize_for_base(url, base_url:)
+      uri = URI.parse(normalize(url, base_url: base_url))
+      base_uri = URI.parse(base_url.to_s)
+      unless base_uri.host.to_s.empty?
+        uri.scheme = base_uri.scheme
+        uri.host = base_uri.host
+        uri.port = base_uri.port
+      end
+
+      normalize(uri.to_s, base_url: base_url)
+    rescue URI::InvalidURIError
+      url.to_s
+    end
+
     def path(url)
       uri = URI.parse(url.to_s)
       value = uri.path.to_s

data/lib/crawlscope/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlscope
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/tasks/crawlscope_tasks.rake

@@ -1,44 +1,33 @@
 namespace :crawlscope do
-  desc "Validate sitemap URLs with the default Crawlscope rules. ENV: BASE_URL, SITEMAP, RULES, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, CONCURRENCY"
+  desc "Validate URLs with all default Crawlscope rules. ENV: URL, SITEMAP, RULES, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, CONCURRENCY"
   task validate: :environment do
-    status = Crawlscope::Cli.start(["validate"], out: $stdout, err: $stderr)
-    exit(status) unless status.zero?
+    Crawlscope::RakeTasks.validate
   end
 
   namespace :validate do
-    desc "Validate JSON-LD on one or more URLs. ENV: URL (required, semicolon-separated), DEBUG=1, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, REPORT_PATH, SUMMARY=1"
+    desc "Directly validate JSON-LD on one or more URLs. ENV: URL (required, semicolon-separated), DEBUG=1, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, REPORT_PATH, SUMMARY=1"
     task ldjson: :environment do
-      status = Crawlscope::Cli.start(["ldjson"], out: $stdout, err: $stderr)
-      exit(status) unless status.zero?
+      Crawlscope::RakeTasks.ldjson
     end
 
-    desc "Validate sitemap URLs with the metadata rule. ENV: BASE_URL, SITEMAP, JS=1"
+    desc "Validate URLs with the metadata rule. ENV: URL, SITEMAP, JS=1"
     task metadata: :environment do
-      crawlscope_task_with_rules("metadata")
+      Crawlscope::RakeTasks.validate_rule("metadata")
     end
 
-    desc "Validate sitemap URLs with the structured_data rule. ENV: BASE_URL, SITEMAP, JS=1"
+    desc "Validate sitemap URLs with the structured_data rule. ENV: URL, SITEMAP, JS=1"
     task structured_data: :environment do
-      crawlscope_task_with_rules("structured_data")
+      Crawlscope::RakeTasks.validate_rule("structured_data")
     end
 
-    desc "Validate sitemap URLs with the uniqueness rule. ENV: BASE_URL, SITEMAP, JS=1"
+    desc "Validate URLs with the uniqueness rule. ENV: URL, SITEMAP, JS=1"
     task uniqueness: :environment do
-      crawlscope_task_with_rules("uniqueness")
+      Crawlscope::RakeTasks.validate_rule("uniqueness")
     end
 
-    desc "Validate sitemap URLs with the links rule. ENV: BASE_URL, SITEMAP, JS=1"
+    desc "Validate URLs with the links rule. ENV: URL, SITEMAP, JS=1"
     task links: :environment do
-      crawlscope_task_with_rules("links")
+      Crawlscope::RakeTasks.validate_rule("links")
     end
   end
-
-  def crawlscope_task_with_rules(rules)
-    original_rules = ENV["RULES"]
-    ENV["RULES"] = rules
-    status = Crawlscope::Cli.start(["validate"], out: $stdout, err: $stderr)
-    exit(status) unless status.zero?
-  ensure
-    ENV["RULES"] = original_rules
-  end
 end