crawlscope 0.1.0 → 0.2.0

data/lib/crawlscope/rules/links.rb

@@ -40,41 +40,49 @@ module Crawlscope
       end
 
       def extract_links(pages)
-        links = []
+        pages.select(&:html?).flat_map { |page| page_links(page) }
+      end
 
-        pages.each do |page|
-          next unless page.html?
+      def page_links(page)
+        source_path = Url.path(page.normalized_url)
+        return [] unless crawlable_path?(source_path)
 
-          source_path = Url.path(page.normalized_url)
-          next if source_path.nil?
+        contextual_links(page.doc).filter_map do |node|
+          link_for(page: page, source_path: source_path, node: node)
+        end
+      end
 
-          contextual_links(page.doc).each do |node|
-            href = node["href"].to_s.strip
-            next if href.empty?
-            next if href.start_with?("#")
-            next if LINK_SCHEMES_TO_SKIP.any? { |prefix| href.start_with?(prefix) }
+      def link_for(page:, source_path:, node:)
+        href = node["href"].to_s.strip
+        return unless crawlable_href?(href)
 
-            anchor_text = normalize_anchor_text(node.text)
-            next if anchor_text.empty?
+        anchor_text = normalize_anchor_text(node.text)
+        return if anchor_text.empty?
 
-            target_url = normalize_internal_link(page.normalized_url, href)
-            next if target_url.nil?
+        target_url = normalize_internal_link(page.normalized_url, href)
+        return if target_url.nil?
 
-            target_path = Url.path(target_url)
-            next if target_path.nil?
-            next if skip_internal_path?(target_path)
+        target_path = Url.path(target_url)
+        return unless crawlable_path?(target_path)
 
-            links << {
-              anchor_text: anchor_text,
-              source_path: source_path,
-              source_url: page.normalized_url,
-              target_path: target_path,
-              target_url: target_url
-            }
-          end
-        end
+        {
+          anchor_text: anchor_text,
+          source_path: source_path,
+          source_url: page.normalized_url,
+          target_path: target_path,
+          target_url: target_url
+        }
+      end
 
-        links
+      def crawlable_href?(href)
+        return false if href.empty?
+        return false if href.start_with?("#")
+
+        LINK_SCHEMES_TO_SKIP.none? { |prefix| href.start_with?(prefix) }
+      end
+
+      def crawlable_path?(path)
+        !path.nil? && !skip_internal_path?(path)
       end
 
       def normalize_anchor_text(text)
@@ -122,39 +130,64 @@ module Crawlscope
         resolved_links = []
 
         links.group_by { |link| link[:target_url] }.each do |target_url, grouped_links|
-          resolution = @resolve_target.call(target_url)
-          if resolution.nil?
-            report_unresolved_target(target_url, grouped_links, issues, resolution)
+          target = resolve_target(target_url)
+
+          if target.unresolved?
+            report_unresolved_target(target_url, grouped_links, issues, target.resolution)
             next
           end
 
-          status = resolution[:status]
-
-          if status.nil?
-            next if resolution[:crawled] && resolution[:error]
-
-            report_unresolved_target(target_url, grouped_links, issues, resolution)
+          if target.ignored_error?
             next
           end
 
-          unless @allowed_statuses.include?(status)
-            report_broken_target(target_url, grouped_links, issues, status)
+          unless target.allowed?(@allowed_statuses)
+            report_broken_target(target_url, grouped_links, issues, target.status)
             next
           end
 
-          final_url = resolution[:final_url].to_s.empty? ? target_url : resolution[:final_url]
-          final_path = Url.path(final_url)
-          next if final_path.nil?
-          next if skip_internal_path?(final_path)
+          next unless crawlable_path?(target.final_path)
 
           grouped_links.each do |link|
-            resolved_links << link.merge(final_path: final_path, final_url: final_url)
+            resolved_links << link.merge(final_path: target.final_path, final_url: target.final_url)
           end
         end
 
         resolved_links
       end
 
+      def resolve_target(target_url)
+        resolution = @resolve_target.call(target_url)
+        LinkTarget.new(target_url: target_url, resolution: resolution)
+      end
+
+      LinkTarget = Data.define(:target_url, :resolution) do
+        def allowed?(statuses)
+          statuses.include?(status)
+        end
+
+        def final_path
+          Url.path(final_url)
+        end
+
+        def final_url
+          value = resolution[:final_url].to_s
+          value.empty? ? target_url : value
+        end
+
+        def ignored_error?
+          resolution && status.nil? && resolution[:crawled] && resolution[:error]
+        end
+
+        def status
+          resolution && resolution[:status]
+        end
+
+        def unresolved?
+          resolution.nil? || (status.nil? && !ignored_error?)
+        end
+      end
+
       def skip_internal_path?(path)
         return true if path == "/"
 

data/lib/crawlscope/rules/structured_data.rb

@@ -23,8 +23,21 @@ module Crawlscope
 
       def validate_page(page, issues, schema_registry)
         document = Crawlscope::StructuredData::Document.new(html: page.body)
+        items = document.items
 
-        document.items.each do |item|
+        if items.empty?
+          issues.add(
+            code: :missing_structured_data,
+            severity: :warning,
+            category: :structured_data,
+            url: page.url,
+            message: "no structured data found; add JSON-LD or microdata markup",
+            details: {expected_sources: ["json-ld", "microdata"]}
+          )
+          return
+        end
+
+        items.each do |item|
           data = item.data
           source = item.source
 

data/lib/crawlscope/run.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class Run
+    def initialize(configuration: Crawlscope.configuration, reporter: Reporter.new(io: configuration.output))
+      @configuration = configuration
+      @reporter = reporter
+    end
+
+    def validate(base_url: nil, sitemap_path: nil, rule_names: nil)
+      resolved_base_url = base_url || default_base_url
+      crawl = @configuration.audit(
+        base_url: resolved_base_url,
+        sitemap_path: sitemap_path || default_sitemap_path(base_url: resolved_base_url),
+        rule_names: rule_names
+      )
+
+      result = crawl.call
+      @reporter.report(result)
+      result
+    end
+
+    def validate_json_ld(urls:, debug: false, renderer: @configuration.renderer, timeout_seconds: @configuration.timeout_seconds, report_path: nil, summary: false)
+      StructuredData::Check.new(configuration: @configuration).call(
+        urls: urls,
+        debug: debug,
+        renderer: renderer,
+        timeout_seconds: timeout_seconds,
+        report_path: report_path,
+        summary: summary
+      )
+    end
+
+    private
+
+    def default_base_url
+      value = @configuration.base_url
+      return value unless value.to_s.strip.empty?
+
+      "http://localhost:3000"
+    end
+
+    def default_sitemap_path(base_url:)
+      value = @configuration.sitemap_path
+      return value unless value.to_s.strip.empty?
+
+      local_path = File.expand_path("public/sitemap.xml", Dir.pwd)
+      return local_path if local_path_default?(base_url: base_url) && File.exist?(local_path)
+
+      "#{base_url.to_s.chomp("/")}/sitemap.xml"
+    end
+
+    def local_path_default?(base_url:)
+      host = URI.parse(base_url.to_s).host.to_s
+      ["localhost", "127.0.0.1"].include?(host)
+    rescue URI::InvalidURIError
+      false
+    end
+  end
+end

data/lib/crawlscope/schema_registry.rb

@@ -2,362 +2,16 @@
 
 require "json-schema"
 
+JSON::Validator.use_multi_json = false
+
 module Crawlscope
   class SchemaRegistry
-    FAQ_PAGE = {
-      "type" => "object",
-      "required" => ["@context", "@type", "mainEntity"],
-      "properties" => {
-        "@context" => {"const" => "https://schema.org"},
-        "@type" => {"const" => "FAQPage"},
-        "mainEntity" => {
-          "type" => "array",
-          "minItems" => 1,
-          "items" => {"$ref" => "#/definitions/Question"}
-        }
-      },
-      "definitions" => {
-        "Question" => {
-          "type" => "object",
-          "required" => ["@type", "name", "acceptedAnswer"],
-          "properties" => {
-            "@type" => {"const" => "Question"},
-            "name" => {"type" => "string"},
-            "acceptedAnswer" => {"$ref" => "#/definitions/Answer"}
-          }
-        },
-        "Answer" => {
-          "type" => "object",
-          "required" => ["@type", "text"],
-          "properties" => {
-            "@type" => {"const" => "Answer"},
-            "text" => {"type" => "string"}
-          }
-        }
-      }
-    }.freeze
-
-    ARTICLE = {
-      type: "object",
-      required: ["@type", "headline"],
-      properties: {
-        "@type" => {enum: ["Article", "NewsArticle", "BlogPosting"]},
-        :headline => {type: "string", maxLength: 110},
-        :image => {type: "string", format: "uri"},
-        :datePublished => {type: "string", format: "date-time"},
-        :dateModified => {type: "string", format: "date-time"},
-        :author => {type: "object"},
-        :publisher => {type: "object"}
-      }
-    }.freeze
-
-    ORGANIZATION = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "Organization"},
-        :name => {type: "string"},
-        :url => {type: "string", format: "uri"},
-        :logo => {
-          anyOf: [
-            {type: "string", format: "uri"},
-            {
-              type: "object",
-              required: ["@type", "url"],
-              properties: {
-                "@type" => {const: "ImageObject"},
-                :url => {type: "string", format: "uri"}
-              }
-            }
-          ]
-        },
-        :description => {type: "string"}
-      }
-    }.freeze
-
-    IMAGE_OBJECT = {
-      type: "object",
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "ImageObject"},
-        :url => {type: "string", format: "uri"},
-        :contentUrl => {type: "string", format: "uri"},
-        :thumbnail => {type: ["string", "object"]}
-      }
-    }.freeze
-
-    OFFER = {
-      type: "object",
-      additionalProperties: true,
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "Offer"},
-        :name => {type: ["string", "null"]},
-        :price => {type: ["string", "number"]},
-        :priceCurrency => {type: ["string", "null"]},
-        :priceSpecification => {type: ["object", "null"]},
-        :availability => {type: "string"},
-        :shippingDetails => {type: "object"},
-        :hasMerchantReturnPolicy => {type: "boolean"},
-        :merchantReturnPolicy => {type: "object"},
-        :url => {type: "string", format: "uri"},
-        :eligibleQuantity => {type: "object"},
-        :additionalProperty => {type: "array", items: {type: "object"}}
-      }
-    }.freeze
-
-    RATING = {
-      type: "object",
-      required: ["@type", "ratingValue"],
-      properties: {
-        "@type" => {const: "Rating"},
-        :ratingValue => {type: ["string", "number"]},
-        :bestRating => {type: ["string", "number"]},
-        :worstRating => {type: ["string", "number"]}
-      }
-    }.freeze
-
-    REVIEW = {
-      type: "object",
-      required: ["@type", "itemReviewed"],
-      properties: {
-        "@type" => {const: "Review"},
-        :itemReviewed => {type: "object"},
-        :reviewRating => RATING,
-        :author => {type: ["object", "string"]},
-        :datePublished => {type: "string", format: "date-time"},
-        :reviewBody => {type: "string"}
-      }
-    }.freeze
-
-    REVIEW_SNIPPET = {
-      type: "object",
-      required: ["@type", "reviewRating"],
-      properties: {
-        "@type" => {const: "Review"},
-        :reviewRating => RATING,
-        :author => {type: ["object", "string"]},
-        :reviewBody => {type: "string"},
-        :datePublished => {type: "string", format: "date-time"}
-      }
-    }.freeze
-
-    AGGREGATE_RATING = {
-      type: "object",
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "AggregateRating"},
-        :ratingValue => {type: ["string", "number"]},
-        :ratingCount => {type: "integer"},
-        :reviewCount => {type: "integer"},
-        :bestRating => {type: ["string", "number"]},
-        :worstRating => {type: ["string", "number"]}
-      }
-    }.freeze
-
-    SOFTWARE_APPLICATION = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "SoftwareApplication"},
-        :name => {type: "string"},
-        :applicationCategory => {type: "string"},
-        :description => {type: "string"},
-        :offers => {
-          anyOf: [
-            OFFER,
-            {type: "array", items: OFFER}
-          ]
-        },
-        :featureList => {type: ["string", "array"]},
-        :aggregateRating => AGGREGATE_RATING,
-        :review => REVIEW_SNIPPET
-      }
-    }.freeze
-
-    WEB_APPLICATION = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "WebApplication"},
-        :name => {type: "string"},
-        :applicationCategory => {type: "string"},
-        :description => {type: "string"},
-        :operatingSystem => {type: "string"},
-        :url => {type: "string", format: "uri"},
-        :offers => {
-          anyOf: [
-            OFFER,
-            {type: "array", items: OFFER}
-          ]
-        },
-        :featureList => {type: ["string", "array"]},
-        :aggregateRating => AGGREGATE_RATING,
-        :review => REVIEW_SNIPPET
-      }
-    }.freeze
-
-    HOW_TO = {
-      type: "object",
-      required: ["@type", "name", "step"],
-      properties: {
-        "@type" => {const: "HowTo"},
-        :name => {type: "string"},
-        :description => {type: "string"},
-        :step => {
-          type: "array",
-          minItems: 1,
-          items: {
-            type: "object",
-            required: ["@type", "name", "text"],
-            properties: {
-              "@type" => {const: "HowToStep"},
-              :name => {type: "string"},
-              :text => {type: "string"},
-              :position => {type: "integer", minimum: 1}
-            }
-          }
-        }
-      }
-    }.freeze
-
-    CONTACT_PAGE = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "ContactPage"},
-        :name => {type: "string"},
-        :description => {type: "string"},
-        :url => {type: "string", format: "uri"}
-      }
-    }.freeze
-
-    PRODUCT = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "Product"},
-        :name => {type: "string"},
-        :image => {
-          anyOf: [
-            {type: "string", format: "uri"},
-            IMAGE_OBJECT,
-            {type: "array", items: {anyOf: [{type: "string", format: "uri"}, IMAGE_OBJECT]}}
-          ]
-        },
-        :description => {type: "string"},
-        :offers => {
-          anyOf: [
-            OFFER,
-            {type: "array", items: OFFER}
-          ]
-        }
-      }
-    }.freeze
-
-    RECIPE = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "Recipe"},
-        :name => {type: "string"},
-        :image => {type: ["string", "array"]},
-        :recipeIngredient => {type: "array", items: {type: "string"}},
-        :recipeInstructions => {type: ["string", "array"]}
-      }
-    }.freeze
-
-    EVENT = {
-      type: "object",
-      required: ["@type", "name", "startDate"],
-      properties: {
-        "@type" => {const: "Event"},
-        :name => {type: "string"},
-        :startDate => {type: "string", format: "date-time"},
-        :endDate => {type: "string", format: "date-time"},
-        :location => {type: "object"}
-      }
-    }.freeze
-
-    VIDEO_OBJECT = {
-      type: "object",
-      required: ["@type", "name", "description"],
-      properties: {
-        "@type" => {const: "VideoObject"},
-        :name => {type: "string"},
-        :description => {type: "string"},
-        :thumbnailUrl => {type: "string", format: "uri"},
-        :uploadDate => {type: "string", format: "date-time"}
-      }
-    }.freeze
-
-    WEBSITE = {
-      type: "object",
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "WebSite"},
-        :name => {type: "string"},
-        :url => {type: "string", format: "uri"},
-        :potentialAction => {type: "object"}
-      }
-    }.freeze
-
-    BREADCRUMB_LIST = {
-      type: "object",
-      required: ["@type", "itemListElement"],
-      properties: {
-        "@type" => {const: "BreadcrumbList"},
-        :itemListElement => {
-          type: "array",
-          minItems: 1,
-          items: {
-            type: "object",
-            required: ["@type", "position", "name", "item"],
-            properties: {
-              "@type" => {const: "ListItem"},
-              :position => {type: "integer", minimum: 1},
-              :name => {type: "string"},
-              :item => {type: "string", format: "uri"}
-            }
-          }
-        }
-      }
-    }.freeze
-
-    WEB_PAGE = {
-      type: "object",
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "WebPage"}
-      }
-    }.freeze
-
     def initialize(schemas: {})
       @schemas = schemas.transform_keys(&:to_s).dup
     end
 
     def self.default
-      new(
-        schemas: {
-          "FAQPage" => FAQ_PAGE,
-          "Article" => ARTICLE,
-          "NewsArticle" => ARTICLE,
-          "BlogPosting" => ARTICLE,
-          "Organization" => ORGANIZATION,
-          "SoftwareApplication" => SOFTWARE_APPLICATION,
-          "WebApplication" => WEB_APPLICATION,
-          "HowTo" => HOW_TO,
-          "ContactPage" => CONTACT_PAGE,
-          "Product" => PRODUCT,
-          "Review" => REVIEW,
-          "WebSite" => WEBSITE,
-          "BreadcrumbList" => BREADCRUMB_LIST,
-          "Recipe" => RECIPE,
-          "Event" => EVENT,
-          "VideoObject" => VIDEO_OBJECT,
-          "WebPage" => WEB_PAGE
-        }
-      )
+      new(schemas: Schemas.schemas)
     end
 
     def dup