RubyGems - crawlscope - Versions diffs - 0.1.0 → 0.3.0 - Mend

crawlscope 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (44) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +7 -8
data/README.md +21 -14
data/lib/crawlscope/browser.rb +8 -0
data/lib/crawlscope/cli.rb +15 -10
data/lib/crawlscope/configuration.rb +20 -5
data/lib/crawlscope/context.rb +9 -0
data/lib/crawlscope/{audit.rb → crawl.rb} +68 -58
data/lib/crawlscope/crawler.rb +19 -1
data/lib/crawlscope/http.rb +1 -1
data/lib/crawlscope/rake_tasks.rb +28 -0
data/lib/crawlscope/rules/links.rb +99 -48
data/lib/crawlscope/rules/metadata.rb +57 -11
data/lib/crawlscope/rules/structured_data.rb +61 -1
data/lib/crawlscope/run.rb +60 -0
data/lib/crawlscope/schema_registry.rb +3 -349
data/lib/crawlscope/schemas.rb +406 -0
data/lib/crawlscope/sitemap.rb +18 -6
data/lib/crawlscope/structured_data/audit.rb +7 -7
data/lib/crawlscope/structured_data/check.rb +35 -0
data/lib/crawlscope/structured_data/reporter.rb +69 -0
data/lib/crawlscope/url.rb +14 -0
data/lib/crawlscope/version.rb +1 -1
data/lib/tasks/crawlscope_tasks.rake +12 -23
data/test/crawlscope/browser_test.rb +155 -0
data/test/crawlscope/cli_test.rb +143 -7
data/test/crawlscope/configuration_test.rb +49 -0
data/test/crawlscope/{audit_test.rb → crawl_test.rb} +23 -7
data/test/crawlscope/crawler_test.rb +34 -0
data/test/crawlscope/http_test.rb +56 -0
data/test/crawlscope/links_rule_test.rb +149 -5
data/test/crawlscope/metadata_rule_test.rb +77 -0
data/test/crawlscope/rule_registry_test.rb +32 -0
data/test/crawlscope/{task_test.rb → run_test.rb} +28 -33
data/test/crawlscope/schema_registry_test.rb +19 -0
data/test/crawlscope/sitemap_test.rb +55 -0
data/test/crawlscope/structured_data_document_test.rb +36 -0
data/test/crawlscope/structured_data_report_test.rb +3 -3
data/test/crawlscope/structured_data_reporter_test.rb +2 -2
data/test/crawlscope/structured_data_rule_test.rb +111 -0
data/test/crawlscope/structured_data_writer_test.rb +2 -2
data/test/crawlscope/url_test.rb +31 -0
metadata +15 -5
data/lib/crawlscope/task.rb +0 -131

data/lib/crawlscope/schema_registry.rb CHANGED Viewed

@@ -2,362 +2,16 @@
 require "json-schema"
+JSON::Validator.use_multi_json = false
 module Crawlscope
   class SchemaRegistry
-    FAQ_PAGE = {
-      "type" => "object",
-      "required" => ["@context", "@type", "mainEntity"],
-      "properties" => {
-        "@context" => {"const" => "https://schema.org"},
-        "@type" => {"const" => "FAQPage"},
-        "mainEntity" => {
-          "type" => "array",
-          "minItems" => 1,
-          "items" => {"$ref" => "#/definitions/Question"}
-        }
-      },
-      "definitions" => {
-        "Question" => {
-          "type" => "object",
-          "required" => ["@type", "name", "acceptedAnswer"],
-          "properties" => {
-            "@type" => {"const" => "Question"},
-            "name" => {"type" => "string"},
-            "acceptedAnswer" => {"$ref" => "#/definitions/Answer"}
-          }
-        },
-        "Answer" => {
-          "type" => "object",
-          "required" => ["@type", "text"],
-          "properties" => {
-            "@type" => {"const" => "Answer"},
-            "text" => {"type" => "string"}
-          }
-        }
-      }
-    }.freeze
-    ARTICLE = {
-      type: "object",
-      required: ["@type", "headline"],
-      properties: {
-        "@type" => {enum: ["Article", "NewsArticle", "BlogPosting"]},
-        :headline => {type: "string", maxLength: 110},
-        :image => {type: "string", format: "uri"},
-        :datePublished => {type: "string", format: "date-time"},
-        :dateModified => {type: "string", format: "date-time"},
-        :author => {type: "object"},
-        :publisher => {type: "object"}
-      }
-    }.freeze
-    ORGANIZATION = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "Organization"},
-        :name => {type: "string"},
-        :url => {type: "string", format: "uri"},
-        :logo => {
-          anyOf: [
-            {type: "string", format: "uri"},
-            {
-              type: "object",
-              required: ["@type", "url"],
-              properties: {
-                "@type" => {const: "ImageObject"},
-                :url => {type: "string", format: "uri"}
-              }
-            }
-          ]
-        },
-        :description => {type: "string"}
-      }
-    }.freeze
-    IMAGE_OBJECT = {
-      type: "object",
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "ImageObject"},
-        :url => {type: "string", format: "uri"},
-        :contentUrl => {type: "string", format: "uri"},
-        :thumbnail => {type: ["string", "object"]}
-      }
-    }.freeze
-    OFFER = {
-      type: "object",
-      additionalProperties: true,
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "Offer"},
-        :name => {type: ["string", "null"]},
-        :price => {type: ["string", "number"]},
-        :priceCurrency => {type: ["string", "null"]},
-        :priceSpecification => {type: ["object", "null"]},
-        :availability => {type: "string"},
-        :shippingDetails => {type: "object"},
-        :hasMerchantReturnPolicy => {type: "boolean"},
-        :merchantReturnPolicy => {type: "object"},
-        :url => {type: "string", format: "uri"},
-        :eligibleQuantity => {type: "object"},
-        :additionalProperty => {type: "array", items: {type: "object"}}
-      }
-    }.freeze
-    RATING = {
-      type: "object",
-      required: ["@type", "ratingValue"],
-      properties: {
-        "@type" => {const: "Rating"},
-        :ratingValue => {type: ["string", "number"]},
-        :bestRating => {type: ["string", "number"]},
-        :worstRating => {type: ["string", "number"]}
-      }
-    }.freeze
-    REVIEW = {
-      type: "object",
-      required: ["@type", "itemReviewed"],
-      properties: {
-        "@type" => {const: "Review"},
-        :itemReviewed => {type: "object"},
-        :reviewRating => RATING,
-        :author => {type: ["object", "string"]},
-        :datePublished => {type: "string", format: "date-time"},
-        :reviewBody => {type: "string"}
-      }
-    }.freeze
-    REVIEW_SNIPPET = {
-      type: "object",
-      required: ["@type", "reviewRating"],
-      properties: {
-        "@type" => {const: "Review"},
-        :reviewRating => RATING,
-        :author => {type: ["object", "string"]},
-        :reviewBody => {type: "string"},
-        :datePublished => {type: "string", format: "date-time"}
-      }
-    }.freeze
-    AGGREGATE_RATING = {
-      type: "object",
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "AggregateRating"},
-        :ratingValue => {type: ["string", "number"]},
-        :ratingCount => {type: "integer"},
-        :reviewCount => {type: "integer"},
-        :bestRating => {type: ["string", "number"]},
-        :worstRating => {type: ["string", "number"]}
-      }
-    }.freeze
-    SOFTWARE_APPLICATION = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "SoftwareApplication"},
-        :name => {type: "string"},
-        :applicationCategory => {type: "string"},
-        :description => {type: "string"},
-        :offers => {
-          anyOf: [
-            OFFER,
-            {type: "array", items: OFFER}
-          ]
-        },
-        :featureList => {type: ["string", "array"]},
-        :aggregateRating => AGGREGATE_RATING,
-        :review => REVIEW_SNIPPET
-      }
-    }.freeze
-    WEB_APPLICATION = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "WebApplication"},
-        :name => {type: "string"},
-        :applicationCategory => {type: "string"},
-        :description => {type: "string"},
-        :operatingSystem => {type: "string"},
-        :url => {type: "string", format: "uri"},
-        :offers => {
-          anyOf: [
-            OFFER,
-            {type: "array", items: OFFER}
-          ]
-        },
-        :featureList => {type: ["string", "array"]},
-        :aggregateRating => AGGREGATE_RATING,
-        :review => REVIEW_SNIPPET
-      }
-    }.freeze
-    HOW_TO = {
-      type: "object",
-      required: ["@type", "name", "step"],
-      properties: {
-        "@type" => {const: "HowTo"},
-        :name => {type: "string"},
-        :description => {type: "string"},
-        :step => {
-          type: "array",
-          minItems: 1,
-          items: {
-            type: "object",
-            required: ["@type", "name", "text"],
-            properties: {
-              "@type" => {const: "HowToStep"},
-              :name => {type: "string"},
-              :text => {type: "string"},
-              :position => {type: "integer", minimum: 1}
-            }
-          }
-        }
-      }
-    }.freeze
-    CONTACT_PAGE = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "ContactPage"},
-        :name => {type: "string"},
-        :description => {type: "string"},
-        :url => {type: "string", format: "uri"}
-      }
-    }.freeze
-    PRODUCT = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "Product"},
-        :name => {type: "string"},
-        :image => {
-          anyOf: [
-            {type: "string", format: "uri"},
-            IMAGE_OBJECT,
-            {type: "array", items: {anyOf: [{type: "string", format: "uri"}, IMAGE_OBJECT]}}
-          ]
-        },
-        :description => {type: "string"},
-        :offers => {
-          anyOf: [
-            OFFER,
-            {type: "array", items: OFFER}
-          ]
-        }
-      }
-    }.freeze
-    RECIPE = {
-      type: "object",
-      required: ["@type", "name"],
-      properties: {
-        "@type" => {const: "Recipe"},
-        :name => {type: "string"},
-        :image => {type: ["string", "array"]},
-        :recipeIngredient => {type: "array", items: {type: "string"}},
-        :recipeInstructions => {type: ["string", "array"]}
-      }
-    }.freeze
-    EVENT = {
-      type: "object",
-      required: ["@type", "name", "startDate"],
-      properties: {
-        "@type" => {const: "Event"},
-        :name => {type: "string"},
-        :startDate => {type: "string", format: "date-time"},
-        :endDate => {type: "string", format: "date-time"},
-        :location => {type: "object"}
-      }
-    }.freeze
-    VIDEO_OBJECT = {
-      type: "object",
-      required: ["@type", "name", "description"],
-      properties: {
-        "@type" => {const: "VideoObject"},
-        :name => {type: "string"},
-        :description => {type: "string"},
-        :thumbnailUrl => {type: "string", format: "uri"},
-        :uploadDate => {type: "string", format: "date-time"}
-      }
-    }.freeze
-    WEBSITE = {
-      type: "object",
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "WebSite"},
-        :name => {type: "string"},
-        :url => {type: "string", format: "uri"},
-        :potentialAction => {type: "object"}
-      }
-    }.freeze
-    BREADCRUMB_LIST = {
-      type: "object",
-      required: ["@type", "itemListElement"],
-      properties: {
-        "@type" => {const: "BreadcrumbList"},
-        :itemListElement => {
-          type: "array",
-          minItems: 1,
-          items: {
-            type: "object",
-            required: ["@type", "position", "name", "item"],
-            properties: {
-              "@type" => {const: "ListItem"},
-              :position => {type: "integer", minimum: 1},
-              :name => {type: "string"},
-              :item => {type: "string", format: "uri"}
-            }
-          }
-        }
-      }
-    }.freeze
-    WEB_PAGE = {
-      type: "object",
-      required: ["@type"],
-      properties: {
-        "@type" => {const: "WebPage"}
-      }
-    }.freeze
     def initialize(schemas: {})
       @schemas = schemas.transform_keys(&:to_s).dup
     end
     def self.default
-      new(
-        schemas: {
-          "FAQPage" => FAQ_PAGE,
-          "Article" => ARTICLE,
-          "NewsArticle" => ARTICLE,
-          "BlogPosting" => ARTICLE,
-          "Organization" => ORGANIZATION,
-          "SoftwareApplication" => SOFTWARE_APPLICATION,
-          "WebApplication" => WEB_APPLICATION,
-          "HowTo" => HOW_TO,
-          "ContactPage" => CONTACT_PAGE,
-          "Product" => PRODUCT,
-          "Review" => REVIEW,
-          "WebSite" => WEBSITE,
-          "BreadcrumbList" => BREADCRUMB_LIST,
-          "Recipe" => RECIPE,
-          "Event" => EVENT,
-          "VideoObject" => VIDEO_OBJECT,
-          "WebPage" => WEB_PAGE
-        }
-      )
+      new(schemas: Schemas.schemas)
     end
     def dup