RubyGems - crawlscope - Versions diffs - 0.1.0 - Mend

crawlscope 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (52) hide show

checksums.yaml +7 -0
data/CHANGELOG.md +31 -0
data/LICENSE.txt +21 -0
data/README.md +323 -0
data/exe/crawlscope +6 -0
data/lib/crawlscope/audit.rb +128 -0
data/lib/crawlscope/browser.rb +88 -0
data/lib/crawlscope/cli.rb +245 -0
data/lib/crawlscope/configuration.rb +123 -0
data/lib/crawlscope/crawler.rb +28 -0
data/lib/crawlscope/http.rb +77 -0
data/lib/crawlscope/issue.rb +17 -0
data/lib/crawlscope/issue_collection.rb +41 -0
data/lib/crawlscope/page.rb +23 -0
data/lib/crawlscope/railtie.rb +9 -0
data/lib/crawlscope/reporter.rb +33 -0
data/lib/crawlscope/result.rb +9 -0
data/lib/crawlscope/rule_registry.rb +39 -0
data/lib/crawlscope/rules/links.rb +220 -0
data/lib/crawlscope/rules/metadata.rb +93 -0
data/lib/crawlscope/rules/structured_data.rb +58 -0
data/lib/crawlscope/rules/uniqueness.rb +88 -0
data/lib/crawlscope/schema_registry.rb +431 -0
data/lib/crawlscope/sitemap.rb +67 -0
data/lib/crawlscope/structured_data/audit.rb +150 -0
data/lib/crawlscope/structured_data/document.rb +93 -0
data/lib/crawlscope/structured_data/report.rb +77 -0
data/lib/crawlscope/structured_data/reporter.rb +73 -0
data/lib/crawlscope/structured_data/writer.rb +26 -0
data/lib/crawlscope/task.rb +131 -0
data/lib/crawlscope/url.rb +43 -0
data/lib/crawlscope/version.rb +5 -0
data/lib/crawlscope.rb +34 -0
data/lib/tasks/crawlscope_tasks.rake +44 -0
data/test/crawlscope/audit_test.rb +165 -0
data/test/crawlscope/cli_test.rb +157 -0
data/test/crawlscope/configuration_test.rb +45 -0
data/test/crawlscope/links_rule_test.rb +87 -0
data/test/crawlscope/loader_test.rb +11 -0
data/test/crawlscope/reporter_test.rb +50 -0
data/test/crawlscope/schema_registry_test.rb +89 -0
data/test/crawlscope/sitemap_test.rb +51 -0
data/test/crawlscope/structured_data_audit_test.rb +118 -0
data/test/crawlscope/structured_data_document_test.rb +28 -0
data/test/crawlscope/structured_data_report_test.rb +37 -0
data/test/crawlscope/structured_data_reporter_test.rb +32 -0
data/test/crawlscope/structured_data_rule_test.rb +78 -0
data/test/crawlscope/structured_data_writer_test.rb +32 -0
data/test/crawlscope/task_test.rb +206 -0
data/test/crawlscope/uniqueness_rule_test.rb +46 -0
data/test/test_helper.rb +23 -0
metadata +271 -0

data/test/crawlscope/structured_data_reporter_test.rb ADDED Viewed

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+require "stringio"
+require "test_helper"
+class CrawlscopeStructuredDataReporterTest < Minitest::Test
+  def test_reports_failures_and_report_path
+    result = Crawlscope::StructuredData::Audit::Result.new(
+      entries: [
+        Crawlscope::StructuredData::Audit::Entry.new(
+          url: "https://example.com/article",
+          status: 200,
+          structured_items: [{source: "json-ld", data: {"@type" => "Article"}}],
+          errors: [{type: "Article", source: "json-ld", errors: [{field: "headline", issue: "is required"}]}],
+          fetch_error: nil,
+          content_type: "text/html",
+          skipped_reason: nil
+        )
+      ]
+    )
+    io = StringIO.new
+    Crawlscope::StructuredData::Reporter.new(io: io, report_path: "/tmp/structured_data_report.json").report(result)
+    output = io.string
+    assert_includes output, "VALIDATION FAILED"
+    assert_includes output, "VALIDATION ERRORS (1):"
+    assert_includes output, "headline: is required"
+    assert_includes output, "/tmp/structured_data_report.json"
+  end
+end

data/test/crawlscope/structured_data_rule_test.rb ADDED Viewed

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+require "test_helper"
+class CrawlscopeStructuredDataRuleTest < Minitest::Test
+  def test_reports_schema_errors_for_invalid_article_markup
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::StructuredData.new
+    page = page(
+      url: "https://example.com/articles/test",
+      body: <<~HTML
+        <html>
+          <head>
+            <script type="application/ld+json">
+              {"@context":"https://schema.org","@type":"Article"}
+            </script>
+          </head>
+          <body>
+            <main><h1>Article</h1></main>
+          </body>
+        </html>
+      HTML
+    )
+    rule.call(
+      urls: [page.url],
+      pages: [page],
+      issues: issues,
+      context: {schema_registry: Crawlscope::SchemaRegistry.default}
+    )
+    assert_equal [:structured_data_schema_error], issues.to_a.map(&:code)
+    assert_includes issues.to_a.first.message, "headline"
+  end
+  def test_reports_parse_errors_for_invalid_json_ld
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::StructuredData.new
+    page = page(
+      url: "https://example.com/articles/test",
+      body: <<~HTML
+        <html>
+          <head>
+            <script type="application/ld+json">
+              {"@context":"https://schema.org","@type":"Article"
+            </script>
+          </head>
+        </html>
+      HTML
+    )
+    rule.call(
+      urls: [page.url],
+      pages: [page],
+      issues: issues,
+      context: {schema_registry: Crawlscope::SchemaRegistry.default}
+    )
+    assert_equal [:structured_data_parse_error], issues.to_a.map(&:code)
+  end
+  private
+  def page(url:, body:)
+    doc = Nokogiri::HTML(body)
+    Crawlscope::Page.new(
+      url: url,
+      normalized_url: url,
+      final_url: url,
+      normalized_final_url: url,
+      status: 200,
+      headers: {"content-type" => "text/html"},
+      body: body,
+      doc: doc
+    )
+  end
+end

data/test/crawlscope/structured_data_writer_test.rb ADDED Viewed

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+require "json"
+require "test_helper"
+class CrawlscopeStructuredDataWriterTest < Minitest::Test
+  def test_writes_json_report
+    result = Crawlscope::StructuredData::Audit::Result.new(
+      entries: [
+        Crawlscope::StructuredData::Audit::Entry.new(
+          url: "https://example.com/article",
+          status: 200,
+          structured_items: [{source: "json-ld", data: {"@type" => "Article"}}],
+          errors: [],
+          fetch_error: nil,
+          content_type: "text/html",
+          skipped_reason: nil
+        )
+      ]
+    )
+    tmp_dir = Dir.mktmpdir
+    path = File.join(tmp_dir, "structured_data_report.json")
+    Crawlscope::StructuredData::Writer.new(path: path).write(result)
+    payload = JSON.parse(File.read(path))
+    assert payload["generated_at"]
+    assert_equal 1, payload["results"]["https://example.com/article"]["json_ld_count"]
+  ensure
+    FileUtils.rm_rf(tmp_dir) if tmp_dir
+  end
+end

data/test/crawlscope/task_test.rb ADDED Viewed

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+require "test_helper"
+class CrawlscopeTaskTest < Minitest::Test
+  FakeResult = Data.define(:reported) do
+    def ok?
+      true
+    end
+  end
+  class FakeReporter
+    attr_reader :result
+    def report(result)
+      @result = result
+    end
+  end
+  class FakeAudit
+    def initialize(result:)
+      @result = result
+    end
+    def call
+      @result
+    end
+  end
+  class FakeConfiguration
+    attr_reader :base_url, :received_arguments, :sitemap_path
+    def initialize(result:, base_url: "https://example.com", sitemap_path: "/tmp/sitemap.xml")
+      @result = result
+      @base_url = base_url
+      @sitemap_path = sitemap_path
+    end
+    def audit(base_url:, sitemap_path:, rule_names:)
+      @received_arguments = {
+        base_url: base_url,
+        sitemap_path: sitemap_path,
+        rule_names: rule_names
+      }
+      FakeAudit.new(result: @result)
+    end
+  end
+  class LdjsonConfiguration
+    attr_reader :output
+    def initialize(output:, browser:)
+      @output = output
+      @browser = browser
+    end
+    def browser_factory
+      -> { @browser }
+    end
+    def network_idle_timeout_seconds
+      5
+    end
+    def renderer
+      :browser
+    end
+    def schema_registry
+      Crawlscope::SchemaRegistry.default
+    end
+    def scroll_page?
+      false
+    end
+    def timeout_seconds
+      20
+    end
+  end
+  class FakeBrowser
+    attr_reader :closed
+    def initialize(page:)
+      @page = page
+      @closed = false
+    end
+    def close
+      @closed = true
+    end
+    def fetch(_url)
+      @page
+    end
+  end
+  def test_validate_passes_rule_names_to_configuration_audit
+    result = FakeResult.new(reported: true)
+    configuration = FakeConfiguration.new(result: result)
+    reporter = FakeReporter.new
+    task = Crawlscope::Task.new(configuration: configuration, reporter: reporter)
+    returned_result = task.validate(rule_names: "links")
+    assert_equal(
+      {
+        base_url: "https://example.com",
+        sitemap_path: "/tmp/sitemap.xml",
+        rule_names: "links"
+      },
+      configuration.received_arguments
+    )
+    assert_same result, reporter.result
+    assert_same result, returned_result
+  end
+  def test_validate_defaults_to_base_url_sitemap_when_not_configured
+    result = FakeResult.new(reported: true)
+    configuration = FakeConfiguration.new(result: result, base_url: "https://example.com", sitemap_path: nil)
+    reporter = FakeReporter.new
+    Crawlscope::Task.new(configuration: configuration, reporter: reporter).validate
+    assert_equal(
+      {
+        base_url: "https://example.com",
+        sitemap_path: "https://example.com/sitemap.xml",
+        rule_names: nil
+      },
+      configuration.received_arguments
+    )
+  end
+  def test_validate_prefers_local_sitemap_for_localhost
+    result = FakeResult.new(reported: true)
+    configuration = FakeConfiguration.new(result: result, base_url: "http://localhost:3000", sitemap_path: nil)
+    reporter = FakeReporter.new
+    tmp_dir = Dir.mktmpdir
+    sitemap_path = File.join(tmp_dir, "public", "sitemap.xml")
+    FileUtils.mkdir_p(File.dirname(sitemap_path))
+    File.write(sitemap_path, "<urlset></urlset>")
+    Dir.chdir(tmp_dir) do
+      Crawlscope::Task.new(configuration: configuration, reporter: reporter).validate
+    end
+    assert_equal(
+      {
+        base_url: "http://localhost:3000",
+        sitemap_path: sitemap_path,
+        rule_names: nil
+      },
+      configuration.received_arguments
+    )
+  ensure
+    FileUtils.rm_rf(tmp_dir) if tmp_dir
+  end
+  def test_validate_ldjson_uses_real_audit_and_writes_report
+    body = <<~HTML
+      <html>
+        <head>
+          <script type="application/ld+json">
+            {"@type":"WebSite","name":"Example","url":"https://example.com"}
+          </script>
+        </head>
+      </html>
+    HTML
+    page = Crawlscope::Page.new(
+      url: "https://example.com",
+      normalized_url: "https://example.com",
+      final_url: "https://example.com",
+      normalized_final_url: "https://example.com",
+      status: 200,
+      headers: {"content-type" => "text/html"},
+      body: body,
+      doc: Nokogiri::HTML(body)
+    )
+    browser = FakeBrowser.new(page: page)
+    output = StringIO.new
+    configuration = LdjsonConfiguration.new(output: output, browser: browser)
+    report_dir = Dir.mktmpdir
+    report_path = File.join(report_dir, "structured-data.json")
+    result = Crawlscope::Task.new(configuration: configuration).validate_ldjson(
+      urls: [page.url],
+      debug: true,
+      report_path: report_path,
+      summary: true
+    )
+    assert result.ok?
+    assert browser.closed
+    assert File.exist?(report_path)
+    assert_includes File.read(report_path), "https://example.com"
+    assert_includes output.string, "JavaScript mode enabled (Ferrum)"
+    assert_includes output.string, "Validating JSON-LD on 1 URL(s)"
+    assert_includes output.string, "All valid!"
+    assert_includes output.string, "All 1 URLs passed validation."
+  ensure
+    FileUtils.rm_rf(report_dir) if report_dir
+  end
+end

data/test/crawlscope/uniqueness_rule_test.rb ADDED Viewed

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+require "test_helper"
+class CrawlscopeUniquenessRuleTest < Minitest::Test
+  def test_reports_duplicate_title_description_and_content
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::Uniqueness.new
+    pages = [
+      page(url: "https://example.com/a"),
+      page(url: "https://example.com/b")
+    ]
+    rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+    assert_equal %i[duplicate_content_fingerprint duplicate_meta_description duplicate_title].sort, issues.to_a.map(&:code).sort
+  end
+  private
+  def page(url:)
+    repeated_text = ("Useful content " * 30).strip
+    body = <<~HTML
+      <html>
+        <head>
+          <title>Example Title</title>
+          <meta name="description" content="Example description">
+        </head>
+        <body>
+          <main>#{repeated_text}</main>
+        </body>
+      </html>
+    HTML
+    Crawlscope::Page.new(
+      url: url,
+      normalized_url: url,
+      final_url: url,
+      normalized_final_url: url,
+      status: 200,
+      headers: {"content-type" => "text/html"},
+      body: body,
+      doc: Nokogiri::HTML(body)
+    )
+  end
+end

data/test/test_helper.rb ADDED Viewed

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "simplecov"
+SimpleCov.start do
+  enable_coverage :branch
+  add_filter "/test/"
+end
+require "minitest/autorun"
+require "tmpdir"
+require "date"
+require "fileutils"
+require "nokogiri"
+require "webmock/minitest"
+require "crawlscope"
+if defined?(JSON::Validator)
+  JSON::Validator.use_multi_json = false
+end

metadata ADDED Viewed

@@ -0,0 +1,271 @@
+--- !ruby/object:Gem::Specification
+name: crawlscope
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Paulo Fidalgo
+- Ethos Link
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: faraday-follow_redirects
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: json-schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: A small Ruby gem for sitemap-driven SEO validation with structured issues,
+  configurable rules and schema registries, optional browser rendering, and Rails
+  rake task integration.
+email:
+- devel@ethos-link.com
+executables:
+- crawlscope
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- exe/crawlscope
+- lib/crawlscope.rb
+- lib/crawlscope/audit.rb
+- lib/crawlscope/browser.rb
+- lib/crawlscope/cli.rb
+- lib/crawlscope/configuration.rb
+- lib/crawlscope/crawler.rb
+- lib/crawlscope/http.rb
+- lib/crawlscope/issue.rb
+- lib/crawlscope/issue_collection.rb
+- lib/crawlscope/page.rb
+- lib/crawlscope/railtie.rb
+- lib/crawlscope/reporter.rb
+- lib/crawlscope/result.rb
+- lib/crawlscope/rule_registry.rb
+- lib/crawlscope/rules/links.rb
+- lib/crawlscope/rules/metadata.rb
+- lib/crawlscope/rules/structured_data.rb
+- lib/crawlscope/rules/uniqueness.rb
+- lib/crawlscope/schema_registry.rb
+- lib/crawlscope/sitemap.rb
+- lib/crawlscope/structured_data/audit.rb
+- lib/crawlscope/structured_data/document.rb
+- lib/crawlscope/structured_data/report.rb
+- lib/crawlscope/structured_data/reporter.rb
+- lib/crawlscope/structured_data/writer.rb
+- lib/crawlscope/task.rb
+- lib/crawlscope/url.rb
+- lib/crawlscope/version.rb
+- lib/tasks/crawlscope_tasks.rake
+- test/crawlscope/audit_test.rb
+- test/crawlscope/cli_test.rb
+- test/crawlscope/configuration_test.rb
+- test/crawlscope/links_rule_test.rb
+- test/crawlscope/loader_test.rb
+- test/crawlscope/reporter_test.rb
+- test/crawlscope/schema_registry_test.rb
+- test/crawlscope/sitemap_test.rb
+- test/crawlscope/structured_data_audit_test.rb
+- test/crawlscope/structured_data_document_test.rb
+- test/crawlscope/structured_data_report_test.rb
+- test/crawlscope/structured_data_reporter_test.rb
+- test/crawlscope/structured_data_rule_test.rb
+- test/crawlscope/structured_data_writer_test.rb
+- test/crawlscope/task_test.rb
+- test/crawlscope/uniqueness_rule_test.rb
+- test/test_helper.rb
+homepage: https://www.ethos-link.com/opensource/crawlscope
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://www.ethos-link.com/opensource/crawlscope
+  source_code_uri: https://github.com/ethos-link/crawlscope
+  bug_tracker_uri: https://github.com/ethos-link/crawlscope/issues
+  changelog_uri: https://github.com/ethos-link/crawlscope/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/ethos-link/crawlscope/blob/main/README.md
+  funding_uri: https://www.reviato.com/
+  github_repo: ssh://github.com/ethos-link/crawlscope
+  allowed_push_host: https://rubygems.org
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: Audit sitemap URLs for metadata, structured data, uniqueness, and links
+test_files: []