RubyGems - crawlscope - Versions diffs - 0.3.0 → 0.5.0 - Mend

crawlscope 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (32) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +73 -0
data/README.md +50 -7
data/lib/crawlscope/cli.rb +4 -1
data/lib/crawlscope/crawl.rb +2 -0
data/lib/crawlscope/document_text.rb +40 -0
data/lib/crawlscope/rake_tasks.rb +27 -12
data/lib/crawlscope/reporter.rb +20 -5
data/lib/crawlscope/rule_registry.rb +3 -1
data/lib/crawlscope/rules/content_quality.rb +99 -0
data/lib/crawlscope/rules/indexability.rb +179 -0
data/lib/crawlscope/rules/links.rb +312 -9
data/lib/crawlscope/rules/metadata.rb +61 -6
data/lib/crawlscope/rules/structured_data.rb +31 -0
data/lib/crawlscope/rules/uniqueness.rb +98 -4
data/lib/crawlscope/sitemap.rb +9 -1
data/lib/crawlscope/version.rb +1 -1
data/lib/tasks/crawlscope_tasks.rake +28 -18
data/test/crawlscope/cli_test.rb +1 -0
data/test/crawlscope/configuration_test.rb +8 -1
data/test/crawlscope/content_quality_rule_test.rb +68 -0
data/test/crawlscope/crawl_test.rb +37 -1
data/test/crawlscope/indexability_rule_test.rb +129 -0
data/test/crawlscope/links_rule_test.rb +148 -3
data/test/crawlscope/metadata_rule_test.rb +36 -0
data/test/crawlscope/rake_tasks_test.rb +70 -0
data/test/crawlscope/reporter_test.rb +7 -3
data/test/crawlscope/sitemap_test.rb +24 -0
data/test/crawlscope/structured_data_rule_test.rb +56 -0
data/test/crawlscope/uniqueness_rule_test.rb +59 -3
data/test/release_task_test.rb +86 -0
metadata +9 -2

data/test/crawlscope/reporter_test.rb CHANGED Viewed

@@ -23,7 +23,7 @@ class CrawlscopeReporterTest < Minitest::Test
     refute_includes output, "Status: FAILED"
   end
-  def test_reports_failed_result_with_severity_counts
+  def test_reports_failed_result_with_grouped_counts_and_offenses
     io = StringIO.new
     issues = Crawlscope::IssueCollection.new
     issues.add(code: :missing_title, severity: :warning, category: :metadata, url: "https://example.com/a", message: "missing <title>", details: {})
@@ -42,9 +42,13 @@ class CrawlscopeReporterTest < Minitest::Test
     assert_includes output, "Status: FAILED"
     assert_includes output, "Issues: 2"
+    assert_includes output, "Severity:"
     assert_includes output, "notice: 1"
     assert_includes output, "warning: 1"
-    assert_includes output, "- [warning] https://example.com/a missing <title>"
-    assert_includes output, "- [notice] https://example.com/b broken internal link"
+    assert_includes output, "Category:"
+    assert_includes output, "links: 1"
+    assert_includes output, "metadata: 1"
+    assert_includes output, "  - [warning] missing_title https://example.com/a missing <title>"
+    assert_includes output, "  - [notice] broken_internal_link https://example.com/b broken internal link"
   end
 end

data/test/crawlscope/sitemap_test.rb CHANGED Viewed

@@ -49,6 +49,30 @@ class CrawlscopeSitemapTest < Minitest::Test
     assert_equal ["https://www.example.com/features/reviews"], parser.urls(base_url: "https://www.example.com")
   end
+  def test_remote_sitemap_http_error_is_explicit
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(status: 500, body: "<html><body>Error</body></html>")
+    parser = Crawlscope::Sitemap.new(path: "https://www.example.com/sitemap.xml")
+    error = assert_raises(Crawlscope::ValidationError) do
+      parser.urls(base_url: "https://www.example.com")
+    end
+    assert_equal "Sitemap https://www.example.com/sitemap.xml returned HTTP 500", error.message
+  end
+  def test_invalid_sitemap_root_is_explicit
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(status: 200, body: "<html><body>Error</body></html>")
+    parser = Crawlscope::Sitemap.new(path: "https://www.example.com/sitemap.xml")
+    error = assert_raises(Crawlscope::ValidationError) do
+      parser.urls(base_url: "https://www.example.com")
+    end
+    assert_equal 'Sitemap https://www.example.com/sitemap.xml has unexpected root "html"', error.message
+  end
   def test_rebases_remote_sitemap_index_children_to_base_url
     stub_request(:get, "http://localhost:3000/sitemap.xml")
       .to_return(

data/test/crawlscope/structured_data_rule_test.rb CHANGED Viewed

@@ -79,6 +79,62 @@ class CrawlscopeStructuredDataRuleTest < Minitest::Test
     assert_equal ["json-ld", "microdata"], issues.to_a.first.details[:expected_sources]
   end
+  def test_reports_structured_data_missing_type
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::StructuredData.new
+    page = page(
+      url: "https://example.com/articles/test",
+      body: <<~HTML
+        <html>
+          <head>
+            <script type="application/ld+json">
+              {"@context":"https://schema.org","headline":"Untyped article"}
+            </script>
+          </head>
+          <body><h1>Article</h1></body>
+        </html>
+      HTML
+    )
+    rule.call(
+      urls: [page.url],
+      pages: [page],
+      issues: issues,
+      context: {schema_registry: Crawlscope::SchemaRegistry.default}
+    )
+    assert_includes issues.to_a.map(&:code), :structured_data_missing_type
+  end
+  def test_reports_graph_entries_missing_type
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::StructuredData.new
+    page = page(
+      url: "https://example.com/articles/test",
+      body: <<~HTML
+        <html>
+          <head>
+            <script type="application/ld+json">
+              {"@context":"https://schema.org","@type":"WebPage","@graph":[{"name":"Untyped node"}]}
+            </script>
+          </head>
+          <body><h1>Article</h1></body>
+        </html>
+      HTML
+    )
+    rule.call(
+      urls: [page.url],
+      pages: [page],
+      issues: issues,
+      context: {schema_registry: Crawlscope::SchemaRegistry.default}
+    )
+    issue = issues.to_a.find { |item| item.code == :structured_data_missing_type }
+    assert issue
+    assert_equal ["$.@graph[0]"], issue.details[:paths]
+  end
   def test_validates_job_posting_markup
     issues = Crawlscope::IssueCollection.new
     rule = Crawlscope::Rules::StructuredData.new

data/test/crawlscope/uniqueness_rule_test.rb CHANGED Viewed

@@ -13,18 +13,74 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
     rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
-    assert_equal %i[duplicate_content_fingerprint duplicate_meta_description duplicate_title].sort, issues.to_a.map(&:code).sort
+    assert_equal %i[duplicate_content_fingerprint duplicate_meta_description duplicate_pages_without_canonical duplicate_title].sort, issues.to_a.map(&:code).sort
+  end
+  def test_allows_duplicate_pages_when_canonicals_are_present
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::Uniqueness.new
+    pages = [
+      page(url: "https://example.com/a", canonical: "https://example.com/a"),
+      page(url: "https://example.com/b", canonical: "https://example.com/a")
+    ]
+    rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+    refute_includes issues.to_a.map(&:code), :duplicate_pages_without_canonical
+  end
+  def test_reports_near_duplicate_content
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::Uniqueness.new
+    pages = [
+      page(url: "https://example.com/a", content: near_duplicate_content("reliable")),
+      page(url: "https://example.com/b", content: near_duplicate_content("dependable"))
+    ]
+    rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+    issue = issues.to_a.find { |item| item.code == :near_duplicate_content }
+    assert issue
+    assert_operator issue.details[:similarity], :>=, issue.details[:threshold]
+  end
+  def test_skips_near_duplicate_scan_when_page_count_exceeds_limit
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::Uniqueness.new(max_near_duplicate_pages: 1)
+    pages = [
+      page(url: "https://example.com/a", content: near_duplicate_content("reliable")),
+      page(url: "https://example.com/b", content: near_duplicate_content("dependable"))
+    ]
+    rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+    skip_issue = issues.to_a.find { |item| item.code == :near_duplicate_scan_skipped }
+    refute issues.to_a.any? { |item| item.code == :near_duplicate_content }
+    assert_equal :warning, skip_issue.severity
+    assert_equal({max_pages: 1, page_count: 2}, skip_issue.details)
   end
   private
-  def page(url:)
-    repeated_text = ("Useful content " * 30).strip
+  def near_duplicate_content(adjective)
+    <<~TEXT.gsub(/\s+/, " ").strip
+      This page summarizes practical hotel review patterns for operators who need #{adjective}
+      service insights across locations. It compares recurring comments about staff, rooms,
+      cleanliness, check-in, breakfast, parking, and amenities so teams can prioritize fixes.
+      The analysis highlights repeat themes, explains why guests mention them, and keeps the
+      wording focused on decisions that improve daily operations.
+    TEXT
+  end
+  def page(url:, content: nil, canonical: nil)
+    repeated_text = content || ("Useful content " * 30).strip
+    canonical_tag = canonical ? %(<link rel="canonical" href="#{canonical}">) : ""
     body = <<~HTML
       <html>
         <head>
           <title>Example Title</title>
           <meta name="description" content="Example description">
+          #{canonical_tag}
         </head>
         <body>
           <main>#{repeated_text}</main>

data/test/release_task_test.rb ADDED Viewed

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+require "test_helper"
+require "rake"
+unless respond_to?(:release_version, true)
+  load File.expand_path("../Rakefile", __dir__)
+end
+class ReleaseTaskTest < Minitest::Test
+  def test_release_version_increments_patch_from_current_version
+    major, minor, patch = Crawlscope::VERSION.split(".").map(&:to_i)
+    assert_equal "#{major}.#{minor}.#{patch + 1}", release_version("patch")
+  end
+  def test_release_version_accepts_explicit_semantic_version
+    assert_equal "0.3.0", release_version("0.3.0")
+  end
+  def test_validate_release_version_rejects_current_version
+    error = assert_raises(ArgumentError) do
+      validate_release_version!("0.2.7", "0.2.7")
+    end
+    assert_equal(
+      "Release version 0.2.7 must be newer than current version 0.2.7.",
+      error.message
+    )
+  end
+  def test_validate_release_version_rejects_existing_local_tag
+    @local_release_tag_exists = true
+    @remote_release_tag_exists = false
+    error = assert_raises(ArgumentError) do
+      validate_release_version!("0.2.8", "0.2.7")
+    end
+    assert_equal "Release tag v0.2.8 already exists locally.", error.message
+  end
+  def test_validate_release_version_rejects_existing_remote_tag
+    @local_release_tag_exists = false
+    @remote_release_tag_exists = true
+    error = assert_raises(ArgumentError) do
+      validate_release_version!("0.2.8", "0.2.7")
+    end
+    assert_equal "Release tag v0.2.8 already exists on origin.", error.message
+  end
+  def test_remote_release_tag_command_asks_git_to_fail_when_no_tag_matches
+    assert_equal(
+      "git ls-remote --exit-code --tags origin refs/tags/v0.2.8",
+      remote_release_tag_command("v0.2.8")
+    )
+  end
+  def test_changelog_command_prepends_the_next_release
+    assert_equal(
+      [
+        "git-cliff",
+        "-c",
+        "cliff.toml",
+        "--unreleased",
+        "--tag",
+        "v0.2.8",
+        "--prepend",
+        "CHANGELOG.md"
+      ],
+      changelog_command("0.2.8")
+    )
+  end
+  private
+  def local_release_tag_exists?(_tag)
+    @local_release_tag_exists || false
+  end
+  def remote_release_tag_exists?(_tag)
+    @remote_release_tag_exists || false
+  end
+end

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -199,6 +199,7 @@ files:
 - lib/crawlscope/context.rb
 - lib/crawlscope/crawl.rb
 - lib/crawlscope/crawler.rb
+- lib/crawlscope/document_text.rb
 - lib/crawlscope/http.rb
 - lib/crawlscope/issue.rb
 - lib/crawlscope/issue_collection.rb
@@ -208,6 +209,8 @@ files:
 - lib/crawlscope/reporter.rb
 - lib/crawlscope/result.rb
 - lib/crawlscope/rule_registry.rb
+- lib/crawlscope/rules/content_quality.rb
+- lib/crawlscope/rules/indexability.rb
 - lib/crawlscope/rules/links.rb
 - lib/crawlscope/rules/metadata.rb
 - lib/crawlscope/rules/structured_data.rb
@@ -228,12 +231,15 @@ files:
 - test/crawlscope/browser_test.rb
 - test/crawlscope/cli_test.rb
 - test/crawlscope/configuration_test.rb
+- test/crawlscope/content_quality_rule_test.rb
 - test/crawlscope/crawl_test.rb
 - test/crawlscope/crawler_test.rb
 - test/crawlscope/http_test.rb
+- test/crawlscope/indexability_rule_test.rb
 - test/crawlscope/links_rule_test.rb
 - test/crawlscope/loader_test.rb
 - test/crawlscope/metadata_rule_test.rb
+- test/crawlscope/rake_tasks_test.rb
 - test/crawlscope/reporter_test.rb
 - test/crawlscope/rule_registry_test.rb
 - test/crawlscope/run_test.rb
@@ -247,6 +253,7 @@ files:
 - test/crawlscope/structured_data_writer_test.rb
 - test/crawlscope/uniqueness_rule_test.rb
 - test/crawlscope/url_test.rb
+- test/release_task_test.rb
 - test/test_helper.rb
 homepage: https://www.ethos-link.com/opensource/crawlscope
 licenses:
@@ -275,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Audit sitemap URLs for metadata, structured data, uniqueness, and links
 test_files: []