crawlscope 0.9.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1aeb91b01f3329f1d9550c47396ed4e100a8426977134fac5371db1324f344ae
-  data.tar.gz: f1c6c3a9db8d540c5e94b247e43bc9e632095b1d4265ee590e6a80bfa6cd1e69
+  metadata.gz: da58b612eb199f881ec2263858feea4bf93e11355ffce7b19921bb266451818e
+  data.tar.gz: bcfa91a8bd74cf5ae670dcc5a1b3e762ad0c6a74037cd82f92b700f8b3a2d7d0
 SHA512:
-  metadata.gz: c7f9e054435cbef172f7f6ee5b35019615d7ced832637e343c90895472cf2dec4c203ddbead1b1e455564ea67adce9663be21004ce73cf20778829cc67a625fe
-  data.tar.gz: 69d2b788ee3e646c5010ade5e9b0f543350a326c4a8189df76c8223d36b7464d95f322303391e24a8d53740693f5503bde974dd54a6b58ab18bc8fc9bfc5916e
+  metadata.gz: e3001de98212ddd2dca5f15feec4bf32eb9b3a975bef21c948c15c588c9a6d9f18bd130985fc3a6cfa78d1dfd34c2c5c4e944563ecd27be87cc49819475aed46
+  data.tar.gz: 784d47ba0dbf3effdbacb53d9211472e044b22da28c2d00e21bbcb8844aa2a1309eba072c0c631412b1dd7bad10f6713ad0adb107f2805df42310b1b31ad7991

data/CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.10.0] - 2026-11-06
+
+
+### Changed
+
+- keep checks local and adopt Gemkit 0.3.1
+
+
+
+
+### Fixed
+
+- distinguish filtered URLs in crawl results
+
+
+
 ## [0.9.2] - 2026-11-05
 
 

data/README.md

@@ -1,7 +1,7 @@
 # Crawlscope
 
 [![Gem Version](https://badge.fury.io/rb/crawlscope.svg)](https://badge.fury.io/rb/crawlscope)
-[![Ruby](https://github.com/ethos-link/crawlscope/actions/workflows/ruby.yml/badge.svg)](https://github.com/ethos-link/crawlscope/actions/workflows/ruby.yml)
+[![Commit messages](https://github.com/ethos-link/crawlscope/actions/workflows/commitlint.yml/badge.svg)](https://github.com/ethos-link/crawlscope/actions/workflows/commitlint.yml)
 
 `crawlscope` is a small Ruby gem for sitemap-driven SEO validation.
 

data/UPGRADE.md

@@ -1,8 +1,11 @@
 # Upgrade Guide
 
+Run `bundle exec rake verify` locally before pushing or preparing a release.
+GitHub Actions runs commitlint and release packaging/publication only.
+
 ## 0.9.1
 
-Development and release tooling uses Gemkit 0.3.0. Update the gem and workflow
+Development and release tooling uses Gemkit 0.3.1. Update the gem and workflow
 pins together; see the [Gemkit upgrade guide](https://github.com/ethos-link/ethos-gemkit/blob/main/UPGRADE.md).
 Runtime APIs are unchanged.
 
@@ -34,6 +37,18 @@ in the README; the shared workflow does not inherit another gem's permission.
 
 ## Next Release
 
+### Query links retain their own URL identity
+
+URL normalization now preserves query strings and still removes fragments.
+Resolvers, caches and issue consumers must keep the query string when identifying
+a URL. A filtered page no longer shares a resolution or inbound-link count with
+its unfiltered page.
+
+Nofollow warnings exclude self-links and destinations that explicitly declare
+`noindex` in their response headers or HTML. Other indexable destinations,
+including query URLs, still produce nofollow warnings. Broken and redirected
+links remain checked. No host configuration changes are required.
+
 ### Structured-data length errors include the measured size
 
 String-length schema errors now include an optional `length` field with the
@@ -141,4 +156,4 @@ Recommended migration:
 4. Set `FETCH_EXECUTOR=threaded` or pass `--fetch-executor threaded` for a
    conservative rollout or for explicit thread-pool execution.
 5. Browser rendering continues to use threaded execution by default because
-   async fetch execution is only supported with HTTP rendering.
+   async fetch execution is only supported with HTTP rendering.

data/lib/crawlscope/rules/links.rb

@@ -31,7 +31,7 @@ module Crawlscope
         links = extract_links(pages)
         validate_url_hygiene(urls, links, issues)
         resolved_links = resolve_links(links, issues)
-        validate_nofollow_outgoing_links(links, issues)
+        validate_nofollow_outgoing_links(resolved_links, issues)
         validate_http_internal_links(links, issues)
         validate_pages_with_no_outgoing_links(urls, pages, links, issues)
         validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
@@ -139,7 +139,9 @@ module Crawlscope
       end
 
       def validate_nofollow_outgoing_links(links, issues)
-        links.select { |link| link[:nofollow] }.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
+        links.select do |link|
+          link[:nofollow] && link[:source_url] != link[:final_url] && !target_for(link[:target_url]).noindex?
+        end.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
           target_urls = grouped_links.map { |link| link[:target_url] }.uniq
 
           issues.add(
@@ -301,17 +303,17 @@ module Crawlscope
       end
 
       def validate_inbound_counts(urls, pages, resolved_links, issues)
-        sitemap_paths = urls.each_with_object({}) do |url, memo|
+        sitemap_urls = urls.filter_map do |url|
           normalized_url = Url.normalize(url, base_url: @base_url)
           path = Url.path(normalized_url)
           next if path.nil?
           next if skip_internal_path?(path)
 
-          memo[path] = normalized_url
-        end
-        return if sitemap_paths.size < 2
+          normalized_url
+        end.to_set
+        return if sitemap_urls.size < 2
 
-        html_paths = pages.each_with_object(Set.new) do |page, result|
+        html_urls = pages.each_with_object(Set.new) do |page, result|
           next unless page.html?
 
           [page.normalized_url, page.normalized_final_url].compact.each do |url|
@@ -319,7 +321,7 @@ module Crawlscope
             next if path.nil?
             next if skip_internal_path?(path)
 
-            result << path
+            result << url
           end
         end
 
@@ -331,36 +333,37 @@ module Crawlscope
         nofollow_sources_by_target = Hash.new { |hash, key| hash[key] = [] }
 
         resolved_links.each do |link|
-          target_path = link[:final_path]
-          next unless sitemap_paths.key?(target_path)
-          next if link[:source_path] == target_path
+          target_url = link[:final_url]
+          next unless sitemap_urls.include?(target_url)
+          next if target_for(link[:target_url]).noindex?
+          next if link[:source_url] == target_url
 
-          inbound_anchor_counts[target_path] += 1
-          source_samples = sample_sources_by_target[target_path]
+          inbound_anchor_counts[target_url] += 1
+          source_samples = sample_sources_by_target[target_url]
           source_samples << link[:source_url] unless source_samples.include?(link[:source_url])
 
           if link[:nofollow]
-            nofollow_inbound_counts[target_path] += 1
-            nofollow_sources = nofollow_sources_by_target[target_path]
+            nofollow_inbound_counts[target_url] += 1
+            nofollow_sources = nofollow_sources_by_target[target_url]
             nofollow_sources << link[:source_url] unless nofollow_sources.include?(link[:source_url])
           else
-            dofollow_inbound_counts[target_path] += 1
-            dofollow_sources = dofollow_sources_by_target[target_path]
+            dofollow_inbound_counts[target_url] += 1
+            dofollow_sources = dofollow_sources_by_target[target_url]
             dofollow_sources << link[:source_url] unless dofollow_sources.include?(link[:source_url])
           end
         end
 
-        sitemap_paths.each do |path, target_url|
-          next unless html_paths.include?(path)
+        sitemap_urls.each do |target_url|
+          next unless html_urls.include?(target_url)
 
-          inbound_count = inbound_anchor_counts[path]
-          dofollow_count = dofollow_inbound_counts[path]
-          nofollow_count = nofollow_inbound_counts[path]
+          inbound_count = inbound_anchor_counts[target_url]
+          dofollow_count = dofollow_inbound_counts[target_url]
+          nofollow_count = nofollow_inbound_counts[target_url]
 
           report_orphan_page(target_url, issues) if inbound_count.zero?
 
           if inbound_count.positive? && inbound_count < MIN_INBOUND_ANCHOR_LINKS
-            source_samples = sample_sources_by_target[path]
+            source_samples = sample_sources_by_target[target_url]
 
             issues.add(
               code: :low_inbound_anchor_links,
@@ -372,9 +375,9 @@ module Crawlscope
             )
           end
 
-          report_low_dofollow_inlinks(target_url, path, dofollow_count, dofollow_sources_by_target, issues)
-          report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources_by_target[path], issues)
-          report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources_by_target[path], dofollow_sources_by_target[path], issues)
+          report_low_dofollow_inlinks(target_url, dofollow_count, dofollow_sources_by_target, issues)
+          report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources_by_target[target_url], issues)
+          report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources_by_target[target_url], dofollow_sources_by_target[target_url], issues)
         end
       end
 
@@ -475,11 +478,11 @@ module Crawlscope
         )
       end
 
-      def report_low_dofollow_inlinks(target_url, path, dofollow_count, sources_by_target, issues)
+      def report_low_dofollow_inlinks(target_url, dofollow_count, sources_by_target, issues)
         return if dofollow_count.zero?
         return if dofollow_count >= MIN_DOFOLLOW_INBOUND_LINKS
 
-        source_samples = sources_by_target[path]
+        source_samples = sources_by_target[target_url]
 
         issues.add(
           code: :low_dofollow_inlinks,
@@ -529,7 +532,7 @@ module Crawlscope
         end
         return if sitemap_pages.size < 2
 
-        dofollow_counts_by_path = dofollow_counts_by_final_path(resolved_links)
+        dofollow_counts_by_url = dofollow_counts_by_final_url(resolved_links)
         canonical_entries = canonical_entries_for(sitemap_pages)
         @resolved_targets_by_url.merge!(resolve_targets(canonical_entries.map { |entry| entry.fetch(:canonical_url) }))
 
@@ -537,7 +540,7 @@ module Crawlscope
           page = entry.fetch(:page)
           canonical_url = entry.fetch(:canonical_url)
           canonical_path = entry.fetch(:canonical_path)
-          if canonical_path && !root_path?(canonical_path) && dofollow_counts_by_path[canonical_path].zero?
+          if canonical_path && !root_path?(canonical_path) && dofollow_counts_by_url[canonical_url].zero?
             issues.add(
               code: :canonical_no_internal_inlinks,
               severity: :warning,
@@ -570,12 +573,13 @@ module Crawlscope
         end
       end
 
-      def dofollow_counts_by_final_path(resolved_links)
+      def dofollow_counts_by_final_url(resolved_links)
         resolved_links.each_with_object(Hash.new(0)) do |link, counts|
           next if link[:nofollow]
-          next if link[:source_path] == link[:final_path]
+          next if link[:source_url] == link[:final_url]
+          next if target_for(link[:target_url]).noindex?
 
-          counts[link[:final_path]] += 1
+          counts[link[:final_url]] += 1
         end
       end
 

data/lib/crawlscope/url.rb

@@ -18,7 +18,8 @@ module Crawlscope
       host = uri.host.to_s
       host = "#{host}:#{uri.port}" if uri.port && uri.port != uri.default_port
 
-      "#{uri.scheme}://#{host}#{normalized_path}"
+      query = uri.query ? "?#{uri.query}" : ""
+      "#{uri.scheme}://#{host}#{normalized_path}#{query}"
     rescue URI::InvalidURIError
       url.to_s
     end

data/lib/crawlscope/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlscope
-  VERSION = "0.9.2"
+  VERSION = "0.10.0"
 end

data/test/crawlscope/crawl_test.rb

@@ -42,6 +42,24 @@ class CrawlscopeCrawlTest < Minitest::Test
     FileUtils.rm_rf(@tmp_dir)
   end
 
+  def test_fetches_nofollow_query_targets_separately_from_cached_sitemap_pages
+    File.write(@sitemap_path, '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>https://example.com/pricing</loc></url></urlset>')
+    stub_request(:get, "https://example.com/pricing").to_return(
+      headers: {"Content-Type" => "text/html"}, body: '<a href="/pricing?sample=cafe#demo" rel="nofollow">Demo</a>'
+    )
+    filtered = stub_request(:get, "https://example.com/pricing?sample=cafe").to_return(
+      headers: {"Content-Type" => "text/html", "X-Robots-Tag" => "noindex, follow"}, body: "Demo"
+    )
+
+    result = Crawlscope::Crawl.new(
+      base_url: "https://example.com", sitemap_path: @sitemap_path,
+      rules: [Crawlscope::Rules::Links.new], schema_registry: Crawlscope::SchemaRegistry.default
+    ).call
+
+    assert_requested filtered
+    assert_empty result.issues.to_a
+  end
+
   def test_http_renderer_defaults_to_async_executor
     crawl = Crawlscope::Crawl.new(
       base_url: "https://example.com",

data/test/crawlscope/links_rule_test.rb

@@ -3,6 +3,54 @@
 require "test_helper"
 
 class CrawlscopeLinksRuleTest < Minitest::Test
+  def test_nofollow_controls_do_not_count_as_links_to_the_indexable_page
+    issues = Crawlscope::IssueCollection.new
+    requested = []
+    resolver = lambda do |url|
+      requested << url
+      {crawled: false, error: nil, final_url: url, status: 200, html: true,
+       doc: Nokogiri::HTML('<meta name="robots" content="noindex,follow">')}
+    end
+    pages = [
+      page(url: "https://example.com/guide", body: '<a href="/pricing">Pricing</a><a href="/pricing?sample=cafe#demo" rel="nofollow">Demo</a><a href="/guide" rel="nofollow">Reset</a>'),
+      page(url: "https://example.com/about", body: '<a href="/pricing">Pricing</a>'),
+      page(url: "https://example.com/pricing", body: '<a href="/guide">Guide</a><a href="/about">About</a>')
+    ]
+    Crawlscope::Rules::Links.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: context(resolver: resolver))
+
+    assert_includes requested, "https://example.com/pricing?sample=cafe"
+    refute_includes issues.map(&:code), :nofollow_internal_outlinks
+    refute_includes issues.map(&:code), :mixed_follow_internal_inlinks
+  end
+
+  def test_still_reports_nofollow_query_links_to_indexable_pages
+    issues = Crawlscope::IssueCollection.new
+    url = "https://example.com/catalog?page=2"
+    Crawlscope::Rules::Links.new.call(
+      urls: [], pages: [page(url: "https://example.com/guide", body: '<a href="/catalog?page=2" rel="nofollow">Next</a>')],
+      issues: issues, context: context(resolver: ->(target) { {crawled: false, final_url: target, status: 200, html: true} })
+    )
+
+    issue = issues.find { |item| item.code == :nofollow_internal_outlinks }
+    assert_equal [url], issue.details[:target_urls]
+  end
+
+  def test_query_variants_have_separate_inbound_link_counts
+    issues = Crawlscope::IssueCollection.new
+    pages = [
+      page(url: "https://example.com/guide", body: '<a href="/catalog?page=2">Next</a><a href="/catalog">Catalog</a>'),
+      page(url: "https://example.com/catalog", body: '<a href="/guide">Guide</a>'),
+      page(url: "https://example.com/catalog?page=2", body: '<a href="/guide">Guide</a>')
+    ]
+    Crawlscope::Rules::Links.new.call(
+      urls: pages.map(&:url), pages: pages, issues: issues,
+      context: context(resolver: ->(target) { {crawled: true, final_url: target, status: 200, html: true} })
+    )
+
+    counts = issues.select { |item| item.code == :low_dofollow_inlinks }.to_h { |item| [item.url, item.details[:dofollow_inbound_count]] }
+    assert_equal({"https://example.com/catalog" => 1, "https://example.com/catalog?page=2" => 1}, counts)
+  end
+
   def test_reports_broken_internal_links
     issues = Crawlscope::IssueCollection.new
     rule = Crawlscope::Rules::Links.new

data/test/crawlscope/url_test.rb

@@ -3,6 +3,10 @@
 require "test_helper"
 
 class CrawlscopeUrlTest < Minitest::Test
+  def test_normalize_preserves_query_strings_and_removes_fragments
+    assert_equal "https://example.com/reviews?rating=1", Crawlscope::Url.normalize("/reviews/?rating=1#results", base_url: "https://example.com")
+  end
+
   def test_normalize_resolves_relative_urls_and_removes_trailing_slash
     assert_equal "https://example.com/pricing", Crawlscope::Url.normalize("/pricing/", base_url: "https://example.com")
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.0
 platform: ruby
 authors:
 - Paulo Fidalgo