crawlscope 0.10.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/UPGRADE.md +76 -0
  4. data/docs/architecture.md +73 -0
  5. data/docs/rules.md +49 -0
  6. data/lib/crawlscope/configuration.rb +2 -1
  7. data/lib/crawlscope/crawl.rb +11 -79
  8. data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
  9. data/lib/crawlscope/crawling/target.rb +45 -0
  10. data/lib/crawlscope/crawling/target_resolver.rb +75 -0
  11. data/lib/crawlscope/documents/canonical.rb +22 -0
  12. data/lib/crawlscope/documents/robots.rb +57 -0
  13. data/lib/crawlscope/page.rb +4 -0
  14. data/lib/crawlscope/reporter.rb +6 -160
  15. data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
  16. data/lib/crawlscope/reporting/url_formatter.rb +29 -0
  17. data/lib/crawlscope/rule_registry.rb +2 -2
  18. data/lib/crawlscope/rules/content_quality.rb +18 -0
  19. data/lib/crawlscope/rules/indexability.rb +9 -14
  20. data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
  21. data/lib/crawlscope/rules/links/extractor.rb +100 -0
  22. data/lib/crawlscope/rules/links/graph.rb +39 -0
  23. data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
  24. data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
  25. data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
  26. data/lib/crawlscope/rules/links/scope.rb +23 -0
  27. data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
  28. data/lib/crawlscope/rules/links/url_checks.rb +54 -0
  29. data/lib/crawlscope/rules/links.rb +18 -600
  30. data/lib/crawlscope/rules/metadata.rb +2 -2
  31. data/lib/crawlscope/rules/structured_data.rb +26 -77
  32. data/lib/crawlscope/rules/uniqueness.rb +1 -1
  33. data/lib/crawlscope/schemas.rb +10 -2
  34. data/lib/crawlscope/server_timing/reporter.rb +2 -13
  35. data/lib/crawlscope/structured_data/audit.rb +17 -76
  36. data/lib/crawlscope/structured_data/check.rb +2 -0
  37. data/lib/crawlscope/structured_data/document.rb +14 -1
  38. data/lib/crawlscope/structured_data/entry.rb +31 -0
  39. data/lib/crawlscope/structured_data/outcome.rb +11 -0
  40. data/lib/crawlscope/structured_data/report.rb +6 -14
  41. data/lib/crawlscope/structured_data/reporter.rb +6 -11
  42. data/lib/crawlscope/structured_data/validator.rb +64 -0
  43. data/lib/crawlscope/version.rb +1 -1
  44. data/test/crawlscope/cli_test.rb +21 -0
  45. data/test/crawlscope/content_quality_rule_test.rb +51 -0
  46. data/test/crawlscope/links_rule_test.rb +16 -0
  47. data/test/crawlscope/reporter_test.rb +12 -0
  48. data/test/crawlscope/schema_registry_test.rb +22 -0
  49. data/test/crawlscope/structured_data_audit_test.rb +60 -0
  50. data/test/crawlscope/structured_data_report_test.rb +32 -0
  51. data/test/crawlscope/structured_data_rule_test.rb +11 -1
  52. data/test/crawlscope/structured_data_validator_test.rb +24 -0
  53. metadata +22 -1
@@ -3,6 +3,57 @@
3
3
  require "test_helper"
4
4
 
5
5
  class CrawlscopeContentQualityRuleTest < Minitest::Test
6
+ def test_reports_missing_image_alt_across_the_whole_page
7
+ issues = Crawlscope::IssueCollection.new
8
+ page = page_with(
9
+ main: '<picture><source srcset="/photo.webp"><img src="/photo.jpg"></picture><img src="/photo.jpg"><img loading="lazy">',
10
+ extra_markup: '<header><img src="/logo.svg"></header><footer><img src="/footer.svg"></footer>'
11
+ )
12
+
13
+ Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
14
+
15
+ matches = issues.to_a.select { |issue| issue.code == :missing_image_alt }
16
+ assert_equal 1, matches.size
17
+ issue = matches.first
18
+ assert_equal :warning, issue.severity
19
+ assert_equal :content_quality, issue.category
20
+ assert_equal page.url, issue.url
21
+ assert_equal "images missing alt attributes: 5", issue.message
22
+ assert_equal 5, issue.details[:count]
23
+ assert_equal ["/logo.svg", "/footer.svg", "/photo.jpg"], issue.details[:image_sources]
24
+ end
25
+
26
+ def test_allows_present_image_alt_attributes_including_decorative_images
27
+ issues = Crawlscope::IssueCollection.new
28
+ page = page_with(main: '<img src="/photo.jpg" alt="A mountain"><img src="/decoration.svg" alt=""><img src="/empty.svg" alt>')
29
+
30
+ Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
31
+
32
+ refute_includes issues.to_a.map(&:code), :missing_image_alt
33
+ end
34
+
35
+ def test_does_not_report_missing_image_alt_when_there_are_no_images
36
+ issues = Crawlscope::IssueCollection.new
37
+ page = page_with(main: "Text only")
38
+
39
+ Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
40
+
41
+ refute_includes issues.to_a.map(&:code), :missing_image_alt
42
+ end
43
+
44
+ def test_skips_non_html_pages
45
+ issues = Crawlscope::IssueCollection.new
46
+ page = Crawlscope::Page.new(
47
+ url: "https://example.com/file", normalized_url: "https://example.com/file",
48
+ final_url: "https://example.com/file", normalized_final_url: "https://example.com/file",
49
+ status: 200, headers: {}, body: "", doc: nil
50
+ )
51
+
52
+ Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
53
+
54
+ assert_empty issues.to_a
55
+ end
56
+
6
57
  def test_reports_thin_visible_text_and_low_html_text_ratio
7
58
  issues = Crawlscope::IssueCollection.new
8
59
  page = page_with(main: "Short page <div>#{"<span></span>" * 500}</div>")
@@ -3,6 +3,22 @@
3
3
  require "test_helper"
4
4
 
5
5
  class CrawlscopeLinksRuleTest < Minitest::Test
6
+ def test_reusing_a_rule_does_not_retain_the_previous_batch_resolver
7
+ rule = Crawlscope::Rules::Links.new
8
+ pages = [page(url: "https://example.com/guide", body: '<a href="/missing">Missing</a>')]
9
+ batch = ->(urls) { urls.to_h { |url| [url, {status: 200, html: true, final_url: url}] } }
10
+ first_issues = Crawlscope::IssueCollection.new
11
+ rule.call(urls: [], pages: pages, issues: first_issues, context: context.merge(resolve_targets: batch))
12
+ refute_includes first_issues.map(&:code), :broken_internal_link
13
+
14
+ second_issues = Crawlscope::IssueCollection.new
15
+ rule.call(urls: [], pages: pages, issues: second_issues, context: context)
16
+
17
+ broken = second_issues.find { |issue| issue.code == :broken_internal_link }
18
+ assert broken
19
+ assert_equal 404, broken.details[:status]
20
+ end
21
+
6
22
  def test_nofollow_controls_do_not_count_as_links_to_the_indexable_page
7
23
  issues = Crawlscope::IssueCollection.new
8
24
  requested = []
@@ -4,6 +4,18 @@ require "stringio"
4
4
  require "test_helper"
5
5
 
6
6
  class CrawlscopeReporterTest < Minitest::Test
7
+ def test_reports_images_missing_alt_with_each_image_source
8
+ output = report_issues(
9
+ {code: :missing_image_alt, category: :content_quality,
10
+ message: "images missing alt attributes: 2",
11
+ details: {count: 2, image_sources: ["https://example.com/photo.jpg", "https://cdn.example.com/logo.svg"]}}
12
+ )
13
+
14
+ assert_includes output, "Content quality: Images missing alt attributes (1 warning)"
15
+ assert_includes output, "/page images missing alt attributes: 2"
16
+ assert_includes output, " image: /photo.jpg\n image: https://cdn.example.com/logo.svg"
17
+ end
18
+
7
19
  def test_reports_ok_result
8
20
  io = StringIO.new
9
21
  result = Crawlscope::Result.new(
@@ -3,6 +3,28 @@
3
3
  require "test_helper"
4
4
 
5
5
  class CrawlscopeSchemaRegistryTest < Minitest::Test
6
+ def test_review_publication_accepts_dates_and_timestamps
7
+ ["2026-09-16", "2026-09-16T12:30:00Z"].each do |published|
8
+ review = {"@type" => "Review", "datePublished" => published, "reviewRating" => {"@type" => "Rating", "ratingValue" => 5}}
9
+ items = [review.merge("itemReviewed" => {"@type" => "SoftwareApplication", "name" => "Example"})]
10
+ items.concat(%w[SoftwareApplication WebApplication Product].map { |type| {"@type" => type, "name" => "Example", "review" => review} })
11
+
12
+ items.each do |item|
13
+ assert_empty Crawlscope::SchemaRegistry.default.validate(item), item.inspect
14
+ end
15
+ end
16
+ end
17
+
18
+ def test_review_publication_rejects_invalid_calendar_dates
19
+ ["2026-02-30", "2026-13-01", "not a date"].each do |published|
20
+ review = {"@type" => "Review", "datePublished" => published, "reviewRating" => {"@type" => "Rating", "ratingValue" => 5}}
21
+ [review.merge("itemReviewed" => {"@type" => "SoftwareApplication"}),
22
+ {"@type" => "SoftwareApplication", "name" => "Example", "review" => review}].each do |item|
23
+ assert Crawlscope::SchemaRegistry.default.validate(item).any? { |error| error.fetch(:field).end_with?("datePublished") }, item.inspect
24
+ end
25
+ end
26
+ end
27
+
6
28
  def test_registers_and_fetches_schema_by_type
7
29
  registry = Crawlscope::SchemaRegistry.default
8
30
  schema = {"type" => "object"}
@@ -3,6 +3,66 @@
3
3
  require "test_helper"
4
4
 
5
5
  class CrawlscopeStructuredDataAuditTest < Minitest::Test
6
+ def test_required_types_are_checked_by_sitemap_and_direct_audits
7
+ required_types = ->(url) { URI(url).path.start_with?("/guides/") ? [["Article", "BlogPosting", "NewsArticle"]] : [] }
8
+ cases = [
9
+ ["/guides/test", {"@type" => "WebPage"}, false],
10
+ ["/features/test", {"@type" => "WebPage"}, true],
11
+ ["/guides/test", {"@graph" => [{"@type" => "BlogPosting", "headline" => "A guide"}], "@type" => "WebPage"}, true],
12
+ ["/guides/test", {"@type" => ["WebPage", "Article"], "headline" => "A guide"}, true],
13
+ ["/guides/test", {"@type" => "ItemList", "itemListElement" => [{"@type" => "Article", "headline" => "Another guide"}]}, false]
14
+ ]
15
+
16
+ cases.each do |path, data, expected_ok|
17
+ page = html_page(url: "https://example.com#{path}", body: %(<script type="application/ld+json">#{JSON.generate(data)}</script>))
18
+ browser = FakeBrowser.new(page: page)
19
+ config = Crawlscope::Configuration.new
20
+ config.browser_factory = -> { -> { browser } }
21
+ config.renderer = :browser
22
+ config.output = StringIO.new
23
+ config.required_structured_data_types = required_types
24
+ outcome = Crawlscope::StructuredData::Check.new(configuration: config).call(urls: [page.url])
25
+ issues = Crawlscope::IssueCollection.new
26
+ config.rule_registry.rules_for([:structured_data]).first.call(urls: [page.url], pages: [page], issues: issues,
27
+ context: {schema_registry: config.schema_registry})
28
+
29
+ assert_equal expected_ok, outcome.ok?, data.inspect
30
+ assert_equal expected_ok ? [] : [:structured_data_missing_required_type], issues.map(&:code), data.inspect
31
+ unless expected_ok
32
+ assert_equal "page", outcome.entries.first.errors.first.fetch(:source)
33
+ assert_equal "Article or BlogPosting or NewsArticle", outcome.entries.first.errors.first.fetch(:type)
34
+ end
35
+ end
36
+ end
37
+
38
+ def test_sitemap_and_direct_audits_agree_on_structured_data_validity
39
+ cases = [
40
+ ["<h1>No markup</h1>", :missing_structured_data],
41
+ ['<script type="application/ld+json">{</script>', :structured_data_parse_error],
42
+ ['<script type="application/ld+json">{"name":"Untyped"}</script>', :structured_data_missing_type],
43
+ ['<script type="application/ld+json">{"@type":"WebPage","@graph":[{"name":"Untyped"}]}</script>', :structured_data_missing_type],
44
+ ['<script type="application/ld+json">{"@type":"Article"}</script>', :structured_data_schema_error],
45
+ ['<script type="application/ld+json">{"@type":"WebPage","name":"Example"}</script>', nil],
46
+ ['<div itemscope itemtype="https://schema.org/WebPage"><span itemprop="name">Example</span></div>', nil]
47
+ ]
48
+ registry = Crawlscope::SchemaRegistry.default
49
+ cases.each do |body, expected_code|
50
+ page = html_page(url: "https://example.com/page", body: body)
51
+ browser = FakeBrowser.new(page: page)
52
+ outcome = Crawlscope::StructuredData::Audit.new(browser_factory: -> { browser }, renderer: :browser,
53
+ schema_registry: registry, timeout_seconds: 20).call(urls: [page.url])
54
+ issues = Crawlscope::IssueCollection.new
55
+ Crawlscope::Rules::StructuredData.new.call(urls: [page.url], pages: [page], issues: issues, context: {schema_registry: registry})
56
+
57
+ assert_equal expected_code.nil?, outcome.ok?, body
58
+ assert_equal expected_code ? [expected_code] : [], issues.map(&:code), body
59
+ assert_equal outcome.ok?, Crawlscope::StructuredData::Report.new(outcome).all_valid?
60
+ schema_issue = issues.find { |issue| issue.code == :structured_data_schema_error }
61
+ assert_equal schema_issue.details[:errors], outcome.entries.first.errors.first[:errors] if schema_issue
62
+ assert browser.closed
63
+ end
64
+ end
65
+
6
66
  class FakeBrowser
7
67
  attr_reader :closed
8
68
 
@@ -3,6 +3,38 @@
3
3
  require "test_helper"
4
4
 
5
5
  class CrawlscopeStructuredDataReportTest < Minitest::Test
6
+ def test_outcome_and_report_agree_for_every_failure_kind_and_success_status
7
+ cases = [
8
+ {status: 200, structured_items: [], failure: :missing_data},
9
+ {status: 201, structured_items: [], failure: :missing_data},
10
+ {status: nil, structured_items: [], failure: :missing_data},
11
+ {status: 200, fetch_error: "Fetch failed", failure: :http},
12
+ {status: 404, fetch_error: "Not found", failure: :http},
13
+ {status: 201, errors: [{errors: [{field: "name", issue: "is required"}]}], failure: :validation},
14
+ {status: nil, errors: [{errors: [{field: "name", issue: "is required"}]}], failure: :validation},
15
+ {status: 200, failure: nil},
16
+ {status: 201, failure: nil},
17
+ {status: nil, failure: nil},
18
+ {status: 200, structured_items: [], skipped_reason: "non-html", failure: nil}
19
+ ]
20
+ cases.each do |attributes|
21
+ attributes = attributes.dup
22
+ failure = attributes.delete(:failure)
23
+ defaults = {url: "https://example.com/page", status: 200, structured_items: [{source: "json-ld", data: {"@type" => "WebPage"}}],
24
+ errors: [], fetch_error: nil, content_type: "text/html", skipped_reason: nil}
25
+ entry = Crawlscope::StructuredData::Audit::Page.new(**defaults.merge(attributes))
26
+ outcome = Crawlscope::StructuredData::Audit::Outcome.new(entries: [entry])
27
+ report = Crawlscope::StructuredData::Report.new(outcome)
28
+
29
+ assert_equal failure.nil?, outcome.ok?, attributes.inspect
30
+ assert_equal outcome.ok?, report.all_valid?, attributes.inspect
31
+ assert_equal failure ? 1 : 0, report.failure_count, attributes.inspect
32
+ assert_equal (failure == :http) ? [entry] : [], report.http_errors
33
+ assert_equal (failure == :missing_data) ? [entry] : [], report.missing_data
34
+ assert_equal (failure == :validation) ? [entry] : [], report.validation_errors
35
+ end
36
+ end
37
+
6
38
  def test_results_maps_validation_errors_and_skips
7
39
  result = Crawlscope::StructuredData::Audit::Outcome.new(
8
40
  entries: [
@@ -3,6 +3,16 @@
3
3
  require "test_helper"
4
4
 
5
5
  class CrawlscopeStructuredDataRuleTest < Minitest::Test
6
+ def test_does_not_assume_host_routes_require_job_posting_markup
7
+ issues = Crawlscope::IssueCollection.new
8
+ document = page(url: "https://example.com/careers/team", body: '<script type="application/ld+json">{"@type":"WebPage","name":"Our team"}</script>')
9
+
10
+ Crawlscope::Rules::StructuredData.new.call(urls: [document.url], pages: [document], issues: issues,
11
+ context: {schema_registry: Crawlscope::SchemaRegistry.default})
12
+
13
+ refute_includes issues.map(&:code), :missing_job_posting
14
+ end
15
+
6
16
  def test_reports_schema_errors_for_invalid_article_markup
7
17
  issues = Crawlscope::IssueCollection.new
8
18
  rule = Crawlscope::Rules::StructuredData.new
@@ -203,7 +213,7 @@ class CrawlscopeStructuredDataRuleTest < Minitest::Test
203
213
 
204
214
  def test_reports_missing_job_posting_for_career_detail_pages
205
215
  issues = Crawlscope::IssueCollection.new
206
- rule = Crawlscope::Rules::StructuredData.new
216
+ rule = Crawlscope::Rules::StructuredData.new(job_posting_path: %r{/careers/[^/]+/?\z})
207
217
  page = page(
208
218
  url: "https://example.com/careers/sales-partner",
209
219
  body: <<~HTML
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "test_helper"
4
+
5
+ class CrawlscopeStructuredDataValidatorTest < Minitest::Test
6
+ def test_requires_every_type_group
7
+ validator = Crawlscope::StructuredData::Validator.new(schema_registry: Crawlscope::SchemaRegistry.default)
8
+ html = '<script type="application/ld+json">{"@type":"WebPage","name":"Benchmark"}</script>'
9
+
10
+ result = validator.call(html: html, required_types: ["WebPage", "Dataset"])
11
+
12
+ assert_equal [:structured_data_missing_required_type], result.findings.map(&:code)
13
+ assert_equal ["Dataset"], result.findings.first.errors.first.fetch(:expected_types)
14
+ end
15
+
16
+ def test_microdata_satisfies_type_requirements
17
+ validator = Crawlscope::StructuredData::Validator.new(schema_registry: Crawlscope::SchemaRegistry.default)
18
+ html = '<article itemscope itemtype="https://schema.org/Article"><h1 itemprop="headline">A guide</h1></article>'
19
+
20
+ result = validator.call(html: html, required_types: [["Article", "BlogPosting", "NewsArticle"]])
21
+
22
+ assert_empty result.findings
23
+ end
24
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crawlscope
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.1
4
+ version: 0.10.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Paulo Fidalgo
@@ -178,6 +178,7 @@ files:
178
178
  - LICENSE.txt
179
179
  - README.md
180
180
  - UPGRADE.md
181
+ - docs/architecture.md
181
182
  - docs/rails.md
182
183
  - docs/rules.md
183
184
  - docs/server-timing.md
@@ -189,7 +190,12 @@ files:
189
190
  - lib/crawlscope/context.rb
190
191
  - lib/crawlscope/crawl.rb
191
192
  - lib/crawlscope/crawler.rb
193
+ - lib/crawlscope/crawling/fetcher_factory.rb
194
+ - lib/crawlscope/crawling/target.rb
195
+ - lib/crawlscope/crawling/target_resolver.rb
192
196
  - lib/crawlscope/document_text.rb
197
+ - lib/crawlscope/documents/canonical.rb
198
+ - lib/crawlscope/documents/robots.rb
193
199
  - lib/crawlscope/fetch_executor.rb
194
200
  - lib/crawlscope/fetch_executor/async.rb
195
201
  - lib/crawlscope/fetch_executor/threaded.rb
@@ -201,12 +207,23 @@ files:
201
207
  - lib/crawlscope/railtie.rb
202
208
  - lib/crawlscope/rake_tasks.rb
203
209
  - lib/crawlscope/reporter.rb
210
+ - lib/crawlscope/reporting/issue_presenter.rb
211
+ - lib/crawlscope/reporting/url_formatter.rb
204
212
  - lib/crawlscope/request_headers.rb
205
213
  - lib/crawlscope/result.rb
206
214
  - lib/crawlscope/rule_registry.rb
207
215
  - lib/crawlscope/rules/content_quality.rb
208
216
  - lib/crawlscope/rules/indexability.rb
209
217
  - lib/crawlscope/rules/links.rb
218
+ - lib/crawlscope/rules/links/canonical_checks.rb
219
+ - lib/crawlscope/rules/links/extractor.rb
220
+ - lib/crawlscope/rules/links/graph.rb
221
+ - lib/crawlscope/rules/links/inbound_checks.rb
222
+ - lib/crawlscope/rules/links/outgoing_checks.rb
223
+ - lib/crawlscope/rules/links/resolution_check.rb
224
+ - lib/crawlscope/rules/links/scope.rb
225
+ - lib/crawlscope/rules/links/target_lookup.rb
226
+ - lib/crawlscope/rules/links/url_checks.rb
210
227
  - lib/crawlscope/rules/metadata.rb
211
228
  - lib/crawlscope/rules/structured_data.rb
212
229
  - lib/crawlscope/rules/uniqueness.rb
@@ -220,9 +237,12 @@ files:
220
237
  - lib/crawlscope/structured_data/audit.rb
221
238
  - lib/crawlscope/structured_data/check.rb
222
239
  - lib/crawlscope/structured_data/document.rb
240
+ - lib/crawlscope/structured_data/entry.rb
223
241
  - lib/crawlscope/structured_data/error_formatter.rb
242
+ - lib/crawlscope/structured_data/outcome.rb
224
243
  - lib/crawlscope/structured_data/report.rb
225
244
  - lib/crawlscope/structured_data/reporter.rb
245
+ - lib/crawlscope/structured_data/validator.rb
226
246
  - lib/crawlscope/structured_data/writer.rb
227
247
  - lib/crawlscope/tasks.rb
228
248
  - lib/crawlscope/url.rb
@@ -258,6 +278,7 @@ files:
258
278
  - test/crawlscope/structured_data_report_test.rb
259
279
  - test/crawlscope/structured_data_reporter_test.rb
260
280
  - test/crawlscope/structured_data_rule_test.rb
281
+ - test/crawlscope/structured_data_validator_test.rb
261
282
  - test/crawlscope/structured_data_writer_test.rb
262
283
  - test/crawlscope/uniqueness_rule_test.rb
263
284
  - test/crawlscope/url_test.rb