crawlscope 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/UPGRADE.md +76 -0
- data/docs/architecture.md +73 -0
- data/docs/rules.md +49 -0
- data/lib/crawlscope/configuration.rb +2 -1
- data/lib/crawlscope/crawl.rb +11 -79
- data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
- data/lib/crawlscope/crawling/target.rb +45 -0
- data/lib/crawlscope/crawling/target_resolver.rb +75 -0
- data/lib/crawlscope/documents/canonical.rb +22 -0
- data/lib/crawlscope/documents/robots.rb +57 -0
- data/lib/crawlscope/page.rb +4 -0
- data/lib/crawlscope/reporter.rb +6 -160
- data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
- data/lib/crawlscope/reporting/url_formatter.rb +29 -0
- data/lib/crawlscope/rule_registry.rb +2 -2
- data/lib/crawlscope/rules/content_quality.rb +18 -0
- data/lib/crawlscope/rules/indexability.rb +9 -14
- data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
- data/lib/crawlscope/rules/links/extractor.rb +100 -0
- data/lib/crawlscope/rules/links/graph.rb +39 -0
- data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
- data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
- data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
- data/lib/crawlscope/rules/links/scope.rb +23 -0
- data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
- data/lib/crawlscope/rules/links/url_checks.rb +54 -0
- data/lib/crawlscope/rules/links.rb +18 -600
- data/lib/crawlscope/rules/metadata.rb +2 -2
- data/lib/crawlscope/rules/structured_data.rb +26 -77
- data/lib/crawlscope/rules/uniqueness.rb +1 -1
- data/lib/crawlscope/schemas.rb +10 -2
- data/lib/crawlscope/server_timing/reporter.rb +2 -13
- data/lib/crawlscope/structured_data/audit.rb +17 -76
- data/lib/crawlscope/structured_data/check.rb +2 -0
- data/lib/crawlscope/structured_data/document.rb +14 -1
- data/lib/crawlscope/structured_data/entry.rb +31 -0
- data/lib/crawlscope/structured_data/outcome.rb +11 -0
- data/lib/crawlscope/structured_data/report.rb +6 -14
- data/lib/crawlscope/structured_data/reporter.rb +6 -11
- data/lib/crawlscope/structured_data/validator.rb +64 -0
- data/lib/crawlscope/version.rb +1 -1
- data/test/crawlscope/cli_test.rb +21 -0
- data/test/crawlscope/content_quality_rule_test.rb +51 -0
- data/test/crawlscope/links_rule_test.rb +16 -0
- data/test/crawlscope/reporter_test.rb +12 -0
- data/test/crawlscope/schema_registry_test.rb +22 -0
- data/test/crawlscope/structured_data_audit_test.rb +60 -0
- data/test/crawlscope/structured_data_report_test.rb +32 -0
- data/test/crawlscope/structured_data_rule_test.rb +11 -1
- data/test/crawlscope/structured_data_validator_test.rb +24 -0
- metadata +22 -1
|
@@ -3,6 +3,57 @@
|
|
|
3
3
|
require "test_helper"
|
|
4
4
|
|
|
5
5
|
class CrawlscopeContentQualityRuleTest < Minitest::Test
|
|
6
|
+
def test_reports_missing_image_alt_across_the_whole_page
|
|
7
|
+
issues = Crawlscope::IssueCollection.new
|
|
8
|
+
page = page_with(
|
|
9
|
+
main: '<picture><source srcset="/photo.webp"><img src="/photo.jpg"></picture><img src="/photo.jpg"><img loading="lazy">',
|
|
10
|
+
extra_markup: '<header><img src="/logo.svg"></header><footer><img src="/footer.svg"></footer>'
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
|
|
14
|
+
|
|
15
|
+
matches = issues.to_a.select { |issue| issue.code == :missing_image_alt }
|
|
16
|
+
assert_equal 1, matches.size
|
|
17
|
+
issue = matches.first
|
|
18
|
+
assert_equal :warning, issue.severity
|
|
19
|
+
assert_equal :content_quality, issue.category
|
|
20
|
+
assert_equal page.url, issue.url
|
|
21
|
+
assert_equal "images missing alt attributes: 5", issue.message
|
|
22
|
+
assert_equal 5, issue.details[:count]
|
|
23
|
+
assert_equal ["/logo.svg", "/footer.svg", "/photo.jpg"], issue.details[:image_sources]
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def test_allows_present_image_alt_attributes_including_decorative_images
|
|
27
|
+
issues = Crawlscope::IssueCollection.new
|
|
28
|
+
page = page_with(main: '<img src="/photo.jpg" alt="A mountain"><img src="/decoration.svg" alt=""><img src="/empty.svg" alt>')
|
|
29
|
+
|
|
30
|
+
Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
|
|
31
|
+
|
|
32
|
+
refute_includes issues.to_a.map(&:code), :missing_image_alt
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def test_does_not_report_missing_image_alt_when_there_are_no_images
|
|
36
|
+
issues = Crawlscope::IssueCollection.new
|
|
37
|
+
page = page_with(main: "Text only")
|
|
38
|
+
|
|
39
|
+
Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
|
|
40
|
+
|
|
41
|
+
refute_includes issues.to_a.map(&:code), :missing_image_alt
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def test_skips_non_html_pages
|
|
45
|
+
issues = Crawlscope::IssueCollection.new
|
|
46
|
+
page = Crawlscope::Page.new(
|
|
47
|
+
url: "https://example.com/file", normalized_url: "https://example.com/file",
|
|
48
|
+
final_url: "https://example.com/file", normalized_final_url: "https://example.com/file",
|
|
49
|
+
status: 200, headers: {}, body: "", doc: nil
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
|
|
53
|
+
|
|
54
|
+
assert_empty issues.to_a
|
|
55
|
+
end
|
|
56
|
+
|
|
6
57
|
def test_reports_thin_visible_text_and_low_html_text_ratio
|
|
7
58
|
issues = Crawlscope::IssueCollection.new
|
|
8
59
|
page = page_with(main: "Short page <div>#{"<span></span>" * 500}</div>")
|
|
@@ -3,6 +3,22 @@
|
|
|
3
3
|
require "test_helper"
|
|
4
4
|
|
|
5
5
|
class CrawlscopeLinksRuleTest < Minitest::Test
|
|
6
|
+
def test_reusing_a_rule_does_not_retain_the_previous_batch_resolver
|
|
7
|
+
rule = Crawlscope::Rules::Links.new
|
|
8
|
+
pages = [page(url: "https://example.com/guide", body: '<a href="/missing">Missing</a>')]
|
|
9
|
+
batch = ->(urls) { urls.to_h { |url| [url, {status: 200, html: true, final_url: url}] } }
|
|
10
|
+
first_issues = Crawlscope::IssueCollection.new
|
|
11
|
+
rule.call(urls: [], pages: pages, issues: first_issues, context: context.merge(resolve_targets: batch))
|
|
12
|
+
refute_includes first_issues.map(&:code), :broken_internal_link
|
|
13
|
+
|
|
14
|
+
second_issues = Crawlscope::IssueCollection.new
|
|
15
|
+
rule.call(urls: [], pages: pages, issues: second_issues, context: context)
|
|
16
|
+
|
|
17
|
+
broken = second_issues.find { |issue| issue.code == :broken_internal_link }
|
|
18
|
+
assert broken
|
|
19
|
+
assert_equal 404, broken.details[:status]
|
|
20
|
+
end
|
|
21
|
+
|
|
6
22
|
def test_nofollow_controls_do_not_count_as_links_to_the_indexable_page
|
|
7
23
|
issues = Crawlscope::IssueCollection.new
|
|
8
24
|
requested = []
|
|
@@ -4,6 +4,18 @@ require "stringio"
|
|
|
4
4
|
require "test_helper"
|
|
5
5
|
|
|
6
6
|
class CrawlscopeReporterTest < Minitest::Test
|
|
7
|
+
def test_reports_images_missing_alt_with_each_image_source
|
|
8
|
+
output = report_issues(
|
|
9
|
+
{code: :missing_image_alt, category: :content_quality,
|
|
10
|
+
message: "images missing alt attributes: 2",
|
|
11
|
+
details: {count: 2, image_sources: ["https://example.com/photo.jpg", "https://cdn.example.com/logo.svg"]}}
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
assert_includes output, "Content quality: Images missing alt attributes (1 warning)"
|
|
15
|
+
assert_includes output, "/page images missing alt attributes: 2"
|
|
16
|
+
assert_includes output, " image: /photo.jpg\n image: https://cdn.example.com/logo.svg"
|
|
17
|
+
end
|
|
18
|
+
|
|
7
19
|
def test_reports_ok_result
|
|
8
20
|
io = StringIO.new
|
|
9
21
|
result = Crawlscope::Result.new(
|
|
@@ -3,6 +3,28 @@
|
|
|
3
3
|
require "test_helper"
|
|
4
4
|
|
|
5
5
|
class CrawlscopeSchemaRegistryTest < Minitest::Test
|
|
6
|
+
def test_review_publication_accepts_dates_and_timestamps
|
|
7
|
+
["2026-09-16", "2026-09-16T12:30:00Z"].each do |published|
|
|
8
|
+
review = {"@type" => "Review", "datePublished" => published, "reviewRating" => {"@type" => "Rating", "ratingValue" => 5}}
|
|
9
|
+
items = [review.merge("itemReviewed" => {"@type" => "SoftwareApplication", "name" => "Example"})]
|
|
10
|
+
items.concat(%w[SoftwareApplication WebApplication Product].map { |type| {"@type" => type, "name" => "Example", "review" => review} })
|
|
11
|
+
|
|
12
|
+
items.each do |item|
|
|
13
|
+
assert_empty Crawlscope::SchemaRegistry.default.validate(item), item.inspect
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def test_review_publication_rejects_invalid_calendar_dates
|
|
19
|
+
["2026-02-30", "2026-13-01", "not a date"].each do |published|
|
|
20
|
+
review = {"@type" => "Review", "datePublished" => published, "reviewRating" => {"@type" => "Rating", "ratingValue" => 5}}
|
|
21
|
+
[review.merge("itemReviewed" => {"@type" => "SoftwareApplication"}),
|
|
22
|
+
{"@type" => "SoftwareApplication", "name" => "Example", "review" => review}].each do |item|
|
|
23
|
+
assert Crawlscope::SchemaRegistry.default.validate(item).any? { |error| error.fetch(:field).end_with?("datePublished") }, item.inspect
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
6
28
|
def test_registers_and_fetches_schema_by_type
|
|
7
29
|
registry = Crawlscope::SchemaRegistry.default
|
|
8
30
|
schema = {"type" => "object"}
|
|
@@ -3,6 +3,66 @@
|
|
|
3
3
|
require "test_helper"
|
|
4
4
|
|
|
5
5
|
class CrawlscopeStructuredDataAuditTest < Minitest::Test
|
|
6
|
+
def test_required_types_are_checked_by_sitemap_and_direct_audits
|
|
7
|
+
required_types = ->(url) { URI(url).path.start_with?("/guides/") ? [["Article", "BlogPosting", "NewsArticle"]] : [] }
|
|
8
|
+
cases = [
|
|
9
|
+
["/guides/test", {"@type" => "WebPage"}, false],
|
|
10
|
+
["/features/test", {"@type" => "WebPage"}, true],
|
|
11
|
+
["/guides/test", {"@graph" => [{"@type" => "BlogPosting", "headline" => "A guide"}], "@type" => "WebPage"}, true],
|
|
12
|
+
["/guides/test", {"@type" => ["WebPage", "Article"], "headline" => "A guide"}, true],
|
|
13
|
+
["/guides/test", {"@type" => "ItemList", "itemListElement" => [{"@type" => "Article", "headline" => "Another guide"}]}, false]
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
cases.each do |path, data, expected_ok|
|
|
17
|
+
page = html_page(url: "https://example.com#{path}", body: %(<script type="application/ld+json">#{JSON.generate(data)}</script>))
|
|
18
|
+
browser = FakeBrowser.new(page: page)
|
|
19
|
+
config = Crawlscope::Configuration.new
|
|
20
|
+
config.browser_factory = -> { -> { browser } }
|
|
21
|
+
config.renderer = :browser
|
|
22
|
+
config.output = StringIO.new
|
|
23
|
+
config.required_structured_data_types = required_types
|
|
24
|
+
outcome = Crawlscope::StructuredData::Check.new(configuration: config).call(urls: [page.url])
|
|
25
|
+
issues = Crawlscope::IssueCollection.new
|
|
26
|
+
config.rule_registry.rules_for([:structured_data]).first.call(urls: [page.url], pages: [page], issues: issues,
|
|
27
|
+
context: {schema_registry: config.schema_registry})
|
|
28
|
+
|
|
29
|
+
assert_equal expected_ok, outcome.ok?, data.inspect
|
|
30
|
+
assert_equal expected_ok ? [] : [:structured_data_missing_required_type], issues.map(&:code), data.inspect
|
|
31
|
+
unless expected_ok
|
|
32
|
+
assert_equal "page", outcome.entries.first.errors.first.fetch(:source)
|
|
33
|
+
assert_equal "Article or BlogPosting or NewsArticle", outcome.entries.first.errors.first.fetch(:type)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def test_sitemap_and_direct_audits_agree_on_structured_data_validity
|
|
39
|
+
cases = [
|
|
40
|
+
["<h1>No markup</h1>", :missing_structured_data],
|
|
41
|
+
['<script type="application/ld+json">{</script>', :structured_data_parse_error],
|
|
42
|
+
['<script type="application/ld+json">{"name":"Untyped"}</script>', :structured_data_missing_type],
|
|
43
|
+
['<script type="application/ld+json">{"@type":"WebPage","@graph":[{"name":"Untyped"}]}</script>', :structured_data_missing_type],
|
|
44
|
+
['<script type="application/ld+json">{"@type":"Article"}</script>', :structured_data_schema_error],
|
|
45
|
+
['<script type="application/ld+json">{"@type":"WebPage","name":"Example"}</script>', nil],
|
|
46
|
+
['<div itemscope itemtype="https://schema.org/WebPage"><span itemprop="name">Example</span></div>', nil]
|
|
47
|
+
]
|
|
48
|
+
registry = Crawlscope::SchemaRegistry.default
|
|
49
|
+
cases.each do |body, expected_code|
|
|
50
|
+
page = html_page(url: "https://example.com/page", body: body)
|
|
51
|
+
browser = FakeBrowser.new(page: page)
|
|
52
|
+
outcome = Crawlscope::StructuredData::Audit.new(browser_factory: -> { browser }, renderer: :browser,
|
|
53
|
+
schema_registry: registry, timeout_seconds: 20).call(urls: [page.url])
|
|
54
|
+
issues = Crawlscope::IssueCollection.new
|
|
55
|
+
Crawlscope::Rules::StructuredData.new.call(urls: [page.url], pages: [page], issues: issues, context: {schema_registry: registry})
|
|
56
|
+
|
|
57
|
+
assert_equal expected_code.nil?, outcome.ok?, body
|
|
58
|
+
assert_equal expected_code ? [expected_code] : [], issues.map(&:code), body
|
|
59
|
+
assert_equal outcome.ok?, Crawlscope::StructuredData::Report.new(outcome).all_valid?
|
|
60
|
+
schema_issue = issues.find { |issue| issue.code == :structured_data_schema_error }
|
|
61
|
+
assert_equal schema_issue.details[:errors], outcome.entries.first.errors.first[:errors] if schema_issue
|
|
62
|
+
assert browser.closed
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
6
66
|
class FakeBrowser
|
|
7
67
|
attr_reader :closed
|
|
8
68
|
|
|
@@ -3,6 +3,38 @@
|
|
|
3
3
|
require "test_helper"
|
|
4
4
|
|
|
5
5
|
class CrawlscopeStructuredDataReportTest < Minitest::Test
|
|
6
|
+
def test_outcome_and_report_agree_for_every_failure_kind_and_success_status
|
|
7
|
+
cases = [
|
|
8
|
+
{status: 200, structured_items: [], failure: :missing_data},
|
|
9
|
+
{status: 201, structured_items: [], failure: :missing_data},
|
|
10
|
+
{status: nil, structured_items: [], failure: :missing_data},
|
|
11
|
+
{status: 200, fetch_error: "Fetch failed", failure: :http},
|
|
12
|
+
{status: 404, fetch_error: "Not found", failure: :http},
|
|
13
|
+
{status: 201, errors: [{errors: [{field: "name", issue: "is required"}]}], failure: :validation},
|
|
14
|
+
{status: nil, errors: [{errors: [{field: "name", issue: "is required"}]}], failure: :validation},
|
|
15
|
+
{status: 200, failure: nil},
|
|
16
|
+
{status: 201, failure: nil},
|
|
17
|
+
{status: nil, failure: nil},
|
|
18
|
+
{status: 200, structured_items: [], skipped_reason: "non-html", failure: nil}
|
|
19
|
+
]
|
|
20
|
+
cases.each do |attributes|
|
|
21
|
+
attributes = attributes.dup
|
|
22
|
+
failure = attributes.delete(:failure)
|
|
23
|
+
defaults = {url: "https://example.com/page", status: 200, structured_items: [{source: "json-ld", data: {"@type" => "WebPage"}}],
|
|
24
|
+
errors: [], fetch_error: nil, content_type: "text/html", skipped_reason: nil}
|
|
25
|
+
entry = Crawlscope::StructuredData::Audit::Page.new(**defaults.merge(attributes))
|
|
26
|
+
outcome = Crawlscope::StructuredData::Audit::Outcome.new(entries: [entry])
|
|
27
|
+
report = Crawlscope::StructuredData::Report.new(outcome)
|
|
28
|
+
|
|
29
|
+
assert_equal failure.nil?, outcome.ok?, attributes.inspect
|
|
30
|
+
assert_equal outcome.ok?, report.all_valid?, attributes.inspect
|
|
31
|
+
assert_equal failure ? 1 : 0, report.failure_count, attributes.inspect
|
|
32
|
+
assert_equal (failure == :http) ? [entry] : [], report.http_errors
|
|
33
|
+
assert_equal (failure == :missing_data) ? [entry] : [], report.missing_data
|
|
34
|
+
assert_equal (failure == :validation) ? [entry] : [], report.validation_errors
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
6
38
|
def test_results_maps_validation_errors_and_skips
|
|
7
39
|
result = Crawlscope::StructuredData::Audit::Outcome.new(
|
|
8
40
|
entries: [
|
|
@@ -3,6 +3,16 @@
|
|
|
3
3
|
require "test_helper"
|
|
4
4
|
|
|
5
5
|
class CrawlscopeStructuredDataRuleTest < Minitest::Test
|
|
6
|
+
def test_does_not_assume_host_routes_require_job_posting_markup
|
|
7
|
+
issues = Crawlscope::IssueCollection.new
|
|
8
|
+
document = page(url: "https://example.com/careers/team", body: '<script type="application/ld+json">{"@type":"WebPage","name":"Our team"}</script>')
|
|
9
|
+
|
|
10
|
+
Crawlscope::Rules::StructuredData.new.call(urls: [document.url], pages: [document], issues: issues,
|
|
11
|
+
context: {schema_registry: Crawlscope::SchemaRegistry.default})
|
|
12
|
+
|
|
13
|
+
refute_includes issues.map(&:code), :missing_job_posting
|
|
14
|
+
end
|
|
15
|
+
|
|
6
16
|
def test_reports_schema_errors_for_invalid_article_markup
|
|
7
17
|
issues = Crawlscope::IssueCollection.new
|
|
8
18
|
rule = Crawlscope::Rules::StructuredData.new
|
|
@@ -203,7 +213,7 @@ class CrawlscopeStructuredDataRuleTest < Minitest::Test
|
|
|
203
213
|
|
|
204
214
|
def test_reports_missing_job_posting_for_career_detail_pages
|
|
205
215
|
issues = Crawlscope::IssueCollection.new
|
|
206
|
-
rule = Crawlscope::Rules::StructuredData.new
|
|
216
|
+
rule = Crawlscope::Rules::StructuredData.new(job_posting_path: %r{/careers/[^/]+/?\z})
|
|
207
217
|
page = page(
|
|
208
218
|
url: "https://example.com/careers/sales-partner",
|
|
209
219
|
body: <<~HTML
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class CrawlscopeStructuredDataValidatorTest < Minitest::Test
|
|
6
|
+
def test_requires_every_type_group
|
|
7
|
+
validator = Crawlscope::StructuredData::Validator.new(schema_registry: Crawlscope::SchemaRegistry.default)
|
|
8
|
+
html = '<script type="application/ld+json">{"@type":"WebPage","name":"Benchmark"}</script>'
|
|
9
|
+
|
|
10
|
+
result = validator.call(html: html, required_types: ["WebPage", "Dataset"])
|
|
11
|
+
|
|
12
|
+
assert_equal [:structured_data_missing_required_type], result.findings.map(&:code)
|
|
13
|
+
assert_equal ["Dataset"], result.findings.first.errors.first.fetch(:expected_types)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def test_microdata_satisfies_type_requirements
|
|
17
|
+
validator = Crawlscope::StructuredData::Validator.new(schema_registry: Crawlscope::SchemaRegistry.default)
|
|
18
|
+
html = '<article itemscope itemtype="https://schema.org/Article"><h1 itemprop="headline">A guide</h1></article>'
|
|
19
|
+
|
|
20
|
+
result = validator.call(html: html, required_types: [["Article", "BlogPosting", "NewsArticle"]])
|
|
21
|
+
|
|
22
|
+
assert_empty result.findings
|
|
23
|
+
end
|
|
24
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawlscope
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.10.
|
|
4
|
+
version: 0.10.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Paulo Fidalgo
|
|
@@ -178,6 +178,7 @@ files:
|
|
|
178
178
|
- LICENSE.txt
|
|
179
179
|
- README.md
|
|
180
180
|
- UPGRADE.md
|
|
181
|
+
- docs/architecture.md
|
|
181
182
|
- docs/rails.md
|
|
182
183
|
- docs/rules.md
|
|
183
184
|
- docs/server-timing.md
|
|
@@ -189,7 +190,12 @@ files:
|
|
|
189
190
|
- lib/crawlscope/context.rb
|
|
190
191
|
- lib/crawlscope/crawl.rb
|
|
191
192
|
- lib/crawlscope/crawler.rb
|
|
193
|
+
- lib/crawlscope/crawling/fetcher_factory.rb
|
|
194
|
+
- lib/crawlscope/crawling/target.rb
|
|
195
|
+
- lib/crawlscope/crawling/target_resolver.rb
|
|
192
196
|
- lib/crawlscope/document_text.rb
|
|
197
|
+
- lib/crawlscope/documents/canonical.rb
|
|
198
|
+
- lib/crawlscope/documents/robots.rb
|
|
193
199
|
- lib/crawlscope/fetch_executor.rb
|
|
194
200
|
- lib/crawlscope/fetch_executor/async.rb
|
|
195
201
|
- lib/crawlscope/fetch_executor/threaded.rb
|
|
@@ -201,12 +207,23 @@ files:
|
|
|
201
207
|
- lib/crawlscope/railtie.rb
|
|
202
208
|
- lib/crawlscope/rake_tasks.rb
|
|
203
209
|
- lib/crawlscope/reporter.rb
|
|
210
|
+
- lib/crawlscope/reporting/issue_presenter.rb
|
|
211
|
+
- lib/crawlscope/reporting/url_formatter.rb
|
|
204
212
|
- lib/crawlscope/request_headers.rb
|
|
205
213
|
- lib/crawlscope/result.rb
|
|
206
214
|
- lib/crawlscope/rule_registry.rb
|
|
207
215
|
- lib/crawlscope/rules/content_quality.rb
|
|
208
216
|
- lib/crawlscope/rules/indexability.rb
|
|
209
217
|
- lib/crawlscope/rules/links.rb
|
|
218
|
+
- lib/crawlscope/rules/links/canonical_checks.rb
|
|
219
|
+
- lib/crawlscope/rules/links/extractor.rb
|
|
220
|
+
- lib/crawlscope/rules/links/graph.rb
|
|
221
|
+
- lib/crawlscope/rules/links/inbound_checks.rb
|
|
222
|
+
- lib/crawlscope/rules/links/outgoing_checks.rb
|
|
223
|
+
- lib/crawlscope/rules/links/resolution_check.rb
|
|
224
|
+
- lib/crawlscope/rules/links/scope.rb
|
|
225
|
+
- lib/crawlscope/rules/links/target_lookup.rb
|
|
226
|
+
- lib/crawlscope/rules/links/url_checks.rb
|
|
210
227
|
- lib/crawlscope/rules/metadata.rb
|
|
211
228
|
- lib/crawlscope/rules/structured_data.rb
|
|
212
229
|
- lib/crawlscope/rules/uniqueness.rb
|
|
@@ -220,9 +237,12 @@ files:
|
|
|
220
237
|
- lib/crawlscope/structured_data/audit.rb
|
|
221
238
|
- lib/crawlscope/structured_data/check.rb
|
|
222
239
|
- lib/crawlscope/structured_data/document.rb
|
|
240
|
+
- lib/crawlscope/structured_data/entry.rb
|
|
223
241
|
- lib/crawlscope/structured_data/error_formatter.rb
|
|
242
|
+
- lib/crawlscope/structured_data/outcome.rb
|
|
224
243
|
- lib/crawlscope/structured_data/report.rb
|
|
225
244
|
- lib/crawlscope/structured_data/reporter.rb
|
|
245
|
+
- lib/crawlscope/structured_data/validator.rb
|
|
226
246
|
- lib/crawlscope/structured_data/writer.rb
|
|
227
247
|
- lib/crawlscope/tasks.rb
|
|
228
248
|
- lib/crawlscope/url.rb
|
|
@@ -258,6 +278,7 @@ files:
|
|
|
258
278
|
- test/crawlscope/structured_data_report_test.rb
|
|
259
279
|
- test/crawlscope/structured_data_reporter_test.rb
|
|
260
280
|
- test/crawlscope/structured_data_rule_test.rb
|
|
281
|
+
- test/crawlscope/structured_data_validator_test.rb
|
|
261
282
|
- test/crawlscope/structured_data_writer_test.rb
|
|
262
283
|
- test/crawlscope/uniqueness_rule_test.rb
|
|
263
284
|
- test/crawlscope/url_test.rb
|