crawlscope 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +38 -0
  3. data/README.md +50 -1
  4. data/lib/crawlscope/cli.rb +16 -0
  5. data/lib/crawlscope/configuration.rb +10 -1
  6. data/lib/crawlscope/context.rb +1 -1
  7. data/lib/crawlscope/crawl.rb +72 -14
  8. data/lib/crawlscope/crawler.rb +3 -17
  9. data/lib/crawlscope/document_text.rb +7 -2
  10. data/lib/crawlscope/fetch_executor/async.rb +32 -0
  11. data/lib/crawlscope/fetch_executor/threaded.rb +32 -0
  12. data/lib/crawlscope/fetch_executor.rb +43 -0
  13. data/lib/crawlscope/http.rb +7 -1
  14. data/lib/crawlscope/railtie.rb +1 -1
  15. data/lib/crawlscope/reporter.rb +123 -14
  16. data/lib/crawlscope/result.rb +1 -1
  17. data/lib/crawlscope/rules/content_quality.rb +1 -1
  18. data/lib/crawlscope/rules/indexability.rb +28 -6
  19. data/lib/crawlscope/rules/links.rb +80 -16
  20. data/lib/crawlscope/rules/uniqueness.rb +23 -4
  21. data/lib/crawlscope/sitemap.rb +30 -11
  22. data/lib/crawlscope/tasks.rb +8 -0
  23. data/lib/crawlscope/version.rb +1 -1
  24. data/lib/crawlscope.rb +1 -0
  25. data/lib/tasks/crawlscope_tasks.rake +1 -1
  26. data/test/crawlscope/cli_test.rb +28 -2
  27. data/test/crawlscope/configuration_test.rb +21 -0
  28. data/test/crawlscope/content_quality_rule_test.rb +18 -0
  29. data/test/crawlscope/crawl_test.rb +142 -4
  30. data/test/crawlscope/crawler_test.rb +61 -0
  31. data/test/crawlscope/fetch_executor_test.rb +44 -0
  32. data/test/crawlscope/links_rule_test.rb +101 -0
  33. data/test/crawlscope/loader_test.rb +37 -0
  34. data/test/crawlscope/reporter_test.rb +136 -11
  35. data/test/crawlscope/result_test.rb +35 -0
  36. data/test/crawlscope/sitemap_test.rb +52 -0
  37. data/test/performance/async_fetch_benchmark.rb +127 -0
  38. data/test/performance/fetch_executor_matrix.rb +162 -0
  39. data/test/performance/sitemap_expansion_benchmark.rb +121 -0
  40. metadata +40 -3
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crawlscope
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Paulo Fidalgo
@@ -24,6 +24,34 @@ dependencies:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: async
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '2.0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '2.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: async-http-faraday
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0.22'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0.22'
27
55
  - !ruby/object:Gem::Dependency
28
56
  name: faraday
29
57
  requirement: !ruby/object:Gem::Requirement
@@ -200,6 +228,9 @@ files:
200
228
  - lib/crawlscope/crawl.rb
201
229
  - lib/crawlscope/crawler.rb
202
230
  - lib/crawlscope/document_text.rb
231
+ - lib/crawlscope/fetch_executor.rb
232
+ - lib/crawlscope/fetch_executor/async.rb
233
+ - lib/crawlscope/fetch_executor/threaded.rb
203
234
  - lib/crawlscope/http.rb
204
235
  - lib/crawlscope/issue.rb
205
236
  - lib/crawlscope/issue_collection.rb
@@ -225,6 +256,7 @@ files:
225
256
  - lib/crawlscope/structured_data/report.rb
226
257
  - lib/crawlscope/structured_data/reporter.rb
227
258
  - lib/crawlscope/structured_data/writer.rb
259
+ - lib/crawlscope/tasks.rb
228
260
  - lib/crawlscope/url.rb
229
261
  - lib/crawlscope/version.rb
230
262
  - lib/tasks/crawlscope_tasks.rake
@@ -234,6 +266,7 @@ files:
234
266
  - test/crawlscope/content_quality_rule_test.rb
235
267
  - test/crawlscope/crawl_test.rb
236
268
  - test/crawlscope/crawler_test.rb
269
+ - test/crawlscope/fetch_executor_test.rb
237
270
  - test/crawlscope/http_test.rb
238
271
  - test/crawlscope/indexability_rule_test.rb
239
272
  - test/crawlscope/links_rule_test.rb
@@ -241,6 +274,7 @@ files:
241
274
  - test/crawlscope/metadata_rule_test.rb
242
275
  - test/crawlscope/rake_tasks_test.rb
243
276
  - test/crawlscope/reporter_test.rb
277
+ - test/crawlscope/result_test.rb
244
278
  - test/crawlscope/rule_registry_test.rb
245
279
  - test/crawlscope/run_test.rb
246
280
  - test/crawlscope/schema_registry_test.rb
@@ -253,6 +287,9 @@ files:
253
287
  - test/crawlscope/structured_data_writer_test.rb
254
288
  - test/crawlscope/uniqueness_rule_test.rb
255
289
  - test/crawlscope/url_test.rb
290
+ - test/performance/async_fetch_benchmark.rb
291
+ - test/performance/fetch_executor_matrix.rb
292
+ - test/performance/sitemap_expansion_benchmark.rb
256
293
  - test/release_task_test.rb
257
294
  - test/test_helper.rb
258
295
  homepage: https://www.ethos-link.com/opensource/crawlscope
@@ -275,14 +312,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
275
312
  requirements:
276
313
  - - ">="
277
314
  - !ruby/object:Gem::Version
278
- version: 3.2.0
315
+ version: 3.3.0
279
316
  required_rubygems_version: !ruby/object:Gem::Requirement
280
317
  requirements:
281
318
  - - ">="
282
319
  - !ruby/object:Gem::Version
283
320
  version: '0'
284
321
  requirements: []
285
- rubygems_version: 4.0.10
322
+ rubygems_version: 4.0.16
286
323
  specification_version: 4
287
324
  summary: Audit sitemap URLs for metadata, structured data, uniqueness, and links
288
325
  test_files: []