crawlscope 0.1.0 → 0.3.0

data/test/crawlscope/structured_data_writer_test.rb

@@ -5,9 +5,9 @@ require "test_helper"
 
 class CrawlscopeStructuredDataWriterTest < Minitest::Test
   def test_writes_json_report
-    result = Crawlscope::StructuredData::Audit::Result.new(
+    result = Crawlscope::StructuredData::Audit::Outcome.new(
       entries: [
-        Crawlscope::StructuredData::Audit::Entry.new(
+        Crawlscope::StructuredData::Audit::Page.new(
           url: "https://example.com/article",
           status: 200,
           structured_items: [{source: "json-ld", data: {"@type" => "Article"}}],

data/test/crawlscope/url_test.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeUrlTest < Minitest::Test
+  def test_normalize_resolves_relative_urls_and_removes_trailing_slash
+    assert_equal "https://example.com/pricing", Crawlscope::Url.normalize("/pricing/", base_url: "https://example.com")
+  end
+
+  def test_normalize_preserves_non_default_port
+    assert_equal "http://localhost:3000/pricing", Crawlscope::Url.normalize("/pricing", base_url: "http://localhost:3000")
+  end
+
+  def test_normalize_for_base_rebases_absolute_urls
+    assert_equal(
+      "http://localhost:3000/features",
+      Crawlscope::Url.normalize_for_base("https://www.example.com/features", base_url: "http://localhost:3000")
+    )
+  end
+
+  def test_path_normalizes_blank_and_trailing_slash
+    assert_equal "/", Crawlscope::Url.path("https://example.com")
+    assert_equal "/features", Crawlscope::Url.path("https://example.com/features/")
+  end
+
+  def test_invalid_urls_are_returned_or_ignored
+    assert_equal "http:// bad", Crawlscope::Url.normalize("http:// bad", base_url: "https://example.com")
+    assert_nil Crawlscope::Url.path("http:// bad")
+    refute Crawlscope::Url.remote?("http:// bad")
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -193,16 +193,18 @@ files:
 - README.md
 - exe/crawlscope
 - lib/crawlscope.rb
-- lib/crawlscope/audit.rb
 - lib/crawlscope/browser.rb
 - lib/crawlscope/cli.rb
 - lib/crawlscope/configuration.rb
+- lib/crawlscope/context.rb
+- lib/crawlscope/crawl.rb
 - lib/crawlscope/crawler.rb
 - lib/crawlscope/http.rb
 - lib/crawlscope/issue.rb
 - lib/crawlscope/issue_collection.rb
 - lib/crawlscope/page.rb
 - lib/crawlscope/railtie.rb
+- lib/crawlscope/rake_tasks.rb
 - lib/crawlscope/reporter.rb
 - lib/crawlscope/result.rb
 - lib/crawlscope/rule_registry.rb
@@ -210,23 +212,31 @@ files:
 - lib/crawlscope/rules/metadata.rb
 - lib/crawlscope/rules/structured_data.rb
 - lib/crawlscope/rules/uniqueness.rb
+- lib/crawlscope/run.rb
 - lib/crawlscope/schema_registry.rb
+- lib/crawlscope/schemas.rb
 - lib/crawlscope/sitemap.rb
 - lib/crawlscope/structured_data/audit.rb
+- lib/crawlscope/structured_data/check.rb
 - lib/crawlscope/structured_data/document.rb
 - lib/crawlscope/structured_data/report.rb
 - lib/crawlscope/structured_data/reporter.rb
 - lib/crawlscope/structured_data/writer.rb
-- lib/crawlscope/task.rb
 - lib/crawlscope/url.rb
 - lib/crawlscope/version.rb
 - lib/tasks/crawlscope_tasks.rake
-- test/crawlscope/audit_test.rb
+- test/crawlscope/browser_test.rb
 - test/crawlscope/cli_test.rb
 - test/crawlscope/configuration_test.rb
+- test/crawlscope/crawl_test.rb
+- test/crawlscope/crawler_test.rb
+- test/crawlscope/http_test.rb
 - test/crawlscope/links_rule_test.rb
 - test/crawlscope/loader_test.rb
+- test/crawlscope/metadata_rule_test.rb
 - test/crawlscope/reporter_test.rb
+- test/crawlscope/rule_registry_test.rb
+- test/crawlscope/run_test.rb
 - test/crawlscope/schema_registry_test.rb
 - test/crawlscope/sitemap_test.rb
 - test/crawlscope/structured_data_audit_test.rb
@@ -235,8 +245,8 @@ files:
 - test/crawlscope/structured_data_reporter_test.rb
 - test/crawlscope/structured_data_rule_test.rb
 - test/crawlscope/structured_data_writer_test.rb
-- test/crawlscope/task_test.rb
 - test/crawlscope/uniqueness_rule_test.rb
+- test/crawlscope/url_test.rb
 - test/test_helper.rb
 homepage: https://www.ethos-link.com/opensource/crawlscope
 licenses:

data/lib/crawlscope/task.rb

@@ -1,131 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module Crawlscope
-  class Task
-    def initialize(configuration: Crawlscope.configuration, reporter: Reporter.new(io: configuration.output))
-      @configuration = configuration
-      @reporter = reporter
-    end
-
-    def validate(base_url: nil, sitemap_path: nil, rule_names: nil)
-      resolved_base_url = base_url || default_base_url
-      audit = @configuration.audit(
-        base_url: resolved_base_url,
-        sitemap_path: sitemap_path || default_sitemap_path(base_url: resolved_base_url),
-        rule_names: rule_names
-      )
-
-      result = audit.call
-      @reporter.report(result)
-      result
-    end
-
-    def validate_ldjson(urls:, debug: false, renderer: @configuration.renderer, timeout_seconds: @configuration.timeout_seconds, report_path: nil, summary: false)
-      audit = StructuredData::Audit.new(
-        browser_factory: @configuration.browser_factory,
-        network_idle_timeout_seconds: @configuration.network_idle_timeout_seconds,
-        renderer: renderer,
-        schema_registry: @configuration.schema_registry,
-        scroll_page: @configuration.scroll_page?,
-        timeout_seconds: timeout_seconds
-      )
-      result = audit.call(urls: urls)
-
-      report_ldjson_result(result, debug: debug, renderer: renderer)
-      StructuredData::Writer.new(path: report_path).write(result) if report_path
-      StructuredData::Reporter.new(io: @configuration.output, report_path: report_path).report(result) if summary
-      result
-    end
-
-    private
-
-    def default_base_url
-      value = @configuration.base_url
-      return value unless value.to_s.strip.empty?
-
-      "http://localhost:3000"
-    end
-
-    def default_sitemap_path(base_url:)
-      value = @configuration.sitemap_path
-      return value unless value.to_s.strip.empty?
-
-      local_path = File.expand_path("public/sitemap.xml", Dir.pwd)
-      if local_path_default?(base_url: base_url) && File.exist?(local_path)
-        return local_path
-      end
-
-      "#{base_url.to_s.chomp("/")}/sitemap.xml"
-    end
-
-    def local_path_default?(base_url:)
-      host = URI.parse(base_url.to_s).host.to_s
-      ["localhost", "127.0.0.1"].include?(host)
-    rescue URI::InvalidURIError
-      false
-    end
-
-    def report_ldjson_result(result, debug:, renderer:)
-      if renderer == :browser
-        @configuration.output.puts("JavaScript mode enabled (Ferrum)")
-      end
-
-      @configuration.output.puts("Validating JSON-LD on #{result.entries.size} URL(s)")
-      @configuration.output.puts("")
-
-      result.entries.each do |entry|
-        @configuration.output.puts("=" * 80)
-        @configuration.output.puts("URL: #{entry.url}")
-        @configuration.output.puts("=" * 80)
-
-        if entry.fetch_error
-          @configuration.output.puts("Error: #{entry.fetch_error}")
-          @configuration.output.puts("")
-          next
-        end
-
-        if entry.status
-          @configuration.output.puts("Status: #{entry.status}")
-        else
-          @configuration.output.puts("Status: JS runtime fetch")
-        end
-
-        @configuration.output.puts("Structured data found: #{entry.structured_items.size} (JSON-LD: #{entry.json_ld_count}, Microdata: #{entry.microdata_count})")
-
-        if debug && entry.structured_items.any?
-          @configuration.output.puts("")
-          @configuration.output.puts("--- Detected Structured Data ---")
-
-          entry.structured_items.each_with_index do |item, index|
-            @configuration.output.puts("")
-            @configuration.output.puts("## Item #{index + 1} [#{item[:source]}]")
-            @configuration.output.puts(JSON.pretty_generate(item[:data]))
-          end
-
-          @configuration.output.puts("")
-          @configuration.output.puts("--- End ---")
-        end
-
-        @configuration.output.puts("")
-        @configuration.output.puts("Validation results:")
-
-        if entry.errors.empty?
-          @configuration.output.puts("  All valid!")
-        else
-          entry.errors.each do |error|
-            @configuration.output.puts("  #{error[:type]}: INVALID [#{error[:source]}]")
-            error[:errors].each do |validation_error|
-              @configuration.output.puts("    - field: #{validation_error[:field]}, issue: #{validation_error[:issue]}")
-            end
-          end
-        end
-
-        @configuration.output.puts("")
-      end
-
-      @configuration.output.puts("STATUS: #{result.ok? ? "OK" : "FAILED"}")
-    end
-  end
-end