aranha 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fae3e1f9e295a208be700db6357172330192e9e4c0586ef0fff9a78db59001a3
-  data.tar.gz: fb90fe66bdca6283dd61b4effd5d9265c7d8c99d5efbb21b22e8519a75aa4cb7
+  metadata.gz: 86615d49ac9b469d088f1fe3404db6efc98bdc19a0f0cd484bd968d1a68e090c
+  data.tar.gz: 32f5a13f4b3e5d3d17867aa2f911f85cda43b9ed4fc1d55ce7b29fe8fb3e8c2e
 SHA512:
-  metadata.gz: 9a9cef98927abcc0ffbca46aed279e757aadbf4959604a0855094e19585364de1eca69ad209c27acbe959614a33856d322e9046005b4aeeb5404d064159c20d3
-  data.tar.gz: 680fcb07943ee059df08c17c51dcc8ebec377be0e153be7118118c5c566ae195c860712e79248e940ca652f21f3bfeff91b37a5375a381bbc62c6c7ea3e72052
+  metadata.gz: 77697e1e182931f7bda03c402bef371f6f89d8cf963960210f80b5d82c665a912d96e0030c30be13d4c10d7c8159cc8d003571ee694e77df08e35d13b7b55838
+  data.tar.gz: 48a89f0b5b20c5ba77d18fe1744b7f3ba1cb964e2de020b2ea38b21a851224834bae19cefd346e9e070408e0c510f7c7cfca11a44e33dcb45f26addf435c9dfd

data/lib/aranha/default_processor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'addressable'
+
 module Aranha
   class DefaultProcessor
     attr_reader :source_uri, :extra_data

data/lib/aranha/dom_elements_traverser/cursor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'eac_ruby_utils/options_consumer'
+
 module Aranha
   class DomElementsTraverser
     module Cursor

data/lib/aranha/dom_elements_traverser.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_dependency 'aranha/dom_elements_traverser/conditions'
-require_dependency 'aranha/dom_elements_traverser/data'
-require_dependency 'aranha/dom_elements_traverser/cursor'
+require 'aranha/dom_elements_traverser/conditions'
+require 'aranha/dom_elements_traverser/data'
+require 'aranha/dom_elements_traverser/cursor'
 
 module Aranha
   class DomElementsTraverser

data/lib/aranha/fixtures/download.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require 'aranha/parsers/base'
 require 'aranha/parsers/source_address'
+require 'aranha/parsers/spec/source_target_fixtures'
 
 module Aranha
   module Fixtures
@@ -29,7 +31,7 @@ module Aranha
 
       def select_path?(path)
         return false unless match_prefix_pattern(path)
-        !pending || !::File.exist?(target(path))
+        !pending || !source_exist?(path)
       end
 
       def match_prefix_pattern(path)
@@ -56,6 +58,11 @@ module Aranha
       def relative_path(path)
         path.sub(%r{^#{Regexp.quote(fixtures_root)}/}, '')
       end
+
+      def source_exist?(path)
+        stf = ::Aranha::Spec::SourceTargetFixtures.new(::File.dirname(path))
+        stf.source_file(::File.basename(path, '.url')).present?
+      end
     end
   end
 end

data/lib/aranha/fixtures.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Aranha
+  module Fixtures
+    require 'aranha/fixtures/download'
+  end
+end

data/lib/aranha/parsers/html/base.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../base'
-require_relative 'node/default'
+require 'nokogiri'
+require 'aranha/parsers/base'
+require 'aranha/parsers/html/node/default'
 
 module Aranha
   module Parsers

data/lib/aranha/parsers/html/item.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'base'
+require 'aranha/parsers/html/base'
 
 module Aranha
   module Parsers

data/lib/aranha/parsers/html/item_list.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'base'
+require 'aranha/parsers/html/base'
 
 module Aranha
   module Parsers
@@ -13,8 +13,7 @@ module Aranha
             node_parser.parse(m)
           end
         rescue StandardError => e
-          e.message << " / Count: #{count}"
-          raise e
+          raise StandardError, "#{e.message} (Count: #{count})"
         end
 
         def items_xpath

data/lib/aranha/parsers/html/node/default.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'base'
+require 'aranha/parsers/html/node/base'
 
 module Aranha
   module Parsers

data/lib/aranha/parsers/html/node.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        require 'aranha/parsers/html/node/base'
+        require 'aranha/parsers/html/node/default'
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+module Aranha
+  module Parsers
+    module Html
+      require 'aranha/parsers/html/base'
+      require 'aranha/parsers/html/item'
+      require 'aranha/parsers/html/item_list'
+      require 'aranha/parsers/html/node'
+    end
+  end
+end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'addressable'
 require 'net/http'
 
 module Aranha
@@ -7,6 +8,10 @@ module Aranha
     class SourceAddress
       class HttpGet
         class << self
+          def location_uri(source_uri, location)
+            ::Addressable::URI.join(source_uri, location).to_s
+          end
+
           def valid_source?(source)
             source.to_s =~ %r{\Ahttps?://}
           end
@@ -45,7 +50,7 @@ module Aranha
           when Net::HTTPSuccess then
             response.body
           when Net::HTTPRedirection then
-            content_fetch(response['location'], limit - 1)
+            content_fetch(self.class.location_uri(uri, response['location']), limit - 1)
           else
             response.value
           end

data/lib/aranha/parsers/spec/source_target_fixtures.rb

@@ -34,8 +34,6 @@ module Aranha
         r
       end
 
-      private
-
       def target_file(basename)
         fixture_file(basename, 'target')
       end
@@ -44,6 +42,8 @@ module Aranha
         fixture_file(basename, 'source')
       end
 
+      private
+
       def fixture_file(basename, suffix)
         prefix = "#{basename}.#{suffix}"
         Dir.foreach(fixtures_directory) do |item|

data/lib/aranha/parsers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+module Aranha
+  module Parsers
+    require 'aranha/parsers/base'
+    require 'aranha/parsers/html'
+    require 'aranha/parsers/invalid_state_exception'
+    require 'aranha/parsers/source_address'
+  end
+end

data/lib/aranha/processor.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'net/http'
-require_dependency 'aranha/parsers/invalid_state_exception'
+require 'aranha/parsers/invalid_state_exception'
 
 module Aranha
   class Processor

data/lib/aranha/selenium/driver_factory.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'active_support/core_ext/hash/indifferent_access'
+require 'aranha/selenium/driver_factory/base'
 require 'aranha/selenium/driver_factory/chrome'
 require 'aranha/selenium/driver_factory/firefox'
 

data/lib/aranha/selenium.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+module Aranha
+  module Selenium
+    require 'aranha/selenium/driver_factory'
+    require 'aranha/selenium/session'
+  end
+end

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.9.0'
+  VERSION = '0.9.1'
 end

data/lib/aranha.rb

@@ -2,18 +2,14 @@
 
 require 'httpclient'
 require 'active_support/dependencies'
-require_dependency 'aranha/engine'
-require_dependency 'active_scaffold'
+require 'active_scaffold'
 
 module Aranha
+  require 'aranha/default_processor'
+  require 'aranha/dom_elements_traverser'
+  require 'aranha/engine'
+  require 'aranha/fixtures'
+  require 'aranha/processor'
+  require 'aranha/parsers'
+  require 'aranha/selenium'
 end
-
-require_dependency 'aranha/default_processor'
-require_dependency 'aranha/fixtures/download'
-require_dependency 'aranha/processor'
-require_dependency 'aranha/parsers/base'
-require_dependency 'aranha/parsers/html/base'
-require_dependency 'aranha/parsers/html/item_list'
-require_dependency 'aranha/parsers/invalid_state_exception'
-require_dependency 'aranha/dom_elements_traverser'
-require_dependency 'aranha/selenium/driver_factory'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-11 00:00:00.000000000 Z
+date: 2019-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -141,11 +141,15 @@ files:
 - lib/aranha/dom_elements_traverser/cursor.rb
 - lib/aranha/dom_elements_traverser/data.rb
 - lib/aranha/engine.rb
+- lib/aranha/fixtures.rb
 - lib/aranha/fixtures/download.rb
+- lib/aranha/parsers.rb
 - lib/aranha/parsers/base.rb
+- lib/aranha/parsers/html.rb
 - lib/aranha/parsers/html/base.rb
 - lib/aranha/parsers/html/item.rb
 - lib/aranha/parsers/html/item_list.rb
+- lib/aranha/parsers/html/node.rb
 - lib/aranha/parsers/html/node/base.rb
 - lib/aranha/parsers/html/node/default.rb
 - lib/aranha/parsers/invalid_state_exception.rb
@@ -157,6 +161,7 @@ files:
 - lib/aranha/parsers/spec/source_target_fixtures.rb
 - lib/aranha/parsers/spec/source_target_fixtures_example.rb
 - lib/aranha/processor.rb
+- lib/aranha/selenium.rb
 - lib/aranha/selenium/driver_factory.rb
 - lib/aranha/selenium/driver_factory/base.rb
 - lib/aranha/selenium/driver_factory/chrome.rb
@@ -194,5 +199,5 @@ specification_version: 4
 summary: Rails utilities for web crawling.
 test_files:
 - test/aranha_test.rb
-- test/test_helper.rb
 - test/integration/navigation_test.rb
+- test/test_helper.rb