aranha 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffc06565f11caae4796f17fde34ca461a30bda384b278205cbe516a870944e3a
-  data.tar.gz: 1d68d7b086aafe552001dfa0441973e5c40347d542f27f4798547ea0f316c84b
+  metadata.gz: b770597c6db06b5641e943c6b758e4bf03da504dddde3b53873042397ebd2d0f
+  data.tar.gz: 97e2aa65591ba5ef13af0d3e121b5e81f6d30ab78d10c5d6b8654b96e40e55fc
 SHA512:
-  metadata.gz: 67b8db80ac9a675177babac7cebdf934383371264b4e398c4ae9c69932f5e3a0796c56aeccbccab6fdde039223b5bc8b1209b6fb6fcb18be9235239fcabb6a0d
-  data.tar.gz: 7a8f00e58e0282a3c899cce7789f3b45a565c199c5c182d6592094c83579c6b17f4136d8fa9de51f4a5f571091591de3b5b34d2eb4405b7259c595330ef91f3d
+  metadata.gz: c18119d14b4b101ed22289c4da45faa78949c2c7871a08c62a182722f7fbe42807f3101526f970f463657dcf6341e596f54eb32d8f040f6f529d3fe961b77db2
+  data.tar.gz: 83f31a41541d7a700e448894b665109b43c2ee8f7a8adfe561292dc675ace9fa118cb59e6891a03d916ac03043c9eb36879eb1fa08e14a2c78ee693de91de3e4

data/app/models/aranha/address.rb

@@ -17,9 +17,10 @@ module Aranha
         end
       end
 
-      def add(url, processor)
+      def add(url, processor, extra_data = nil)
         a = find_or_initialize_by(url: url)
         a.processor = processor
+        a.extra_data = extra_data.to_yaml
         a.save!
       end
 
@@ -49,10 +50,30 @@ module Aranha
 
     def process
       ActiveRecord::Base.transaction do
-        processor.constantize.new(url).process
+        instanciate_processor.process
         self.processed_at = Time.zone.now
         save!
       end
     end
+
+    private
+
+    def instanciate_processor
+      if processor_instancier_arity == 2 || processor_instancier_arity < 0
+        processor_instancier.call(url, YAML.load(extra_data))
+      elsif processor_instancier_arity == 1
+        processor_instancier.call(url)
+      else
+        raise("#{processor}.initialize should has 1 or 2 or * arguments")
+      end
+    end
+
+    def processor_instancier
+      processor.constantize.method(:new)
+    end
+
+    def processor_instancier_arity
+      processor.constantize.instance_method(:initialize).arity
+    end
   end
 end

data/db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+class AddExtraDataToAranhaAddresses < ActiveRecord::Migration
+  def change
+    add_column :aranha_addresses, :extra_data, :text
+  end
+end

data/lib/aranha/default_processor.rb

@@ -2,13 +2,14 @@
 
 module Aranha
   class DefaultProcessor
-    attr_reader :source_uri
+    attr_reader :source_uri, :extra_data
 
-    def initialize(source_uri)
+    def initialize(source_uri, extra_data)
       unless source_uri.is_a?(Addressable::URI)
         source_uri = source_uri.to_s.gsub(%r{\A/}, 'file:///')
       end
       @source_uri = Addressable::URI.parse(source_uri)
+      @extra_data = extra_data
     end
 
     def process

data/lib/aranha/parsers/base.rb

@@ -33,7 +33,7 @@ module Aranha
       end
 
       def content_file
-        ::File.open(@url.gsub(%r{\Afile://}, ''), &:read)
+        ::File.open(@url.to_s.gsub(%r{\Afile://}, ''), &:read)
       end
 
       def content_get

data/lib/aranha/processor.rb

@@ -5,10 +5,19 @@ require_dependency 'aranha/parsers/invalid_state_exception'
 
 module Aranha
   class Processor
-    NETWORK_EXCEPTIONS = [::HTTPClient::BadResponseError, Errno::ECONNRESET,
-                          ::Net::HTTPFatalError, HTTPClient::ConnectTimeoutError,
-                          ::HTTPClient::ReceiveTimeoutError,
-                          ::Aranha::Parsers::InvalidStateException].freeze
+    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
+    CORE_EXCEPTIONS = [::SocketError].freeze
+    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
+    HTTPCLIENT_EXCEPTIONS = [
+      ::HTTPClient::BadResponseError,
+      ::HTTPClient::ConnectTimeoutError,
+      ::HTTPClient::ReceiveTimeoutError
+    ].freeze
+    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
+
+    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
+                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
+
     DEFAULT_MAX_TRIES = 3
 
     def initialize
@@ -48,8 +57,8 @@ module Aranha
       begin
         address.process
         @failed.delete(address.id)
-      rescue StandardError => ex
-        process_exception(address, ex)
+      rescue StandardError => e
+        process_exception(address, e)
       end
     end
 
@@ -70,7 +79,8 @@ module Aranha
     end
 
     def network_exception?(exception)
-      NETWORK_EXCEPTIONS.any? { |klass| exception.is_a?(klass) }
+      return true if NETWORK_EXCEPTIONS.any? { |klass| exception.is_a?(klass) }
+      exception.cause.present? ? network_exception?(exception.cause) : false
     end
 
     def not_try_ids
@@ -85,8 +95,8 @@ module Aranha
       @max_tries ||= begin
         r = Integer(ENV['ARANHA_MAX_TRIES'])
         r <= 0 ? 0 : r
-                     rescue ArgumentError, TypeError
-                       DEFAULT_MAX_TRIES
+      rescue ArgumentError, TypeError
+        DEFAULT_MAX_TRIES
       end
     end
   end

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.2.1'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-14 00:00:00.000000000 Z
+date: 2019-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -99,6 +99,7 @@ files:
 - app/views/layouts/aranha/application.html.erb
 - config/routes.rb
 - db/migrate/20171201021251_create_aranha_addresses.rb
+- db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb
 - lib/aranha.rb
 - lib/aranha/default_processor.rb
 - lib/aranha/dom_elements_traverser.rb