aranha 0.15.2 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf75a29a21e24f2e85a425b8b9291c058a46eef895a8100992514dd03138b065
-  data.tar.gz: d52cc8cf889e0d9174f13771e3c9ac15971a1f673dc8e4e15085aab3755e169b
+  metadata.gz: cc769d0b380643afe3740301d894fd70e93d3df19d3f6cf4bde1d7633d059d4f
+  data.tar.gz: 8d019e0d77e967ca19ff7325dc0081aa19bcdb114ea1812e49e245d121ca57c2
 SHA512:
-  metadata.gz: 4844ac22fc5d9b3a16e7b03d8da6e5e5fc82a95dd705a0bbbb11c029f206fdc6fb762d1fd2cd71a2c8f869084597ad3f8d65e88f79f95bf37dfcd02a67cc59c6
-  data.tar.gz: c768ba132ef0b9afbf9630e62aeb4759ab5bdb09e8d6af2658c59b19360c2a9c11dd3c1abd49431111fa25273c895ede8eb2536d7f63ebd607a1965c9e9a0813
+  metadata.gz: 16fd673ddb8be560b62a3dbab2e4e81e4b7b66842d5594ee4eb63639ba7a3bec6a43f239d69918c0417ed7a132069eea9bdf6ac741e41988106cb8f0ec1e0e56
+  data.tar.gz: 7b5c39602e3694ffbca23db1578cb87fc7dab222dfe67dd3b87e4c19fc36277efa339f10692bd2fd77c3d3218fd4201e31229d6d37b75ad209b200cc87b1229b

data/lib/aranha/address_processor.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class AddressProcessor
+    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
+    CORE_EXCEPTIONS = [::SocketError].freeze
+    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
+    HTTPCLIENT_EXCEPTIONS = [
+      ::HTTPClient::BadResponseError,
+      ::HTTPClient::ConnectTimeoutError,
+      ::HTTPClient::ReceiveTimeoutError
+    ].freeze
+    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
+
+    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
+                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
+
+    class << self
+      def rescuable_error?(error)
+        return true if NETWORK_EXCEPTIONS.any? { |klass| error.is_a?(klass) }
+
+        error.cause.present? ? network_error?(error.cause) : false
+      end
+    end
+
+    enable_simple_cache
+    common_constructor :address
+
+    def successful?
+      error.blank?
+    end
+
+    def rescuable_error?
+      self.class.rescuable_error?(error)
+    end
+
+    private
+
+    def error_uncached
+      address.process
+      nil
+    rescue ::StandardError => e
+      e
+    end
+  end
+end

data/lib/aranha/processor.rb

@@ -7,19 +7,6 @@ require 'aranha/manager'
 
 module Aranha
   class Processor
-    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
-    CORE_EXCEPTIONS = [::SocketError].freeze
-    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
-    HTTPCLIENT_EXCEPTIONS = [
-      ::HTTPClient::BadResponseError,
-      ::HTTPClient::ConnectTimeoutError,
-      ::HTTPClient::ReceiveTimeoutError
-    ].freeze
-    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
-
-    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
-                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
-
     DEFAULT_MAX_TRIES = 3
 
     attr_reader :manager
@@ -58,20 +45,20 @@ module Aranha
     def process_address(address)
       manager.log_info("Processing #{address} (Try: #{@try}/#{max_tries_s}," \
           " Unprocessed: #{unprocessed.count}/#{::Aranha::Manager.default.addresses_count})")
-      begin
-        address.process
-        @failed.delete(address.id)
-      rescue StandardError => e
-        process_exception(address, e)
+      ap = ::Aranha::AddressProcessor.new(address)
+      if ap.successful?
+        @failed.delete(ap.address.id)
+      else
+        process_exception(ap)
       end
     end
 
-    def process_exception(address, exception)
-      raise exception unless network_exception?(exception)
+    def process_exception(address_processor)
+      raise address_processor.error unless address_processor.rescuable_error?
 
-      @failed[address.id] ||= 0
-      @failed[address.id] += 1
-      manager.log_warn(exception)
+      @failed[address_processor.address.id] ||= 0
+      @failed[address_processor.address.id] += 1
+      manager.log_warn(address_processor.error)
     end
 
     def next_address
@@ -82,12 +69,6 @@ module Aranha
       ::Aranha::Manager.default.unprocessed_addresses
     end
 
-    def network_exception?(exception)
-      return true if NETWORK_EXCEPTIONS.any? { |klass| exception.is_a?(klass) }
-
-      exception.cause.present? ? network_exception?(exception.cause) : false
-    end
-
     def not_try_ids
       @failed.select { |_k, v| v > @try }.map { |k, _v| k }
     end

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.15.2'
+  VERSION = '0.16.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.15.2
+  version: 0.16.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-28 00:00:00.000000000 Z
+date: 2021-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aranha-parsers
@@ -96,6 +96,7 @@ files:
 - MIT-LICENSE
 - README.rdoc
 - lib/aranha.rb
+- lib/aranha/address_processor.rb
 - lib/aranha/default_processor.rb
 - lib/aranha/manager.rb
 - lib/aranha/processor.rb
@@ -119,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Ruby utilities for web crawling.