aranha 0.15.0 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5fac0411750b6def655452009d3d091801905c696158499d8b4b26f99ccc426
-  data.tar.gz: cb8cd648b8603cfd1f578ba92e5017e37d6e1e19427b176d33e097e413c17baf
+  metadata.gz: 33a45e4db83f3d2743b53386005803da35188866dad28e15313a56b747efd4d1
+  data.tar.gz: 77ac8f2bcd922652fe62999f43d148a6557fb66d62e9edb21ba51d00ad5cb6d5
 SHA512:
-  metadata.gz: c4864a35aa117b9bb00d544c013ae6caaf356b3d674657d5b1d5a3371f644b949b4fa32b6ec97d5f44fa58f65ea71db0499f18208ed885103a4f93d891d41309
-  data.tar.gz: c20f17bb1c3d04ce9b7678d2bec181c74dcf3b930202f9c838f5046cd68b326eb660ad490c4ad4c92647f60474eefc705e6e2734a62f3b42541760d689ab9686
+  metadata.gz: c9424bbcd6540d5627052427cdbc746db9f36470e35e3c6deed3b2668160de22b2163e3f0a8b5ef03c13681bb05bb42b20c1a7348b71e616facdd82bb464e1fc
+  data.tar.gz: 54800ff8cc979371da63d2cea2543951f08f3f9a6423d0b20de7a9399cdc19338775f087d9a6791bcd3a0ec27d0500983fd47056f4438d9603e1fac29879a7cb

data/lib/aranha/address_processor.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class AddressProcessor
+    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
+    CORE_EXCEPTIONS = [::SocketError].freeze
+    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
+    HTTPCLIENT_EXCEPTIONS = [
+      ::HTTPClient::BadResponseError,
+      ::HTTPClient::ConnectTimeoutError,
+      ::HTTPClient::ReceiveTimeoutError
+    ].freeze
+    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
+
+    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
+                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
+
+    class << self
+      def rescuable_error?(error)
+        return true if NETWORK_EXCEPTIONS.any? { |klass| error.is_a?(klass) }
+
+        error.cause.present? ? network_error?(error.cause) : false
+      end
+    end
+
+    enable_simple_cache
+    common_constructor :address
+
+    def successful?
+      error.blank?
+    end
+
+    def rescuable_error?
+      self.class.rescuable_error?(error)
+    end
+
+    private
+
+    def error_uncached
+      address.process
+      nil
+    rescue ::StandardError => e
+      e
+    end
+  end
+end

data/lib/aranha/default_processor.rb

@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
 require 'addressable'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   class DefaultProcessor
-    attr_reader :source_uri, :extra_data
-
     class << self
       def sanitize_uri(uri)
         return uri if uri.is_a?(Hash)
@@ -15,17 +14,14 @@ module Aranha
       end
     end
 
-    def initialize(source_uri, extra_data)
-      @source_uri = self.class.sanitize_uri(source_uri)
-      @extra_data = extra_data
+    common_constructor :source_uri, :extra_data do
+      self.source_uri = self.class.sanitize_uri(source_uri)
     end
 
     def process
       raise 'Implement method process'
     end
 
-    protected
-
     def target_uri
       source_uri
     end

data/lib/aranha/processor.rb

@@ -7,19 +7,6 @@ require 'aranha/manager'
 
 module Aranha
   class Processor
-    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
-    CORE_EXCEPTIONS = [::SocketError].freeze
-    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
-    HTTPCLIENT_EXCEPTIONS = [
-      ::HTTPClient::BadResponseError,
-      ::HTTPClient::ConnectTimeoutError,
-      ::HTTPClient::ReceiveTimeoutError
-    ].freeze
-    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
-
-    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
-                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
-
     DEFAULT_MAX_TRIES = 3
 
     attr_reader :manager
@@ -58,20 +45,20 @@ module Aranha
     def process_address(address)
       manager.log_info("Processing #{address} (Try: #{@try}/#{max_tries_s}," \
           " Unprocessed: #{unprocessed.count}/#{::Aranha::Manager.default.addresses_count})")
-      begin
-        address.process
-        @failed.delete(address.id)
-      rescue StandardError => e
-        process_exception(address, e)
+      ap = ::Aranha::AddressProcessor.new(address)
+      if ap.successful?
+        @failed.delete(ap.address.id)
+      else
+        process_exception(ap)
       end
     end
 
-    def process_exception(address, exception)
-      raise exception unless network_exception?(exception)
+    def process_exception(address_processor)
+      raise address_processor.error unless address_processor.rescuable_error?
 
-      @failed[address.id] ||= 0
-      @failed[address.id] += 1
-      manager.log_warn(exception)
+      @failed[address_processor.address.id] ||= 0
+      @failed[address_processor.address.id] += 1
+      manager.log_warn(address_processor.error)
     end
 
     def next_address
@@ -82,12 +69,6 @@ module Aranha
       ::Aranha::Manager.default.unprocessed_addresses
     end
 
-    def network_exception?(exception)
-      return true if NETWORK_EXCEPTIONS.any? { |klass| exception.is_a?(klass) }
-
-      exception.cause.present? ? network_exception?(exception.cause) : false
-    end
-
     def not_try_ids
       @failed.select { |_k, v| v > @try }.map { |k, _v| k }
     end

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.15.0'
+  VERSION = '0.16.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.1
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-28 00:00:00.000000000 Z
+date: 2021-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aranha-parsers
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -78,14 +78,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
 description: 
 email:
 - eduardobogoni@gmail.com
@@ -96,6 +102,7 @@ files:
 - MIT-LICENSE
 - README.rdoc
 - lib/aranha.rb
+- lib/aranha/address_processor.rb
 - lib/aranha/default_processor.rb
 - lib/aranha/manager.rb
 - lib/aranha/processor.rb
@@ -119,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Ruby utilities for web crawling.