aranha 0.15.2 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf75a29a21e24f2e85a425b8b9291c058a46eef895a8100992514dd03138b065
-  data.tar.gz: d52cc8cf889e0d9174f13771e3c9ac15971a1f673dc8e4e15085aab3755e169b
+  metadata.gz: 31001c1c5af252c6cb791aa0a362177859b9f8ff647fe3150df6f6515d81904a
+  data.tar.gz: eb574bcb2047e5cf2de295ea437d9e70d5cb680f6341e853da7097b7b2e181b8
 SHA512:
-  metadata.gz: 4844ac22fc5d9b3a16e7b03d8da6e5e5fc82a95dd705a0bbbb11c029f206fdc6fb762d1fd2cd71a2c8f869084597ad3f8d65e88f79f95bf37dfcd02a67cc59c6
-  data.tar.gz: c768ba132ef0b9afbf9630e62aeb4759ab5bdb09e8d6af2658c59b19360c2a9c11dd3c1abd49431111fa25273c895ede8eb2536d7f63ebd607a1965c9e9a0813
+  metadata.gz: d9ec9528940576d0502b37ead25eb749118211ca5d15cf3d17cb23273d789309c7507e2f74d8053e17c5709766c0b87dabe6d64180df4a43ce060470f556029d
+  data.tar.gz: d5ec7d1b7432ecbfbc8e10864214aee4ce9b256b5e359b3ac11f4411b2279e44b20369c17c67cdc08fe9fda52225a1a408ec4977adc0825cdd28da5bcda8ed26

data/lib/aranha/address_processor.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class AddressProcessor
+    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
+    CORE_EXCEPTIONS = [::SocketError].freeze
+    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
+    HTTPCLIENT_EXCEPTIONS = [
+      ::HTTPClient::BadResponseError,
+      ::HTTPClient::ConnectTimeoutError,
+      ::HTTPClient::ReceiveTimeoutError
+    ].freeze
+    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
+
+    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
+                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
+
+    class << self
+      def rescuable_error?(error)
+        return true if NETWORK_EXCEPTIONS.any? { |klass| error.is_a?(klass) }
+
+        error.cause.present? ? network_error?(error.cause) : false
+      end
+    end
+
+    enable_simple_cache
+    common_constructor :address
+
+    def successful?
+      error.blank?
+    end
+
+    def rescuable_error?
+      self.class.rescuable_error?(error)
+    end
+
+    private
+
+    def error_uncached
+      address.process
+      nil
+    rescue ::StandardError => e
+      e
+    end
+  end
+end

data/lib/aranha/default_processor.rb

@@ -18,6 +18,8 @@ module Aranha
       self.source_uri = self.class.sanitize_uri(source_uri)
     end
 
+    delegate :source_address, to: :parser
+
     def process
       raise 'Implement method process'
     end
@@ -27,7 +29,11 @@ module Aranha
     end
 
     def data
-      @data ||= parser_class.new(target_uri).data
+      @data ||= parser.data
+    end
+
+    def parser
+      @parser ||= parser_class.new(target_uri)
     end
 
     def parser_class

data/lib/aranha/processor.rb

@@ -7,19 +7,6 @@ require 'aranha/manager'
 
 module Aranha
   class Processor
-    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
-    CORE_EXCEPTIONS = [::SocketError].freeze
-    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
-    HTTPCLIENT_EXCEPTIONS = [
-      ::HTTPClient::BadResponseError,
-      ::HTTPClient::ConnectTimeoutError,
-      ::HTTPClient::ReceiveTimeoutError
-    ].freeze
-    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
-
-    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
-                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
-
     DEFAULT_MAX_TRIES = 3
 
     attr_reader :manager
@@ -58,20 +45,20 @@ module Aranha
     def process_address(address)
       manager.log_info("Processing #{address} (Try: #{@try}/#{max_tries_s}," \
           " Unprocessed: #{unprocessed.count}/#{::Aranha::Manager.default.addresses_count})")
-      begin
-        address.process
-        @failed.delete(address.id)
-      rescue StandardError => e
-        process_exception(address, e)
+      ap = ::Aranha::AddressProcessor.new(address)
+      if ap.successful?
+        @failed.delete(ap.address.id)
+      else
+        process_exception(ap)
       end
     end
 
-    def process_exception(address, exception)
-      raise exception unless network_exception?(exception)
+    def process_exception(address_processor)
+      raise address_processor.error unless address_processor.rescuable_error?
 
-      @failed[address.id] ||= 0
-      @failed[address.id] += 1
-      manager.log_warn(exception)
+      @failed[address_processor.address.id] ||= 0
+      @failed[address_processor.address.id] += 1
+      manager.log_warn(address_processor.error)
     end
 
     def next_address
@@ -82,12 +69,6 @@ module Aranha
       ::Aranha::Manager.default.unprocessed_addresses
     end
 
-    def network_exception?(exception)
-      return true if NETWORK_EXCEPTIONS.any? { |klass| exception.is_a?(klass) }
-
-      exception.cause.present? ? network_exception?(exception.cause) : false
-    end
-
     def not_try_ids
       @failed.select { |_k, v| v > @try }.map { |k, _v| k }
     end

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.15.2'
+  VERSION = '0.17.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.15.2
+  version: 0.17.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-28 00:00:00.000000000 Z
+date: 2022-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aranha-parsers
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -78,14 +78,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
 description: 
 email:
 - eduardobogoni@gmail.com
@@ -96,6 +102,7 @@ files:
 - MIT-LICENSE
 - README.rdoc
 - lib/aranha.rb
+- lib/aranha/address_processor.rb
 - lib/aranha/default_processor.rb
 - lib/aranha/manager.rb
 - lib/aranha/processor.rb
@@ -119,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Ruby utilities for web crawling.