eac_tools 0.117.2 → 0.117.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bbec5a523a23d7b9d6e27eb3bf376e53b226162f3785a984db4a54efafb13cc
-  data.tar.gz: 70b40c961629634334e2df07fde8dfde01503c0697c1d48c2a5e0b833ea52a79
+  metadata.gz: 84e29eceef86e4a0712f5075518bea8ec19bc37301da8ad19f5d6278f089eaab
+  data.tar.gz: 94dda91b9546c51ebf700438016cc8b6e20b56387891b407bdd3d510be35d4b3
 SHA512:
-  metadata.gz: d46470727d52353fad5d8a08a89fa0e6bf26e7004071e91a83cacb947dbdd0d1863d069793f887e251234c438e5cdd83274511e8f91a8304da1ae05b30c98e96
-  data.tar.gz: c4121f58cb6669f53d5153da7e2728684e891f8f4ef595f54958d210594b534f3536c220db34fbe63cbdbe1310eca3a9d03de9d57e3d7bd557c4cb55a0696eb5
+  metadata.gz: b503949aa1be8d78ada4ebbce6d029711131ed6ed5b3118796c0152e6dec1c263d6f82b9b4fcec2b29817c1d585a70078a82cf74dd2023ff275a4f04dfe6d272
+  data.tar.gz: 7cc2be3a74aa830abab3c8d714d4b43df1e446064b0e7868054450b5a00a5186b0ef254a651681aa08caf1077be0bde9577cc887bca6b14fac72b24fe4db549b

data/Gemfile.lock

@@ -1,55 +1,142 @@
 PATH
   remote: .
   specs:
-    eac_tools (0.117.2)
-      avm (~> 0.102, >= 0.102.1)
-      avm-eac_asciidoctor_base0 (~> 0.27)
+    eac_tools (0.117.4)
+      avm (~> 0.102, >= 0.102.3)
+      avm-eac_asciidoctor_base0 (~> 0.27, >= 0.27.1)
       avm-eac_generic_base0 (~> 0.15, >= 0.15.3)
       avm-eac_github_base0 (~> 0.4, >= 0.4.1)
       avm-eac_gitlab_base0 (~> 0.6)
-      avm-eac_latex_base0 (~> 0.3, >= 0.3.2)
+      avm-eac_latex_base0 (~> 0.3, >= 0.3.3)
       avm-eac_php_base0 (~> 0.5)
       avm-eac_postgresql_base0 (~> 0.6)
       avm-eac_python_base0 (~> 0.2, >= 0.2.2)
       avm-eac_rails_base0 (~> 0.11, >= 0.11.2)
-      avm-eac_rails_base1 (~> 0.12)
-      avm-eac_redmine_base0 (~> 0.26)
+      avm-eac_rails_base1 (~> 0.12, >= 0.12.1)
+      avm-eac_redmine_base0 (~> 0.26, >= 0.26.1)
       avm-eac_redmine_plugin_base0 (~> 0.5)
-      avm-eac_ruby_base0 (~> 0.3)
-      avm-eac_ruby_base1 (~> 0.43)
-      avm-eac_webapp_base0 (~> 0.22, >= 0.22.2)
+      avm-eac_ruby_base0 (~> 0.3, >= 0.3.1)
+      avm-eac_ruby_base1 (~> 0.43, >= 0.43.2)
+      avm-eac_webapp_base0 (~> 0.22, >= 0.22.3)
       avm-eac_wordpress_base0 (~> 0.3, >= 0.3.2)
       avm-git (~> 0.21, >= 0.21.2)
-      avm-tools (~> 0.164)
+      avm-tools (~> 0.164, >= 0.164.1)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+
+PATH
+  remote: sub/aranha
+  specs:
+    aranha (0.20.1)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+
+PATH
+  remote: sub/avm-eac_asciidoctor_base0
+  specs:
+    avm-eac_asciidoctor_base0 (0.27.1)
+      asciidoctor (~> 2.0, >= 2.0.26)
+      avm-eac_webapp_base0 (~> 0.22, >= 0.22.3)
+      eac_fs (~> 0.20, >= 0.20.2)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+      os (~> 1.1, >= 1.1.4)
+      rouge (~> 3.30)
+
+PATH
+  remote: sub/avm-eac_latex_base0
+  specs:
+    avm-eac_latex_base0 (0.3.3)
+      avm (~> 0.102, >= 0.102.3)
+      avm-eac_webapp_base0 (~> 0.22, >= 0.22.3)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+
+PATH
+  remote: sub/avm-eac_rails_base1
+  specs:
+    avm-eac_rails_base1 (0.12.1)
+      avm (~> 0.102, >= 0.102.3)
+      avm-eac_ruby_base1 (~> 0.43, >= 0.43.2)
+      avm-eac_webapp_base0 (~> 0.22, >= 0.22.3)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+
+PATH
+  remote: sub/avm-eac_redmine_base0
+  specs:
+    avm-eac_redmine_base0 (0.26.1)
+      avm (~> 0.102, >= 0.102.3)
+      avm-eac_generic_base0 (~> 0.15, >= 0.15.3)
+      avm-eac_rails_base1 (~> 0.12, >= 0.12.1)
+      avm-eac_ubuntu_base0 (~> 0.7, >= 0.7.1)
+      eac_fs (~> 0.20, >= 0.20.2)
+      eac_rest (~> 0.13)
       eac_ruby_base1 (~> 0.1, >= 0.1.1)
 
 PATH
   remote: sub/avm-eac_ruby_base0
   specs:
-    avm-eac_ruby_base0 (0.3.0)
-      avm-eac_ruby_base1 (~> 0.43)
-      eac_ruby_utils (~> 0.131, >= 0.131.2)
+    avm-eac_ruby_base0 (0.3.1)
+      avm-eac_ruby_base1 (~> 0.43, >= 0.43.2)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
 
 PATH
   remote: sub/avm-eac_ruby_base1
   specs:
-    avm-eac_ruby_base1 (0.43.0)
+    avm-eac_ruby_base1 (0.43.2)
       aranha-parsers (~> 0.29)
-      avm (~> 0.102, >= 0.102.1)
+      avm (~> 0.102, >= 0.102.3)
       avm-eac_generic_base0 (~> 0.15, >= 0.15.3)
       eac_envs-http (~> 0.7, >= 0.7.1)
       eac_ruby_base1 (~> 0.1, >= 0.1.1)
 
 PATH
-  remote: sub/eac_ruby_gem_support
+  remote: sub/avm-eac_ubuntu_base0
   specs:
-    eac_ruby_gem_support (0.14.0)
-      eac_ruby_base1 (~> 0.1)
-      rspec (~> 3.13, >= 3.13.2)
-      rubocop (~> 1.88, >= 1.88.2)
-      rubocop-rails (~> 2.36)
-      rubocop-rspec (~> 3.10, >= 3.10.2)
-      super_diff (~> 0.19)
+    avm-eac_ubuntu_base0 (0.7.1)
+      avm (~> 0.102, >= 0.102.3)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+      eac_templates (~> 0.9)
+
+PATH
+  remote: sub/avm-eac_webapp_base0
+  specs:
+    avm-eac_webapp_base0 (0.22.3)
+      avm (~> 0.102, >= 0.102.3)
+      avm-eac_generic_base0 (~> 0.15, >= 0.15.3)
+      avm-eac_postgresql_base0 (~> 0.6)
+      avm-eac_ubuntu_base0 (~> 0.7, >= 0.7.1)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+      htmlbeautifier (~> 1.4, >= 1.4.3)
+
+PATH
+  remote: sub/avm-tools
+  specs:
+    avm-tools (0.164.1)
+      avm (~> 0.102, >= 0.102.2)
+      eac_ruby_base0 (~> 0.19, >= 0.19.3)
+
+PATH
+  remote: sub/avm
+  specs:
+    avm (0.102.3)
+      eac_cli (~> 0.47)
+      eac_config (~> 0.15)
+      eac_docker (~> 0.8, >= 0.8.1)
+      eac_fs (~> 0.20, >= 0.20.2)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+      eac_templates (~> 0.9)
+      minitar (~> 0.12, >= 0.12.1)
+
+PATH
+  remote: sub/eac_docker
+  specs:
+    eac_docker (0.8.1)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
+      eac_templates (~> 0.9)
+
+PATH
+  remote: sub/eac_templates
+  specs:
+    eac_templates (0.9.0)
+      eac_config (~> 0.15)
+      eac_ruby_base1 (~> 0.1, >= 0.1.1)
 
 GEM
   remote: https://rubygems.org/
@@ -69,8 +156,6 @@ GEM
       uri (>= 0.13.1)
     addressable (2.9.0)
       public_suffix (>= 2.0.2, < 8.0)
-    aranha (0.20.0)
-      eac_ruby_utils (~> 0.128)
     aranha-parsers (0.29.0)
       activesupport (>= 4.0.0)
       addressable (~> 2.9)
@@ -81,21 +166,6 @@ GEM
     asciidoctor (2.0.26)
     ast (2.4.3)
     attr_extras (7.1.0)
-    avm (0.102.1)
-      eac_cli (~> 0.47)
-      eac_config (~> 0.15)
-      eac_docker (~> 0.8)
-      eac_fs (~> 0.20, >= 0.20.2)
-      eac_ruby_base1 (~> 0.1)
-      eac_templates (~> 0.8, >= 0.8.3)
-      minitar (~> 0.12, >= 0.12.1)
-    avm-eac_asciidoctor_base0 (0.27.0)
-      asciidoctor (~> 2.0, >= 2.0.26)
-      avm-eac_webapp_base0 (~> 0.22, >= 0.22.1)
-      eac_fs (~> 0.20, >= 0.20.1)
-      eac_ruby_utils (~> 0.130)
-      os (~> 1.1, >= 1.1.4)
-      rouge (~> 3.30)
     avm-eac_generic_base0 (0.15.3)
       avm (~> 0.100)
       eac_ruby_utils (~> 0.131)
@@ -107,10 +177,6 @@ GEM
     avm-eac_gitlab_base0 (0.6.0)
       avm-eac_github_base0 (~> 0.4)
       eac_ruby_utils (~> 0.130)
-    avm-eac_latex_base0 (0.3.2)
-      avm (~> 0.84, >= 0.84.2)
-      avm-eac_webapp_base0 (~> 0.18, >= 0.18.3)
-      eac_ruby_utils (~> 0.121)
     avm-eac_php_base0 (0.5.0)
       avm-eac_generic_base0 (~> 0.13)
       avm-eac_webapp_base0 (~> 0.20)
@@ -124,33 +190,9 @@ GEM
     avm-eac_rails_base0 (0.11.2)
       avm-eac_rails_base1 (~> 0.11, >= 0.11.2)
       eac_ruby_utils (~> 0.129, >= 0.129.1)
-    avm-eac_rails_base1 (0.12.0)
-      avm (~> 0.98, >= 0.98.4)
-      avm-eac_ruby_base1 (~> 0.39)
-      avm-eac_webapp_base0 (~> 0.22)
-      eac_ruby_utils (~> 0.130)
-    avm-eac_redmine_base0 (0.26.0)
-      avm (~> 0.98, >= 0.98.5)
-      avm-eac_generic_base0 (~> 0.15, >= 0.15.2)
-      avm-eac_rails_base1 (~> 0.12)
-      avm-eac_ubuntu_base0 (~> 0.7)
-      eac_fs (~> 0.20, >= 0.20.1)
-      eac_rest (~> 0.13)
-      eac_ruby_utils (~> 0.130)
     avm-eac_redmine_plugin_base0 (0.5.0)
       avm-eac_ruby_base1 (~> 0.38, >= 0.38.1)
       eac_ruby_utils (~> 0.128, >= 0.128.6)
-    avm-eac_ubuntu_base0 (0.7.0)
-      avm (~> 0.98, >= 0.98.2)
-      eac_ruby_utils (~> 0.128, >= 0.128.6)
-      eac_templates (~> 0.8, >= 0.8.3)
-    avm-eac_webapp_base0 (0.22.2)
-      avm (~> 0.100)
-      avm-eac_generic_base0 (~> 0.15, >= 0.15.2)
-      avm-eac_postgresql_base0 (~> 0.6)
-      avm-eac_ubuntu_base0 (~> 0.7)
-      eac_ruby_utils (~> 0.131)
-      htmlbeautifier (~> 1.4, >= 1.4.3)
     avm-eac_wordpress_base0 (0.3.2)
       avm (~> 0.86, >= 0.86.2)
       avm-eac_webapp_base0 (~> 0.18, >= 0.18.4)
@@ -161,9 +203,6 @@ GEM
       eac_ruby_utils (~> 0.129, >= 0.129.1)
       filesize
       git (~> 1.19, >= 1.19.1)
-    avm-tools (0.164.0)
-      avm (~> 0.98, >= 0.98.5)
-      eac_ruby_base0 (~> 0.19, >= 0.19.3)
     base64 (0.3.0)
     bigdecimal (4.1.2)
     clipboard (2.0.0)
@@ -183,9 +222,6 @@ GEM
     eac_config (0.15.0)
       addressable (~> 2.8, >= 2.8.7)
       eac_ruby_utils (~> 0.127)
-    eac_docker (0.8.0)
-      eac_ruby_utils (~> 0.128, >= 0.128.2)
-      eac_templates (~> 0.8, >= 0.8.1)
     eac_envs-http (0.7.1)
       eac_fs (~> 0.20)
       eac_ruby_utils (~> 0.129)
@@ -215,16 +251,20 @@ GEM
       eac_ruby_utils (~> 0.129)
     eac_ruby_base1 (0.1.1)
       eac_ruby_utils (~> 0.131, >= 0.131.2)
-    eac_ruby_utils (0.131.2)
+    eac_ruby_gem_support (0.14.0)
+      eac_ruby_base1 (~> 0.1)
+      rspec (~> 3.13, >= 3.13.2)
+      rubocop (~> 1.88, >= 1.88.2)
+      rubocop-rails (~> 2.36)
+      rubocop-rspec (~> 3.10, >= 3.10.2)
+      super_diff (~> 0.19)
+    eac_ruby_utils (0.131.3)
       activesupport (>= 6.1.7.10)
       addressable (~> 2.9)
       bundler
       memoized (~> 1.1, >= 1.1.3)
       net-ssh (~> 4.2)
-      zeitwerk (~> 2.7, >= 2.7.5)
-    eac_templates (0.8.3)
-      eac_config (~> 0.15)
-      eac_ruby_utils (~> 0.128, >= 0.128.5)
+      zeitwerk (~> 2.8, >= 2.8.2)
     faraday (2.14.3)
       faraday-net_http (>= 2.0, < 3.5)
       json
@@ -353,9 +393,20 @@ PLATFORMS
   x86_64-linux-gnu
 
 DEPENDENCIES
+  aranha!
+  avm!
+  avm-eac_asciidoctor_base0!
+  avm-eac_latex_base0!
+  avm-eac_rails_base1!
+  avm-eac_redmine_base0!
   avm-eac_ruby_base0!
   avm-eac_ruby_base1!
-  eac_ruby_gem_support (~> 0.14, >= 0)!
+  avm-eac_ubuntu_base0!
+  avm-eac_webapp_base0!
+  avm-tools!
+  eac_docker!
+  eac_ruby_gem_support (~> 0.14)
+  eac_templates!
   eac_tools!
 
 BUNDLED WITH

data/eac_tools.gemspec

@@ -18,25 +18,25 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = '>= 3.2'
 
-  s.add_dependency 'avm', '~> 0.102', '>= 0.102.1'
-  s.add_dependency 'avm-eac_asciidoctor_base0', '~> 0.27'
+  s.add_dependency 'avm', '~> 0.102', '>= 0.102.3'
+  s.add_dependency 'avm-eac_asciidoctor_base0', '~> 0.27', '>= 0.27.1'
   s.add_dependency 'avm-eac_generic_base0', '~> 0.15', '>= 0.15.3'
   s.add_dependency 'avm-eac_github_base0', '~> 0.4', '>= 0.4.1'
   s.add_dependency 'avm-eac_gitlab_base0', '~> 0.6'
-  s.add_dependency 'avm-eac_latex_base0', '~> 0.3', '>= 0.3.2'
+  s.add_dependency 'avm-eac_latex_base0', '~> 0.3', '>= 0.3.3'
   s.add_dependency 'avm-eac_php_base0', '~> 0.5'
   s.add_dependency 'avm-eac_postgresql_base0', '~> 0.6'
   s.add_dependency 'avm-eac_python_base0', '~> 0.2', '>= 0.2.2'
   s.add_dependency 'avm-eac_rails_base0', '~> 0.11', '>= 0.11.2'
-  s.add_dependency 'avm-eac_rails_base1', '~> 0.12'
-  s.add_dependency 'avm-eac_redmine_base0', '~> 0.26'
+  s.add_dependency 'avm-eac_rails_base1', '~> 0.12', '>= 0.12.1'
+  s.add_dependency 'avm-eac_redmine_base0', '~> 0.26', '>= 0.26.1'
   s.add_dependency 'avm-eac_redmine_plugin_base0', '~> 0.5'
-  s.add_dependency 'avm-eac_ruby_base0', '~> 0.3'
-  s.add_dependency 'avm-eac_ruby_base1', '~> 0.43'
-  s.add_dependency 'avm-eac_webapp_base0', '~> 0.22', '>= 0.22.2'
+  s.add_dependency 'avm-eac_ruby_base0', '~> 0.3', '>= 0.3.1'
+  s.add_dependency 'avm-eac_ruby_base1', '~> 0.43', '>= 0.43.2'
+  s.add_dependency 'avm-eac_webapp_base0', '~> 0.22', '>= 0.22.3'
   s.add_dependency 'avm-eac_wordpress_base0', '~> 0.3', '>= 0.3.2'
   s.add_dependency 'avm-git', '~> 0.21', '>= 0.21.2'
-  s.add_dependency 'avm-tools', '~> 0.164'
+  s.add_dependency 'avm-tools', '~> 0.164', '>= 0.164.1'
   s.add_dependency 'eac_ruby_base1', '~> 0.1', '>= 0.1.1'
 
   s.add_development_dependency 'eac_ruby_gem_support', '~> 0.14'

data/lib/eac_tools/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EacTools
-  VERSION = '0.117.2'
+  VERSION = '0.117.4'
 end

data/sub/aranha/.gitignore

@@ -0,0 +1,5 @@
+.bundle/
+Gemfile.local
+Gemfile.lock
+log/*.log
+pkg/

data/sub/aranha/.gitrepo

@@ -0,0 +1,7 @@
+[subrepo]
+  cmdver = 0.4.9
+  commit = 52ce301beeaeb535e13e49a1a420d045b67a7206
+  method = merge
+  parent = 7ee9f9e1afdb9ec7eaea9e7bdbb1c148dc2404ae
+  branch = master
+  remote = https://github.com/esquilo-azul/aranha.git

data/sub/aranha/.rspec

@@ -0,0 +1 @@
+--require spec/spec_helper --color --format doc --default-path . --pattern 'spec/**/*_spec.rb'

data/sub/aranha/.rubocop.yml

@@ -0,0 +1,3 @@
+---
+inherit_gem:
+  eac_ruby_gem_support: ".rubocop.yml"

data/sub/aranha/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2017 Eduardo H. Bogoni
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/sub/aranha/README.rdoc

@@ -0,0 +1,3 @@
+= Aranha
+
+Ruby utilities for web crawling.

data/sub/aranha/aranha.gemspec

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path('lib', __dir__)
+
+# Maintain your gem's version:
+require 'aranha/version'
+
+# Describe your gem and declare its dependencies:
+Gem::Specification.new do |s|
+  s.name        = 'aranha'
+  s.version     = Aranha::VERSION
+  s.authors     = ['Eduardo H. Bogoni']
+  s.email       = ['eduardobogoni@gmail.com']
+  s.summary     = 'Ruby utilities for web crawling.'
+  s.license     = 'MIT'
+
+  s.files = Dir['{app,config,db,lib}/**/*', 'MIT-LICENSE', 'README.rdoc']
+  s.required_ruby_version = '>= 3.2'
+
+  s.add_dependency 'eac_ruby_base1', '~> 0.1', '>= 0.1.1'
+
+  s.add_development_dependency 'eac_ruby_gem_support', '~> 0.14'
+end

data/sub/aranha/lib/aranha/address_processor.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Aranha
+  class AddressProcessor
+    enable_simple_cache
+    common_constructor :address
+
+    def successful?
+      error.blank?
+    end
+
+    def rescuable_error?
+      ::Aranha::TemporaryErrorsManager.temporary_error?(error)
+    end
+
+    private
+
+    def error_uncached
+      address.process
+      nil
+    rescue ::StandardError => e
+      e
+    end
+  end
+end

data/sub/aranha/lib/aranha/default_processor.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Aranha
+  class DefaultProcessor
+    class << self
+      def sanitize_uri(uri)
+        return uri if uri.is_a?(Hash)
+
+        uri = uri.to_s.gsub(%r{\A/}, 'file:///') unless uri.is_a?(Addressable::URI)
+        Addressable::URI.parse(uri)
+      end
+    end
+
+    common_constructor :source_uri, :extra_data do
+      self.source_uri = self.class.sanitize_uri(source_uri)
+    end
+
+    delegate :source_address, to: :parser
+
+    def process
+      raise 'Implement method process'
+    end
+
+    def target_uri
+      source_uri
+    end
+
+    def data
+      @data ||= parser.data
+    end
+
+    def parser
+      @parser ||= parser_class.new(target_uri)
+    end
+
+    def parser_class
+      r = self.class.name.gsub('::Processors::', '::Parsers::').constantize
+      return r unless is_a?(r)
+
+      raise "Parser can be not the process class: #{r}"
+    end
+  end
+end

data/sub/aranha/lib/aranha/manager.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Aranha
+  class Manager
+    acts_as_abstract
+
+    class << self
+      attr_accessor :default
+    end
+
+    def addresses_count
+      raise_abstract_method(__method__)
+    end
+
+    def add_address(_uri, _processor_class, _extra_data = nil)
+      raise_abstract_method(__method__)
+    end
+
+    def add_start_point(uri, processor_class, extra_data = nil)
+      start_points_var << ::EacRubyUtils::Struct.new(
+        uri: uri, processor_class: processor_class, extra_data: extra_data
+      )
+    end
+
+    def clear_expired_addresses
+      raise_abstract_method(__method__)
+    end
+
+    def init
+      clear_expired_addresses
+      start_points_to_addresses
+    end
+
+    def log_info(_message)
+      raise_abstract_method(__method__)
+    end
+
+    def log_warn(_message)
+      raise_abstract_method(__method__)
+    end
+
+    def start_points
+      start_points_var.to_enum
+    end
+
+    def start_points_to_addresses
+      start_points_var.each do |sp|
+        add_address(sp.uri, sp.processor_class, sp.extra_data)
+      end
+    end
+
+    def unprocessed_addresses
+      raise_abstract_method(__method__)
+    end
+
+    private
+
+    def start_points_var
+      @start_points_var ||= []
+    end
+  end
+end

data/sub/aranha/lib/aranha/processor.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Aranha
+  class Processor
+    DEFAULT_MAX_TRIES = 3
+
+    attr_reader :manager
+
+    def initialize(manager = nil)
+      @manager = manager || ::Aranha::Manager.default
+      @failed = {}
+      @try = 0
+      self.manager.init
+      process_loop
+      raise "Addresses failed: #{@failed.count}" if @failed.any?
+    end
+
+    private
+
+    def process_loop
+      manager.log_info("Max tries: #{max_tries_s}")
+      loop do
+        break if process_next_address
+      end
+    end
+
+    def process_next_address # rubocop:disable Naming/PredicateMethod
+      a = next_address
+      if a
+        process_address(a)
+        false
+      elsif @failed.any?
+        @try += 1
+        max_tries.positive? && @try >= max_tries
+      else
+        true
+      end
+    end
+
+    def process_address(address)
+      manager.log_info("Processing #{address} (Try: #{@try}/#{max_tries_s}, " \
+                       "Unprocessed: #{unprocessed.count}" \
+                       "/#{::Aranha::Manager.default.addresses_count})")
+      ap = ::Aranha::AddressProcessor.new(address)
+      if ap.successful?
+        @failed.delete(ap.address.id)
+      else
+        process_exception(ap)
+      end
+    end
+
+    def process_exception(address_processor)
+      raise address_processor.error unless address_processor.rescuable_error?
+
+      @failed[address_processor.address.id] ||= 0
+      @failed[address_processor.address.id] += 1
+      manager.log_warn(address_processor.error)
+    end
+
+    def next_address
+      unprocessed.where.not(id: not_try_ids).first
+    end
+
+    def unprocessed
+      ::Aranha::Manager.default.unprocessed_addresses
+    end
+
+    def not_try_ids
+      @failed.select { |_k, v| v > @try }.map { |k, _v| k }
+    end
+
+    def max_tries_s
+      max_tries <= 0 ? 'INF' : max_tries
+    end
+
+    def max_tries
+      @max_tries ||= begin
+        r = Integer(ENV.fetch('ARANHA_MAX_TRIES', nil))
+        [r, 0].max
+      rescue ArgumentError, TypeError
+        DEFAULT_MAX_TRIES
+      end
+    end
+  end
+end

data/sub/aranha/lib/aranha/temporary_errors.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Aranha
+  module TemporaryErrors
+    CORE_ERRORS = [::SocketError].freeze
+    ERRNO_ERRORS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
+    NET_ERRORS = [::Net::HTTPFatalError, Net::HTTPClientException, ::Net::OpenTimeout].freeze
+
+    ALL_ERRORS = CORE_ERRORS + ERRNO_ERRORS + NET_ERRORS
+
+    class << self
+      def errors
+        ALL_ERRORS
+      end
+    end
+  end
+end