aranha 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/app/views/layouts/aranha/application.html.erb +9 -11
  3. data/lib/aranha.rb +1 -0
  4. data/lib/aranha/processor.rb +73 -3
  5. data/lib/aranha/version.rb +1 -1
  6. data/lib/tasks/aranha_tasks.rake +4 -0
  7. data/test/dummy/app/views/layouts/application.html.erb +9 -11
  8. data/test/dummy/config/application.rb +0 -1
  9. data/test/integration/navigation_test.rb +0 -1
  10. metadata +44 -30
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7776940b9f7bcb2d542e4993240fd10c18ff2024
4
- data.tar.gz: d6579b4134a9a4ce5ec98cf335db1d11ab1b7038
3
+ metadata.gz: c3565d60364c02cd8a739da311d56d7aad7277d9
4
+ data.tar.gz: 517009c9e93d8d639e21a94dc809c6443f51b00c
5
5
  SHA512:
6
- metadata.gz: 8d4924f70f27bb9a4809c2c56034fcaba0013b50925e8b204391034fa0cb88d5c2a3dc6de0fc365b8da8db5c81f45458c5b87b34ed058ab037dcc604d4f4fe0b
7
- data.tar.gz: f5774605515bc18a2b57c4c794e1a0d050026dd96d16d13e7eb8683492199a48032281995a773f8b937b8e9c90ae0514fd304759b4a27e830629caefd59e19c4
6
+ metadata.gz: 56f2404b53245fa4c9cda8095862c0059630518cecd25bf9b1018d5024060845319db12d3675ef7c378508790e279d461970e12227cb6ed1b698518f126e9323
7
+ data.tar.gz: b7017d7df069c1160d92be0fc76dd76a4f3fb31421128f430ddab7ffe7cece3eaea3ac1f8e7f2a82024498079f54cbfe3e136486751b51455603044fea13fef7
data/app/views/layouts/aranha/application.html.erb CHANGED
@@ -1,14 +1,12 @@
1
1
  <!DOCTYPE html>
2
2
  <html>
3
- <head>
4
- <title>Aranha</title>
5
- <%= stylesheet_link_tag "aranha/application", media: "all" %>
6
- <%= javascript_include_tag "aranha/application" %>
7
- <%= csrf_meta_tags %>
8
- </head>
9
- <body>
10
-
11
- <%= yield %>
12
-
13
- </body>
3
+ <head>
4
+ <title>Aranha</title>
5
+ <%= stylesheet_link_tag "aranha/application", media: "all" %>
6
+ <%= javascript_include_tag "aranha/application" %>
7
+ <%= csrf_meta_tags %>
8
+ </head>
9
+ <body>
10
+ <%= yield %>
11
+ </body>
14
12
  </html>
data/lib/aranha.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ require 'httpclient'
2
3
  require 'active_support/dependencies'
3
4
  require_dependency 'aranha/engine'
4
5
  require_dependency 'active_scaffold'
data/lib/aranha/processor.rb CHANGED
@@ -1,14 +1,84 @@
1
1
  # frozen_string_literal: true
2
2
  module Aranha
3
3
  class Processor
4
+ NETWORK_EXCEPTIONS = [::HTTPClient::BadResponseError, Errno::ECONNRESET].freeze
5
+ DEFAULT_MAX_TRIES = 3
6
+
4
7
  def initialize
5
8
  ::Aranha::Address.clear_expired
6
9
  ::Aranha::Address.add_start_points
10
+ @failed = {}
11
+ @try = 0
12
+ process_loop
13
+ raise "Addresses failed: #{@failed.count}" if @failed.any?
14
+ end
15
+
16
+ private
17
+
18
+ def process_loop
19
+ Rails.logger.info("Max tries: #{max_tries_s}")
7
20
  loop do
8
- a = ::Aranha::Address.unprocessed.first
9
- break unless a
10
- Rails.logger.info("Processing #{a}")
21
+ break if process_next_address
22
+ end
23
+ end
24
+
25
+ def process_next_address
26
+ a = next_address
27
+ if a
28
+ process_address(a)
29
+ false
30
+ elsif @failed.any?
31
+ @try += 1
32
+ max_tries > 0 && @try >= max_tries
33
+ else
34
+ true
35
+ end
36
+ end
37
+
38
+ def process_address(a)
39
+ Rails.logger.info("Processing #{a} (Try: #{@try}/#{max_tries_s}," \
40
+ " Unprocessed: #{unprocessed.count}/#{Aranha::Address.count})")
41
+ begin
11
42
  a.process
43
+ @failed.delete(a.id)
44
+ rescue StandardError => ex
45
+ process_exception(a, ex)
46
+ end
47
+ end
48
+
49
+ def process_exception(a, ex)
50
+ raise ex unless network_exception?(ex)
51
+ @failed[a.id] ||= 0
52
+ @failed[a.id] += 1
53
+ Rails.logger.warn(ex)
54
+ end
55
+
56
+ def next_address
57
+ unprocessed.where.not(id: not_try_ids).first
58
+ end
59
+
60
+ def unprocessed
61
+ ::Aranha::Address.unprocessed
62
+ end
63
+
64
+ def network_exception?(ex)
65
+ NETWORK_EXCEPTIONS.any? { |klass| ex.is_a?(klass) }
66
+ end
67
+
68
+ def not_try_ids
69
+ @failed.select { |_k, v| v > @try }.map { |k, _v| k }
70
+ end
71
+
72
+ def max_tries_s
73
+ max_tries <= 0 ? 'INF' : max_tries
74
+ end
75
+
76
+ def max_tries
77
+ @max_tries ||= begin
78
+ r = Integer(ENV['ARANHA_MAX_TRIES'])
79
+ r <= 0 ? 0 : r
80
+ rescue ArgumentError
81
+ DEFAULT_MAX_TRIES
12
82
  end
13
83
  end
14
84
  end
data/lib/aranha/version.rb CHANGED
@@ -1,4 +1,4 @@
1
1
  # frozen_string_literal: true
2
2
  module Aranha
3
- VERSION = '0.0.1'
3
+ VERSION = '0.0.2'
4
4
  end
data/lib/tasks/aranha_tasks.rake CHANGED
@@ -3,4 +3,8 @@ namespace(:aranha) do
3
3
  task process: :environment do
4
4
  ::Aranha::Processor.new
5
5
  end
6
+
7
+ task clear: :environment do
8
+ Rails.logger.info("Addresses deleted: #{::Aranha::Address.destroy_all.count}")
9
+ end
6
10
  end
data/test/dummy/app/views/layouts/application.html.erb CHANGED
@@ -1,14 +1,12 @@
1
1
  <!DOCTYPE html>
2
2
  <html>
3
- <head>
4
- <title>Dummy</title>
5
- <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track' => true %>
6
- <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %>
7
- <%= csrf_meta_tags %>
8
- </head>
9
- <body>
10
-
11
- <%= yield %>
12
-
13
- </body>
3
+ <head>
4
+ <title>Dummy</title>
5
+ <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track' => true %>
6
+ <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %>
7
+ <%= csrf_meta_tags %>
8
+ </head>
9
+ <body>
10
+ <%= yield %>
11
+ </body>
14
12
  </html>
data/test/dummy/config/application.rb CHANGED
@@ -23,4 +23,3 @@ module Dummy
23
23
  config.active_record.raise_in_transactional_callbacks = true
24
24
  end
25
25
  end
26
-
data/test/integration/navigation_test.rb CHANGED
@@ -5,4 +5,3 @@ class NavigationTest < ActionDispatch::IntegrationTest
5
5
  # assert true
6
6
  # end
7
7
  end
8
-
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: aranha
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eduardo H. Bogoni
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-12-18 00:00:00.000000000 Z
11
+ date: 2018-02-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: active_scaffold
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: 4.2.10
41
+ - !ruby/object:Gem::Dependency
42
+ name: httpclient
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '2.6'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '2.6'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: sqlite3
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -135,47 +149,47 @@ required_rubygems_version: !ruby/object:Gem::Requirement
135
149
  version: '0'
136
150
  requirements: []
137
151
  rubyforge_project:
138
- rubygems_version: 2.6.12
152
+ rubygems_version: 2.4.8
139
153
  signing_key:
140
154
  specification_version: 4
141
155
  summary: Rails utilities for web crawling.
142
156
  test_files:
143
- - test/dummy/Rakefile
144
- - test/dummy/README.rdoc
157
+ - test/integration/navigation_test.rb
145
158
  - test/dummy/config.ru
159
+ - test/dummy/README.rdoc
160
+ - test/dummy/db/schema.rb
161
+ - test/dummy/app/views/layouts/application.html.erb
162
+ - test/dummy/app/assets/stylesheets/application.css
163
+ - test/dummy/app/assets/javascripts/application.js
164
+ - test/dummy/app/helpers/application_helper.rb
165
+ - test/dummy/app/controllers/application_controller.rb
166
+ - test/dummy/bin/bundle
167
+ - test/dummy/bin/rails
168
+ - test/dummy/bin/setup
169
+ - test/dummy/bin/rake
170
+ - test/dummy/Rakefile
171
+ - test/dummy/config/environments/production.rb
172
+ - test/dummy/config/environments/test.rb
173
+ - test/dummy/config/environments/development.rb
174
+ - test/dummy/config/application.rb
146
175
  - test/dummy/config/boot.rb
147
- - test/dummy/config/database.yml
148
- - test/dummy/config/secrets.yml
149
176
  - test/dummy/config/locales/en.yml
150
- - test/dummy/config/application.rb
151
- - test/dummy/config/environments/development.rb
152
- - test/dummy/config/environments/test.rb
153
- - test/dummy/config/environments/production.rb
154
- - test/dummy/config/environment.rb
155
- - test/dummy/config/routes.rb
156
- - test/dummy/config/initializers/assets.rb
157
- - test/dummy/config/initializers/cookies_serializer.rb
158
- - test/dummy/config/initializers/inflections.rb
159
177
  - test/dummy/config/initializers/session_store.rb
160
- - test/dummy/config/initializers/wrap_parameters.rb
161
- - test/dummy/config/initializers/to_time_preserves_timezone.rb
162
178
  - test/dummy/config/initializers/filter_parameter_logging.rb
179
+ - test/dummy/config/initializers/wrap_parameters.rb
163
180
  - test/dummy/config/initializers/backtrace_silencers.rb
181
+ - test/dummy/config/initializers/inflections.rb
182
+ - test/dummy/config/initializers/to_time_preserves_timezone.rb
183
+ - test/dummy/config/initializers/assets.rb
184
+ - test/dummy/config/initializers/cookies_serializer.rb
164
185
  - test/dummy/config/initializers/mime_types.rb
165
- - test/dummy/db/schema.rb
166
- - test/dummy/app/views/layouts/application.html.erb
167
- - test/dummy/app/controllers/application_controller.rb
168
- - test/dummy/app/helpers/application_helper.rb
169
- - test/dummy/app/assets/stylesheets/application.css
170
- - test/dummy/app/assets/javascripts/application.js
186
+ - test/dummy/config/secrets.yml
187
+ - test/dummy/config/database.yml
188
+ - test/dummy/config/routes.rb
189
+ - test/dummy/config/environment.rb
171
190
  - test/dummy/public/422.html
172
- - test/dummy/public/404.html
173
191
  - test/dummy/public/favicon.ico
192
+ - test/dummy/public/404.html
174
193
  - test/dummy/public/500.html
175
- - test/dummy/bin/bundle
176
- - test/dummy/bin/setup
177
- - test/dummy/bin/rails
178
- - test/dummy/bin/rake
179
194
  - test/aranha_test.rb
180
195
  - test/test_helper.rb
181
- - test/integration/navigation_test.rb