spidy 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43e5538d56f4d4115a1bb2cb662f0722d688ad44d8212506436492f0ee3e0570
-  data.tar.gz: b0db640263836c7244876a50219ed4f916c6c2f7777e11c7e13d11c2142714dc
+  metadata.gz: 44d16509821e1779d821effb7c571d2a11cbb79166168943242628b2e53457ec
+  data.tar.gz: 5b52389a8042c9e69aaeead0d96b81553da2e5f5e51ad16696f87249e38067bd
 SHA512:
-  metadata.gz: 3d6dcfea624a7710eec941e55238334596edced66d60a09b046c5b057de06c2fb7d279c3d912a8e054ef206d9d5c148ee1e43d3394d6d8c30d364fa17fcc69fb
-  data.tar.gz: fe42ba4880e255aa2a09d360cfec59aaf199ec60407d045641d8ac3cad8319fb3277f2b3e91760e24ab879ec224860628c30fce2282e8c4467a6bfc75826fee5
+  metadata.gz: 3475b05ea1235b388960416a03ab96adb64608ce4046d1812b4d80f038a91b99343882e45413ee6a2f74e29626869aca8a48d34f8e57c64b480c8893efcda123
+  data.tar.gz: 7e4fd1abf5898a7a7273d4eaf53021e4c1898d995e8e0d23abfcbf80c3e1b45b2f059c95e48707577bae72a4f158d7cba7f9be097e7e8a277d89e683386c32d6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.2.6)
+    spidy (0.2.9)
       activesupport
       mechanize
       pry

data/lib/spidy/connector.rb

@@ -22,11 +22,19 @@ module Spidy::Connector
     'Safari/537.36'
   ].join(' ')
 
+  #
+  # error output logger
+  #
+  DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
+
+  #
+  # static method
+  #
   module StaticAccessor
     extend ActiveSupport::Concern
     class_methods do
-      def call(url, wait_time: nil, user_agent: Spidy::Connector::USER_AGENT, &block)
-        new(wait_time: wait_time, user_agent: user_agent).call(url, &block)
+      def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
+        new(wait_time: wait_time, user_agent: user_agent, logger: logger).call(url, &block)
       end
     end
   end
@@ -75,10 +83,14 @@ module Spidy::Connector
   #
   # get connection handller
   #
-  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
     return value if value.respond_to?(:call)
 
-    builder = Builder.new(const_get(value.to_s.classify).new(wait_time: wait_time || 5, user_agent: user_agent || USER_AGENT), socks_proxy)
+    builder = Builder.new(const_get(value.to_s.classify).new(
+      wait_time: wait_time || 5,
+      user_agent: user_agent || USER_AGENT,
+      logger: logger || DEFAULT_LOGGER,
+    ), socks_proxy)
     return builder.origin_connector if socks_proxy.nil? || builder.proxy_disabled?
 
     builder.proxy_connector

data/lib/spidy/connector/direct.rb

@@ -8,6 +8,6 @@ class Spidy::Connector::Direct
     yielder.call(resource)
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(wait_time: nil, user_agent: nil, logger: nil)
   end
 end

data/lib/spidy/connector/html.rb

@@ -8,7 +8,7 @@ class Spidy::Connector::Html
 
   def initialize(wait_time:, user_agent:, logger: nil)
     @wait_time = wait_time
-    @logger = logger || proc { |values| STDERR.puts(values.to_json) }
+    @logger = logger
     @agent = Mechanize.new
     @user_agent = user_agent
     @agent.user_agent = user_agent

data/lib/spidy/connector/json.rb

@@ -6,8 +6,12 @@
 class Spidy::Connector::Json
   include Spidy::Connector::StaticAccessor
 
-  def initialize(wait_time: nil, user_agent: nil)
+  attr_reader :logger
+
+  def initialize(wait_time: nil, user_agent: nil, logger: nil)
+    @wait_time = wait_time
     @user_agent = user_agent
+    @logger = logger
   end
 
   def call(url, &block)
@@ -15,7 +19,19 @@ class Spidy::Connector::Json
     connect(url, &block)
   end
 
-  def connect(url)
+  def connect(url, retry_count: 5)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+  rescue OpenURI::HTTPError => e
+    logger.call('retry.accessed': Time.current,
+                'retry.uri': url,
+                'retry.response_code': e.message,
+                'retry.rest_count': retry_count)
+
+    retry_count -= 1
+    if retry_count.positive?
+      sleep @wait_time
+      retry
+    end
+    raise e
   end
 end

data/lib/spidy/connector/xml.rb

@@ -14,7 +14,7 @@ class Spidy::Connector::Xml
     end
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent: nil, logger: nil)
     @user_agent = user_agent
   end
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.8'
+  VERSION = '0.2.9'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-07 00:00:00.000000000 Z
+date: 2020-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler