spidy 0.0.21 → 0.0.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ac50168f51ba6f3687d83cf9f3331d3952946b08032b902390af3a7e405acbd
-  data.tar.gz: 324e6318e9c2c4dd9ebc1d24bfb57b8fa21b499b050d930a780bbc68f860cb53
+  metadata.gz: 54a354e11d7404229aff8394c535cdf06f8fdf2fcd1afc6081a7bf9dceeb00fb
+  data.tar.gz: bc89e11adeeffe65fc12bb844014814f2dc56b17db41ab2f08a5c40cfdbd5d18
 SHA512:
-  metadata.gz: f8a83b0d24d19f8adf09aea6f26025fee31b1fbb52accf0d66bd017c2d1ebd9e93ad7c624def3fb4f599a05ab1e75a9c8c0a356718482adadcec5280d9910ed1
-  data.tar.gz: 15800fe10242aa6e7049d5b4cec118cc882093ac104b1e5c243cbee257ebf418e23b2433d33b1cb83b0123fb2d50f2dce842c90aed9e1ff4567d5eaae72300d3
+  metadata.gz: 271c80cfcef9dc172a442c22c1db133d0250b016b78e0fefea78b5e6ad287e7d994780345dcec1a681a443956a6b16cac703dce96dc1ae3d410a35a728386ff8
+  data.tar.gz: b4a0ae085bf77a4e3f7f19604088b9e18c6444cff622cacacfeef2ca0a5a463aadf30a289b2ce2df14ffbe9887e95e9a79cc341f763779a5f14ef9281a8796ac

data/lib/spidy/connector/html.rb

@@ -8,10 +8,12 @@ module Spidy::Connector::Html
   # retry class
   #
   class Retry < StandardError
+    attr_reader :page
     attr_reader :response_code
     attr_reader :wait_time
 
     def initialize(wait_time: 2, page: nil, error: nil)
+      @page = page
       @wait_time = wait_time
       @response_code = error.try(:response_code) || page.try(:response_code)
     end
@@ -33,7 +35,7 @@ module Spidy::Connector::Html
     attr_reader :agent
     attr_accessor :logger
 
-    def call(url, encoding: nil, retry_count: 3, &yielder)
+    def call(url, encoding: nil, retry_count: 5, &yielder)
       fail 'url is not specified' if url.blank?
       if encoding
         agent.default_encoding = encoding
@@ -45,16 +47,8 @@ module Spidy::Connector::Html
 
     private
 
-    # rubocop:disable Metrics/MethodLength
     def get(url, retry_count, yielder)
-      agent.get(url) do |page|
-        fail Retry, page: page, wait_time: 5 if page.title == 'Sorry, unable to access page...'
-
-        yielder.call(page)
-      end
-    rescue Mechanize::ResponseCodeError => e
-      raise Retry, error: e if e.response_code == '429'
-      raise e
+      connect(url, retry_count, yielder)
     rescue Retry => e
       logger.call('retry.accessed': Time.current,
                   'retry.uri': url,
@@ -71,6 +65,19 @@ module Spidy::Connector::Html
       end
       raise e
     end
-    # rubocop:enable Metrics/MethodLength
+
+    def connect(url, retry_count, yielder)
+      result = nil
+      agent.get(url) do |page|
+        fail Retry, page: page, wait_time: 5 if page.title == 'Sorry, unable to access page...'
+
+        result = yielder.call(page)
+      end
+      result
+    rescue Mechanize::ResponseCodeError => e
+      raise Retry, error: e if e.response_code == '429'
+      raise e
+    end
+
   end
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.0.21'
+  VERSION = '0.0.22'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.0.21
+  version: 0.0.22
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-11 00:00:00.000000000 Z
+date: 2019-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler