spidy 0.2.8 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43e5538d56f4d4115a1bb2cb662f0722d688ad44d8212506436492f0ee3e0570
-  data.tar.gz: b0db640263836c7244876a50219ed4f916c6c2f7777e11c7e13d11c2142714dc
+  metadata.gz: a70bf5d610f60d0b71c719cf870995a7e93b6e9abd71ef9823a71e2ed506f190
+  data.tar.gz: 203bc7721020e244b9ad3ecc526b27e1c8d83a807cc2798bcb8930b1e17d5277
 SHA512:
-  metadata.gz: 3d6dcfea624a7710eec941e55238334596edced66d60a09b046c5b057de06c2fb7d279c3d912a8e054ef206d9d5c148ee1e43d3394d6d8c30d364fa17fcc69fb
-  data.tar.gz: fe42ba4880e255aa2a09d360cfec59aaf199ec60407d045641d8ac3cad8319fb3277f2b3e91760e24ab879ec224860628c30fce2282e8c4467a6bfc75826fee5
+  metadata.gz: 9977aeb13ff786bd8fbeb7d8ca0ee3ef7b67dfb577739c9aba0a310c46741fa48d86596de6f69ad05eaf2d64f9de4259bea84d4939ee2c7288781106fb25f2b3
+  data.tar.gz: d64e9e66b25d8985f2c009abddd6f6b862aa8f533b67e125fa35008d308424e766d3ad4a8a64c16a0c9ed3448cdad14a09c80f6f77437e604826808286270d4f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.2.6)
+    spidy (0.3.3)
       activesupport
       mechanize
       pry
@@ -11,12 +11,24 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.2)
+    activesupport (6.0.3.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2, >= 2.2.2)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    capybara (3.33.0)
+      addressable
+      mini_mime (>= 0.1.3)
+      nokogiri (~> 1.8)
+      rack (>= 1.6.0)
+      rack-test (>= 0.6.3)
+      regexp_parser (~> 1.5)
+      xpath (~> 3.2)
+    capybara_discoball (0.1.0)
+      capybara (>= 2.7, < 4)
     coderay (1.1.2)
     concurrent-ruby (1.1.7)
     connection_pool (2.2.3)
@@ -41,9 +53,12 @@ GEM
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
+    mini_mime (1.0.2)
     mini_portile2 (2.4.0)
     minitest (5.14.2)
     mixlib-shellout (2.4.4)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     net-http-digest_auth (1.4.1)
     net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
@@ -53,7 +68,14 @@ GEM
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
+    public_suffix (4.0.6)
+    rack (2.2.3)
+    rack-protection (2.0.8.1)
+      rack
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
     rake (10.5.0)
+    regexp_parser (1.8.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -74,8 +96,15 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.2)
+    ruby2_keywords (0.0.2)
+    sinatra (2.0.8.1)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.8.1)
+      tilt (~> 2.0)
     socksify (1.7.1)
     thread_safe (0.3.6)
+    tilt (2.0.10)
     tor (0.1.4)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
@@ -83,6 +112,8 @@ GEM
       unf_ext
     unf_ext (0.0.7.7)
     webrobots (0.1.2)
+    xpath (3.2.0)
+      nokogiri (~> 1.8)
     zeitwerk (2.4.0)
 
 PLATFORMS
@@ -90,11 +121,13 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 2.0)
+  capybara_discoball
   ffaker
   pry
   rake (~> 10.0)
   rspec (~> 3.0)
   rspec-command
+  sinatra
   spidy!
 
 BUNDLED WITH

data/example/master_detail.rb

@@ -0,0 +1,73 @@
+
+
+Spidy.define do
+  url_to_params = ->(url) {
+    uri = URI.parse(url)
+    params = URI.decode_www_form(uri.query).to_h if uri.query.present?
+    params if params.present?
+  }
+
+  master_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    page = params&.dig('page')&.to_i || 0
+
+    limit_page = 3
+    per_page = 25
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.main {
+            (page * per_page + 1).upto((page + 1) * per_page).each do |i|
+              doc.a("page #{i}", href: "http://localhost/?id=#{i}")
+            end
+          }
+          doc.a('NEXT', href: "http://localhost/?page=#{page + 1}", class: 'next') if page < limit_page
+        }
+      }
+    }.doc)
+  }
+
+  detail_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    id = params['id']
+
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.h1("title_#{id}", id: 'title')
+          doc.main("body_#{id}", id: 'body')
+          doc.div.sub do
+            doc.span.name('testtest')
+          end
+        }
+      }
+    }.doc)
+  }
+
+  define(as: :html, connector: detail_page) do
+    let(:title, '#title')
+    let(:body, '#body')
+  end
+
+  define(:sub, as: :html, connector: :direct) do
+    let(:name, '.name')
+  end
+
+  spider(as: :html, connector: master_page) do |yielder, connector|
+    next_url = 'http://localhost'
+    while next_url.present?
+      connector.call(next_url) do |page|
+        page.search('main a').each do |a|
+          yielder.call(a.attr('href'))
+        end
+        next_url = page.at('a.next')&.attr('href')
+      end
+    end
+  end
+end

data/example/proxy.rb

@@ -0,0 +1,10 @@
+Spidy.define do
+  user_agent 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0'
+  socks_proxy '127.0.0.1', 9050
+
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/ip') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/example/retry.rb

@@ -0,0 +1,7 @@
+Spidy.define do
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/status/500') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/lib/spidy/connector.rb

@@ -10,6 +10,8 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
+  DEFAULT_WAIT_TIME = 5
+
   #
   # default user agent
   #
@@ -22,11 +24,19 @@ module Spidy::Connector
     'Safari/537.36'
   ].join(' ')
 
+  #
+  # error output logger
+  #
+  DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
+
+  #
+  # static method
+  #
   module StaticAccessor
     extend ActiveSupport::Concern
     class_methods do
-      def call(url, wait_time: nil, user_agent: Spidy::Connector::USER_AGENT, &block)
-        new(wait_time: wait_time, user_agent: user_agent).call(url, &block)
+      def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
+        ::Spidy::Connector::RetryableCaller.new(new(user_agent: user_agent), wait_time: wait_time, logger: logger).call(url, &block)
       end
     end
   end
@@ -35,52 +45,101 @@ module Spidy::Connector
   # retry class
   #
   class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
+    attr_reader :object, :response_code, :error
 
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
+    def initialize(object: nil, error: nil, response_code: nil)
+      @object = object
+      @response_code = response_code
+      @error = error
+    end
+  end
+
+  #
+  # retry
+  #
+  class RetryableCaller
+    attr_reader :origin_connector
+
+    def initialize(connector, logger:, wait_time:)
+      @origin_connector = connector
+      @logger = logger
       @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
+      @retry_attempt_count = 5
+    end
+
+    def call(url, &block)
+      block ||= ->(result) { break result }
+      connect(url, &block)
+    end
+
+    def connect(url, retry_attempt_count: @retry_attempt_count, &block)
+      @logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+      @origin_connector.call(url, &block)
+    rescue Spidy::Connector::Retry => e
+      @logger.call('retry.accessed': Time.current,
+                   'retry.uri': url,
+                   'retry.response_code': e.response_code,
+                   'retry.attempt_count': retry_attempt_count)
+
+      retry_attempt_count -= 1
+      if retry_attempt_count.positive?
+        sleep @wait_time
+        @origin_connector.refresh! if @origin_connector.respond_to?(:refresh!)
+        retry
+      end
+      raise e.error
     end
   end
 
-  class Builder
-    attr_reader :origin_connector, :proxy_connector
+  #
+  # tor proxy
+  #
+  class TorConnector
+    attr_reader :connector, :socks_proxy
 
     def initialize(connector, socks_proxy)
+      @connector = connector
       @socks_proxy = socks_proxy
-      @origin_connector = connector
-      @proxy_connector =
-        lambda do |url, &block|
-          Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
-            connector.call(url, &block)
-          end
-        end
     end
 
-    def proxy_disabled?
-      !tor?
+    def call(url, &block)
+      Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
+        connector.call(url, &block)
+      end
     end
 
-    def tor?
-      Tor::Controller.new(host: @socks_proxy[:host], port: @socks_proxy[:port]).close
+    def try_connection?
+      try_connection!
       true
     rescue Errno::ECONNREFUSED
       false
     end
+
+    def try_connection!
+      Tor::Controller.new(host: @socks_proxy[:host], port: @socks_proxy[:port]).close
+    end
+  end
+
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
+    user_agent ||= USER_AGENT
+    logger ||= DEFAULT_LOGGER
+    wait_time ||= DEFAULT_WAIT_TIME
+
+    connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
+    RetryableCaller.new(connector, wait_time: wait_time, logger: logger)
   end
 
   #
   # get connection handller
   #
-  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
+  def self.get_connector(value, user_agent: nil, socks_proxy: nil)
     return value if value.respond_to?(:call)
 
-    builder = Builder.new(const_get(value.to_s.classify).new(wait_time: wait_time || 5, user_agent: user_agent || USER_AGENT), socks_proxy)
-    return builder.origin_connector if socks_proxy.nil? || builder.proxy_disabled?
+    connector = const_get(value.to_s.classify).new(user_agent: user_agent)
+    fail "Not defined connnector[#{value}]" if connector.nil?
+    return connector if socks_proxy.nil?
 
-    builder.proxy_connector
+    tor = TorConnector.new(connector, socks_proxy)
+    tor
   end
 end

data/lib/spidy/connector/direct.rb

@@ -8,6 +8,6 @@ class Spidy::Connector::Direct
     yielder.call(resource)
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent:)
   end
 end

data/lib/spidy/connector/html.rb

@@ -6,16 +6,13 @@
 class Spidy::Connector::Html
   include Spidy::Connector::StaticAccessor
 
-  def initialize(wait_time:, user_agent:, logger: nil)
-    @wait_time = wait_time
-    @logger = logger || proc { |values| STDERR.puts(values.to_json) }
+  def initialize(user_agent:)
     @agent = Mechanize.new
     @user_agent = user_agent
     @agent.user_agent = user_agent
   end
 
   attr_reader :agent
-  attr_reader :logger
 
   def call(url, encoding: nil, retry_count: 5, &yielder)
     fail 'url is not specified' if url.blank?
@@ -23,42 +20,27 @@ class Spidy::Connector::Html
       agent.default_encoding = encoding
       agent.force_default_encoding = true
     end
-    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-    get(url, retry_count, yielder)
-  end
-
-  private
-
-  def get(url, retry_count, yielder)
     connect(url, retry_count, yielder)
-  rescue Spidy::Connector::Retry => e
-    logger.call('retry.accessed': Time.current,
-                'retry.uri': url,
-                'retry.response_code': e.response_code,
-                'retry.rest_count': retry_count)
+  end
 
+  def refresh!
     @agent = Mechanize.new
     @agent.user_agent = @user_agent
-
-    retry_count -= 1
-    if retry_count.positive?
-      sleep e.wait_time
-      retry
-    end
-    raise e
   end
 
+  private
+
   def connect(url, retry_count, yielder)
     result = nil
     agent.get(url) do |page|
-      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
+      fail Spidy::Connector::Retry, object: page, response_code: page.try(:response_code) if page.title == 'Sorry, unable to access page...'
 
       result = yielder.call(page)
     end
     result
   rescue Mechanize::ResponseCodeError => e
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
-    raise e
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '502'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code)
   end
 end

data/lib/spidy/connector/json.rb

@@ -6,7 +6,9 @@
 class Spidy::Connector::Json
   include Spidy::Connector::StaticAccessor
 
-  def initialize(wait_time: nil, user_agent: nil)
+  attr_reader :logger
+
+  def initialize(user_agent:)
     @user_agent = user_agent
   end
 
@@ -15,7 +17,9 @@ class Spidy::Connector::Json
     connect(url, &block)
   end
 
-  def connect(url)
+  def connect(url, retry_count: 5)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 end

data/lib/spidy/connector/xml.rb

@@ -6,15 +6,21 @@
 class Spidy::Connector::Xml
   include Spidy::Connector::StaticAccessor
 
-  def call(url)
+  def call(url, &block)
     fail 'URL is undefined' if url.blank?
 
+    connect(url, &block)
+  end
+
+  def connect(url, &block)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
-      yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
+      block.call Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent:)
     @user_agent = user_agent
   end
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.8'
+  VERSION = '0.3.3'
 end

data/spidy.gemspec

@@ -30,6 +30,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'
+  spec.add_development_dependency 'capybara_discoball'
+  spec.add_development_dependency 'sinatra'
 
   spec.add_runtime_dependency 'tor'
   spec.add_runtime_dependency 'activesupport'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.3
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-07 00:00:00.000000000 Z
+date: 2020-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara_discoball
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: tor
   requirement: !ruby/object:Gem::Requirement
@@ -187,6 +215,9 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- example/master_detail.rb
+- example/proxy.rb
+- example/retry.rb
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb