spidy 0.2.9 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44d16509821e1779d821effb7c571d2a11cbb79166168943242628b2e53457ec
-  data.tar.gz: 5b52389a8042c9e69aaeead0d96b81553da2e5f5e51ad16696f87249e38067bd
+  metadata.gz: f87cda14101ec7c184d3e3134f25601cf3c0d2ef2d22b9766f5ce2b417734212
+  data.tar.gz: 803f7012304d475280949f742fb5da7af588dc9f2ab85e3991973b4c60a81a46
 SHA512:
-  metadata.gz: 3475b05ea1235b388960416a03ab96adb64608ce4046d1812b4d80f038a91b99343882e45413ee6a2f74e29626869aca8a48d34f8e57c64b480c8893efcda123
-  data.tar.gz: 7e4fd1abf5898a7a7273d4eaf53021e4c1898d995e8e0d23abfcbf80c3e1b45b2f059c95e48707577bae72a4f158d7cba7f9be097e7e8a277d89e683386c32d6
+  metadata.gz: cd725d2f1a697d4f912e5b37113de561bb002423db61a666cf06404a4bcb3c46765ef1b54dd024f6dd4760f73dc75e285e828e2672348d70e3cb8d71861f5c5f
+  data.tar.gz: ba4550f6871bad2c10f8ad1964e6fd30534f66e02d6f034f916e1afa5247a0f23249fa91842b60477f6f2cbbe2829d8f7edf279655b020eeeb4d242b26d81970

data/example/master_detail.rb

@@ -0,0 +1,73 @@
+
+
+Spidy.define do
+  url_to_params = ->(url) {
+    uri = URI.parse(url)
+    params = URI.decode_www_form(uri.query).to_h if uri.query.present?
+    params if params.present?
+  }
+
+  master_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    page = params&.dig('page')&.to_i || 0
+
+    limit_page = 3
+    per_page = 25
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.main {
+            (page * per_page + 1).upto((page + 1) * per_page).each do |i|
+              doc.a("page #{i}", href: "http://localhost/?id=#{i}")
+            end
+          }
+          doc.a('NEXT', href: "http://localhost/?page=#{page + 1}", class: 'next') if page < limit_page
+        }
+      }
+    }.doc)
+  }
+
+  detail_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    id = params['id']
+
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.h1("title_#{id}", id: 'title')
+          doc.main("body_#{id}", id: 'body')
+          doc.div.sub do
+            doc.span.name('testtest')
+          end
+        }
+      }
+    }.doc)
+  }
+
+  define(as: :html, connector: detail_page) do
+    let(:title, '#title')
+    let(:body, '#body')
+  end
+
+  define(:sub, as: :html, connector: :direct) do
+    let(:name, '.name')
+  end
+
+  spider(as: :html, connector: master_page) do |yielder, connector|
+    next_url = 'http://localhost'
+    while next_url.present?
+      connector.call(next_url) do |page|
+        page.search('main a').each do |a|
+          yielder.call(a.attr('href'))
+        end
+        next_url = page.at('a.next')&.attr('href')
+      end
+    end
+  end
+end

data/example/proxy.rb

@@ -0,0 +1,10 @@
+Spidy.define do
+  user_agent 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0'
+  socks_proxy '127.0.0.1', 9050
+
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/ip') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/example/retry.rb

@@ -0,0 +1,7 @@
+Spidy.define do
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/status/500') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/lib/spidy/connector.rb

@@ -10,10 +10,12 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
+  DEFAULT_WAIT_TIME = 5
+
   #
   # default user agent
   #
-  USER_AGENT = [
+  DEFAULT_USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
     'AppleWebKit/537.36',
@@ -43,56 +45,101 @@ module Spidy::Connector
   # retry class
   #
   class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
+    attr_reader :object, :response_code, :error
+
+    def initialize(object: nil, error: nil, response_code: nil)
+      @object = object
+      @response_code = response_code
+      @error = error
+    end
+  end
+
+  #
+  # retry
+  #
+  class Retryable
+    attr_reader :origin_connector
 
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
+    def initialize(connector, logger:, wait_time:)
+      @origin_connector = connector
+      @logger = logger
       @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
+      @retry_attempt_count = 5
+    end
+
+    def call(url, &block)
+      connect(url, &block)
+    end
+
+    def connect(url, retry_attempt_count: @retry_attempt_count, &block)
+      @logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+      @origin_connector.call(url, &block)
+    rescue Spidy::Connector::Retry => e
+      @logger.call('retry.accessed': Time.current,
+                   'retry.uri': url,
+                   'retry.response_code': e.response_code,
+                   'retry.attempt_count': retry_attempt_count)
+
+      retry_attempt_count -= 1
+      if retry_attempt_count.positive?
+        sleep @wait_time
+        @origin_connector.refresh! if @origin_connector.respond_to?(:refresh!)
+        retry
+      end
+      raise e.error
     end
   end
 
-  class Builder
-    attr_reader :origin_connector, :proxy_connector
+  #
+  # tor proxy
+  #
+  class TorConnector
+    attr_reader :connector, :socks_proxy
 
     def initialize(connector, socks_proxy)
+      @connector = connector
       @socks_proxy = socks_proxy
-      @origin_connector = connector
-      @proxy_connector =
-        lambda do |url, &block|
-          Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
-            connector.call(url, &block)
-          end
-        end
     end
 
-    def proxy_disabled?
-      !tor?
+    def call(url, &block)
+      Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
+        connector.call(url, &block)
+      end
     end
 
-    def tor?
-      Tor::Controller.new(host: @socks_proxy[:host], port: @socks_proxy[:port]).close
+    def try_connection?
+      try_connection!
       true
     rescue Errno::ECONNREFUSED
       false
     end
+
+    def try_connection!
+      Tor::Controller.new(host: @socks_proxy[:host], port: @socks_proxy[:port]).close
+    end
+  end
+
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
+    logger ||= DEFAULT_LOGGER
+    user_agent ||= DEFAULT_USER_AGENT
+    wait_time ||= DEFAULT_WAIT_TIME
+
+    connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
+    Retryable.new(connector, wait_time: wait_time, logger: logger)
   end
 
   #
   # get connection handller
   #
-  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
+  def self.get_connector(value, user_agent: nil, socks_proxy: nil)
     return value if value.respond_to?(:call)
 
-    builder = Builder.new(const_get(value.to_s.classify).new(
-      wait_time: wait_time || 5,
-      user_agent: user_agent || USER_AGENT,
-      logger: logger || DEFAULT_LOGGER,
-    ), socks_proxy)
-    return builder.origin_connector if socks_proxy.nil? || builder.proxy_disabled?
+    connector = const_get(value.to_s.classify).new(user_agent: user_agent)
+    fail "Not defined connnector[#{value}]" if connector.nil?
+    return connector if socks_proxy.nil?
 
-    builder.proxy_connector
+    tor = TorConnector.new(connnector, socks_proxy)
+    tor.try_connection!
+    tor
   end
 end

data/lib/spidy/connector/direct.rb

@@ -8,6 +8,6 @@ class Spidy::Connector::Direct
     yielder.call(resource)
   end
 
-  def initialize(wait_time: nil, user_agent: nil, logger: nil)
+  def initialize(user_agent:)
   end
 end

data/lib/spidy/connector/html.rb

@@ -6,16 +6,13 @@
 class Spidy::Connector::Html
   include Spidy::Connector::StaticAccessor
 
-  def initialize(wait_time:, user_agent:, logger: nil)
-    @wait_time = wait_time
-    @logger = logger
+  def initialize(user_agent:)
     @agent = Mechanize.new
     @user_agent = user_agent
     @agent.user_agent = user_agent
   end
 
   attr_reader :agent
-  attr_reader :logger
 
   def call(url, encoding: nil, retry_count: 5, &yielder)
     fail 'url is not specified' if url.blank?
@@ -23,42 +20,27 @@ class Spidy::Connector::Html
       agent.default_encoding = encoding
       agent.force_default_encoding = true
     end
-    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-    get(url, retry_count, yielder)
-  end
-
-  private
-
-  def get(url, retry_count, yielder)
     connect(url, retry_count, yielder)
-  rescue Spidy::Connector::Retry => e
-    logger.call('retry.accessed': Time.current,
-                'retry.uri': url,
-                'retry.response_code': e.response_code,
-                'retry.rest_count': retry_count)
+  end
 
+  def refresh!
     @agent = Mechanize.new
     @agent.user_agent = @user_agent
-
-    retry_count -= 1
-    if retry_count.positive?
-      sleep e.wait_time
-      retry
-    end
-    raise e
   end
 
+  private
+
   def connect(url, retry_count, yielder)
     result = nil
     agent.get(url) do |page|
-      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
+      fail Spidy::Connector::Retry, object: page, response_code: page.try(:response_code) if page.title == 'Sorry, unable to access page...'
 
       result = yielder.call(page)
     end
     result
   rescue Mechanize::ResponseCodeError => e
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
-    raise e
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '502'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code)
   end
 end

data/lib/spidy/connector/json.rb

@@ -8,10 +8,8 @@ class Spidy::Connector::Json
 
   attr_reader :logger
 
-  def initialize(wait_time: nil, user_agent: nil, logger: nil)
-    @wait_time = wait_time
+  def initialize(user_agent: nil)
     @user_agent = user_agent
-    @logger = logger
   end
 
   def call(url, &block)
@@ -22,16 +20,6 @@ class Spidy::Connector::Json
   def connect(url, retry_count: 5)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
   rescue OpenURI::HTTPError => e
-    logger.call('retry.accessed': Time.current,
-                'retry.uri': url,
-                'retry.response_code': e.message,
-                'retry.rest_count': retry_count)
-
-    retry_count -= 1
-    if retry_count.positive?
-      sleep @wait_time
-      retry
-    end
-    raise e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 end

data/lib/spidy/connector/xml.rb

@@ -6,15 +6,21 @@
 class Spidy::Connector::Xml
   include Spidy::Connector::StaticAccessor
 
-  def call(url)
+  def call(url, &block)
     fail 'URL is undefined' if url.blank?
 
+    connect(url, &block)
+  end
+
+  def connect(url, &block)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
-      yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
+      block.call Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 
-  def initialize(user_agent: nil, logger: nil)
+  def initialize(user_agent:)
     @user_agent = user_agent
   end
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.9'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.3.0
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -187,6 +187,9 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- example/master_detail.rb
+- example/proxy.rb
+- example/retry.rb
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb