spidy 0.2.6 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6c2e2e85979c5d5492564fec243c23c44ef1d762230cd1a59400fcc46bf42a3
-  data.tar.gz: d551534240f528923ccb862e17ae521aaccdc5596e45c09e1cee0e90f0c990c5
+  metadata.gz: 52bd83a0c2e46c24579454780bc89ea49c9cfab9a38403b117f14727e44b571f
+  data.tar.gz: 3d258302ce5a59d48a68b1bc673f7517b2bd25be7129f29ec3e2ea11a8161640
 SHA512:
-  metadata.gz: 3348c6243500f90f7157435bf36f1675744f46c0025dc4bab81fad4ccfb65c4e2ac0d029a837fb589c768b4868fbaf8ab7c258ad67aa4e1a9f33f0949701b2da
-  data.tar.gz: ef8d804eae97e2300747abb1b9e97f84e100b6b151e01b587fff95c98327ce21e0024456c6d8208ecd81ceaead8a0353d251f15a4ef4deff7e3b077e13c872cb
+  metadata.gz: 26cdfeefd087471eb5ed62937176014a01b0b083e2ebed8dd046a27c9ec9cd795a36a1bed59fdfdc29fca9cf186a97125ee8db0b255701869c51ca268a0f737f
+  data.tar.gz: d796fe650f6fd98a52c9aaad2aed53f73de3efb1a9a7b5147daf3e455a63d38d294a41471e9ac7198f38e7c79f0f308cbb70b71786bbb33f5413bfd73138f242

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.2.6)
+    spidy (0.2.9)
       activesupport
       mechanize
       pry

data/example/master_detail.rb

@@ -0,0 +1,73 @@
+
+
+Spidy.define do
+  url_to_params = ->(url) {
+    uri = URI.parse(url)
+    params = URI.decode_www_form(uri.query).to_h if uri.query.present?
+    params if params.present?
+  }
+
+  master_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    page = params&.dig('page')&.to_i || 0
+
+    limit_page = 3
+    per_page = 25
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.main {
+            (page * per_page + 1).upto((page + 1) * per_page).each do |i|
+              doc.a("page #{i}", href: "http://localhost/?id=#{i}")
+            end
+          }
+          doc.a('NEXT', href: "http://localhost/?page=#{page + 1}", class: 'next') if page < limit_page
+        }
+      }
+    }.doc)
+  }
+
+  detail_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    id = params['id']
+
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.h1("title_#{id}", id: 'title')
+          doc.main("body_#{id}", id: 'body')
+          doc.div.sub do
+            doc.span.name('testtest')
+          end
+        }
+      }
+    }.doc)
+  }
+
+  define(as: :html, connector: detail_page) do
+    let(:title, '#title')
+    let(:body, '#body')
+  end
+
+  define(:sub, as: :html, connector: :direct) do
+    let(:name, '.name')
+  end
+
+  spider(as: :html, connector: master_page) do |yielder, connector|
+    next_url = 'http://localhost'
+    while next_url.present?
+      connector.call(next_url) do |page|
+        page.search('main a').each do |a|
+          yielder.call(a.attr('href'))
+        end
+        next_url = page.at('a.next')&.attr('href')
+      end
+    end
+  end
+end

data/example/proxy.rb

@@ -0,0 +1,10 @@
+Spidy.define do
+  user_agent 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0'
+  socks_proxy '127.0.0.1', 9050
+
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/ip') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/example/retry.rb

@@ -0,0 +1,7 @@
+Spidy.define do
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/status/500') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/lib/spidy/connector.rb

@@ -10,25 +10,12 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
-  #
-  # retry class
-  #
-  class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
-
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
-      @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
-    end
-  end
+  DEFAULT_WAIT_TIME = 5
 
   #
   # default user agent
   #
-  USER_AGENT = [
+  DEFAULT_USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
     'AppleWebKit/537.36',
@@ -38,18 +25,121 @@ module Spidy::Connector
   ].join(' ')
 
   #
-  # get connection handller
+  # error output logger
   #
-  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
-    return value if value.respond_to?(:call)
+  DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
 
-    connector = const_get(value.to_s.classify).new(wait_time: wait_time || 5, user_agent: user_agent || USER_AGENT)
-    return connector if socks_proxy.nil?
+  #
+  # static method
+  #
+  module StaticAccessor
+    extend ActiveSupport::Concern
+    class_methods do
+      def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
+        new(wait_time: wait_time, user_agent: user_agent, logger: logger).call(url, &block)
+      end
+    end
+  end
+
+  #
+  # retry class
+  #
+  class Retry < StandardError
+    attr_reader :object, :response_code, :error
+
+    def initialize(object: nil, error: nil, response_code: nil)
+      @object = object
+      @response_code = response_code
+      @error = error
+    end
+  end
+
+  #
+  # retry
+  #
+  class Retryable
+    attr_reader :origin_connector
+
+    def initialize(connector, logger:, wait_time:)
+      @origin_connector = connector
+      @logger = logger
+      @wait_time = wait_time
+      @retry_attempt_count = 5
+    end
+
+    def call(url, &block)
+      connect(url, &block)
+    end
+
+    def connect(url, retry_attempt_count: @retry_attempt_count, &block)
+      @logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+      @origin_connector.call(url, &block)
+    rescue Spidy::Connector::Retry => e
+      @logger.call('retry.accessed': Time.current,
+                   'retry.uri': url,
+                   'retry.response_code': e.response_code,
+                   'retry.attempt_count': retry_attempt_count)
+
+      retry_attempt_count -= 1
+      if retry_attempt_count.positive?
+        sleep @wait_time
+        @origin_connector.refresh! if @origin_connector.respond_to?(:refresh!)
+        retry
+      end
+      raise e.error
+    end
+  end
 
-    lambda do |url, &block|
+  #
+  # tor proxy
+  #
+  class TorConnector
+    attr_reader :connector, :socks_proxy
+
+    def initialize(connector, socks_proxy)
+      @connector = connector
+      @socks_proxy = socks_proxy
+    end
+
+    def call(url, &block)
       Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
         connector.call(url, &block)
       end
     end
+
+    def try_connection?
+      try_connection!
+      true
+    rescue Errno::ECONNREFUSED
+      false
+    end
+
+    def try_connection!
+      Tor::Controller.new(host: @socks_proxy[:host], port: @socks_proxy[:port]).close
+    end
+  end
+
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
+    logger ||= DEFAULT_LOGGER
+    user_agent ||= DEFAULT_USER_AGENT
+    wait_time ||= DEFAULT_WAIT_TIME
+
+    connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
+    Retryable.new(connector, wait_time: wait_time, logger: logger)
+  end
+
+  #
+  # get connection handller
+  #
+  def self.get_connector(value, user_agent: nil, socks_proxy: nil)
+    return value if value.respond_to?(:call)
+
+    connector = const_get(value.to_s.classify).new(user_agent: user_agent)
+    fail "Not defined connnector[#{value}]" if connector.nil?
+    return connector if socks_proxy.nil?
+
+    tor = TorConnector.new(connector, socks_proxy)
+    tor.try_connection!
+    tor
   end
 end

data/lib/spidy/connector/direct.rb

@@ -8,6 +8,6 @@ class Spidy::Connector::Direct
     yielder.call(resource)
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent:)
   end
 end

data/lib/spidy/connector/html.rb

@@ -4,16 +4,15 @@
 # Mechanize wrapper
 #
 class Spidy::Connector::Html
-  def initialize(wait_time:, user_agent:, logger: nil)
-    @wait_time = wait_time
-    @logger = logger || proc { |values| STDERR.puts(values.to_json) }
+  include Spidy::Connector::StaticAccessor
+
+  def initialize(user_agent:)
     @agent = Mechanize.new
     @user_agent = user_agent
     @agent.user_agent = user_agent
   end
 
   attr_reader :agent
-  attr_reader :logger
 
   def call(url, encoding: nil, retry_count: 5, &yielder)
     fail 'url is not specified' if url.blank?
@@ -21,42 +20,27 @@ class Spidy::Connector::Html
       agent.default_encoding = encoding
       agent.force_default_encoding = true
     end
-    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-    get(url, retry_count, yielder)
-  end
-
-  private
-
-  def get(url, retry_count, yielder)
     connect(url, retry_count, yielder)
-  rescue Spidy::Connector::Retry => e
-    logger.call('retry.accessed': Time.current,
-                'retry.uri': url,
-                'retry.response_code': e.response_code,
-                'retry.rest_count': retry_count)
+  end
 
+  def refresh!
     @agent = Mechanize.new
     @agent.user_agent = @user_agent
-
-    retry_count -= 1
-    if retry_count.positive?
-      sleep e.wait_time
-      retry
-    end
-    raise e
   end
 
+  private
+
   def connect(url, retry_count, yielder)
     result = nil
     agent.get(url) do |page|
-      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
+      fail Spidy::Connector::Retry, object: page, response_code: page.try(:response_code) if page.title == 'Sorry, unable to access page...'
 
       result = yielder.call(page)
     end
     result
   rescue Mechanize::ResponseCodeError => e
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
-    raise e
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '502'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code)
   end
 end

data/lib/spidy/connector/json.rb

@@ -4,7 +4,11 @@
 # OpenURI to JSON.parse
 #
 class Spidy::Connector::Json
-  def initialize(wait_time: nil, user_agent: nil)
+  include Spidy::Connector::StaticAccessor
+
+  attr_reader :logger
+
+  def initialize(user_agent: nil)
     @user_agent = user_agent
   end
 
@@ -13,7 +17,9 @@ class Spidy::Connector::Json
     connect(url, &block)
   end
 
-  def connect(url)
+  def connect(url, retry_count: 5)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 end

data/lib/spidy/connector/xml.rb

@@ -4,15 +4,23 @@
 # xml
 #
 class Spidy::Connector::Xml
-  def call(url)
+  include Spidy::Connector::StaticAccessor
+
+  def call(url, &block)
     fail 'URL is undefined' if url.blank?
 
+    connect(url, &block)
+  end
+
+  def connect(url, &block)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
-      yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
+      block.call Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent:)
     @user_agent = user_agent
   end
 end

data/lib/spidy/definition.rb

@@ -28,10 +28,6 @@ module Spidy::Definition
     @socks_proxy = { host: host, port: port }
   end
 
-  def tor?
-    Tor.running?
-  end
-
   def each(source = nil, name: :default, &yielder)
     name = name.presence || :default
     spidy = @namespace[:"#{name}_spider"]
@@ -50,7 +46,7 @@ module Spidy::Definition
 
   def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
     binder = Spidy::Binder.get(self, binder || as)
     @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
   end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.6'
+  VERSION = '0.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.1
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-01 00:00:00.000000000 Z
+date: 2020-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -187,6 +187,9 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- example/master_detail.rb
+- example/proxy.rb
+- example/retry.rb
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb