spidy 0.2.7 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 177e6b5e3b4f864251d7071d23b2b42b219d8193673a5b57ee9b376e393092a9
-  data.tar.gz: 92b39a277d752eb5b1944f822702eae803356f7f5e3211b1f91e2d9ad635090e
+  metadata.gz: 693834c20f838dfc7bfed8bfe3d77f21b519d68840f5f5ce3c18e971de496b8e
+  data.tar.gz: b3a43c5ef746fc550fa617157e3a779916cb6f1963b48896c8e6ef919cc5f9ad
 SHA512:
-  metadata.gz: b4078e3d1ad9813b0273f267d99bf7a9bf4473bb86b6cfa4fa283e6c5b060fa248e0b6fb517bede448f94e0dcebbb90ee71ef4eeb4bd5a45b1405c50005985fe
-  data.tar.gz: 00aa2cd512cf3a083082227234f59148d86e8ee0e4f602e07d1c67cbf9f3e8160b9144ebd29f3a67efded607c1705c2ce3008df235325afc7f3a715a6dd7527d
+  metadata.gz: 4dd3ffbc9f786b7c7b4d20f4ca07f7873922ca6633730b143f7b071063a8a08a30a83cd4444efc66ca1a5bc820d1e6ea3d5c2c16bd83447ee59454084f82d6ba
+  data.tar.gz: 299293fdf0fb701e5adfd49f6a7f91c712bf911ec78ee555eefb07e7f0c0247f48bc63e9ab414fcc900545a506a0ac1bd47625edf1905973ed2e447d376f2b80

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.2.6)
+    spidy (0.2.9)
       activesupport
       mechanize
       pry

data/example/master_detail.rb

@@ -0,0 +1,73 @@
+
+
+Spidy.define do
+  url_to_params = ->(url) {
+    uri = URI.parse(url)
+    params = URI.decode_www_form(uri.query).to_h if uri.query.present?
+    params if params.present?
+  }
+
+  master_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    page = params&.dig('page')&.to_i || 0
+
+    limit_page = 3
+    per_page = 25
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.main {
+            (page * per_page + 1).upto((page + 1) * per_page).each do |i|
+              doc.a("page #{i}", href: "http://localhost/?id=#{i}")
+            end
+          }
+          doc.a('NEXT', href: "http://localhost/?page=#{page + 1}", class: 'next') if page < limit_page
+        }
+      }
+    }.doc)
+  }
+
+  detail_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    id = params['id']
+
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.h1("title_#{id}", id: 'title')
+          doc.main("body_#{id}", id: 'body')
+          doc.div.sub do
+            doc.span.name('testtest')
+          end
+        }
+      }
+    }.doc)
+  }
+
+  define(as: :html, connector: detail_page) do
+    let(:title, '#title')
+    let(:body, '#body')
+  end
+
+  define(:sub, as: :html, connector: :direct) do
+    let(:name, '.name')
+  end
+
+  spider(as: :html, connector: master_page) do |yielder, connector|
+    next_url = 'http://localhost'
+    while next_url.present?
+      connector.call(next_url) do |page|
+        page.search('main a').each do |a|
+          yielder.call(a.attr('href'))
+        end
+        next_url = page.at('a.next')&.attr('href')
+      end
+    end
+  end
+end

data/example/proxy.rb

@@ -0,0 +1,10 @@
+Spidy.define do
+  user_agent 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0'
+  socks_proxy '127.0.0.1', 9050
+
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/ip') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/example/retry.rb

@@ -0,0 +1,7 @@
+Spidy.define do
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/status/500') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/lib/spidy/connector.rb

@@ -10,11 +10,33 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
+  DEFAULT_WAIT_TIME = 5
+
+  #
+  # default user agent
+  #
+  DEFAULT_USER_AGENT = [
+    'Mozilla/5.0',
+    '(Macintosh; Intel Mac OS X 10_12_6)',
+    'AppleWebKit/537.36',
+    '(KHTML, like Gecko)',
+    'Chrome/64.0.3282.186',
+    'Safari/537.36'
+  ].join(' ')
+
+  #
+  # error output logger
+  #
+  DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
+
+  #
+  # static method
+  #
   module StaticAccessor
     extend ActiveSupport::Concern
     class_methods do
-      def call(url, wait_time: nil, user_agent: Spidy::Connector::USER_AGENT, &block)
-        new(wait_time: wait_time, user_agent: user_agent).call(url, &block)
+      def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
+        new(wait_time: wait_time, user_agent: user_agent, logger: logger).call(url, &block)
       end
     end
   end
@@ -23,42 +45,100 @@ module Spidy::Connector
   # retry class
   #
   class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
+    attr_reader :object, :response_code, :error
 
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
-      @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
+    def initialize(object: nil, error: nil, response_code: nil)
+      @object = object
+      @response_code = response_code
+      @error = error
     end
   end
 
   #
-  # default user agent
+  # retry
   #
-  USER_AGENT = [
-    'Mozilla/5.0',
-    '(Macintosh; Intel Mac OS X 10_12_6)',
-    'AppleWebKit/537.36',
-    '(KHTML, like Gecko)',
-    'Chrome/64.0.3282.186',
-    'Safari/537.36'
-  ].join(' ')
+  class Retryable
+    attr_reader :origin_connector
+
+    def initialize(connector, logger:, wait_time:)
+      @origin_connector = connector
+      @logger = logger
+      @wait_time = wait_time
+      @retry_attempt_count = 5
+    end
+
+    def call(url, &block)
+      connect(url, &block)
+    end
+
+    def connect(url, retry_attempt_count: @retry_attempt_count, &block)
+      @logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+      @origin_connector.call(url, &block)
+    rescue Spidy::Connector::Retry => e
+      @logger.call('retry.accessed': Time.current,
+                   'retry.uri': url,
+                   'retry.response_code': e.response_code,
+                   'retry.attempt_count': retry_attempt_count)
+
+      retry_attempt_count -= 1
+      if retry_attempt_count.positive?
+        sleep @wait_time
+        @origin_connector.refresh! if @origin_connector.respond_to?(:refresh!)
+        retry
+      end
+      raise e.error
+    end
+  end
 
   #
-  # get connection handller
+  # tor proxy
   #
-  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
-    return value if value.respond_to?(:call)
+  class TorConnector
+    attr_reader :connector, :socks_proxy
 
-    connector = const_get(value.to_s.classify).new(wait_time: wait_time || 5, user_agent: user_agent || USER_AGENT)
-    return connector if socks_proxy.nil?
+    def initialize(connector, socks_proxy)
+      @connector = connector
+      @socks_proxy = socks_proxy
+    end
 
-    lambda do |url, &block|
+    def call(url, &block)
       Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
         connector.call(url, &block)
       end
     end
+
+    def try_connection?
+      try_connection!
+      true
+    rescue Errno::ECONNREFUSED
+      false
+    end
+
+    def try_connection!
+      Tor::Controller.new(host: @socks_proxy[:host], port: @socks_proxy[:port]).close
+    end
+  end
+
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
+    logger ||= DEFAULT_LOGGER
+    user_agent ||= DEFAULT_USER_AGENT
+    wait_time ||= DEFAULT_WAIT_TIME
+
+    connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
+    Retryable.new(connector, wait_time: wait_time, logger: logger)
+  end
+
+  #
+  # get connection handller
+  #
+  def self.get_connector(value, user_agent: nil, socks_proxy: nil)
+    return value if value.respond_to?(:call)
+
+    connector = const_get(value.to_s.classify).new(user_agent: user_agent)
+    fail "Not defined connnector[#{value}]" if connector.nil?
+    return connector if socks_proxy.nil?
+
+    tor = TorConnector.new(connector, socks_proxy)
+    tor
   end
 end

data/lib/spidy/connector/direct.rb

@@ -8,6 +8,6 @@ class Spidy::Connector::Direct
     yielder.call(resource)
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent:)
   end
 end

data/lib/spidy/connector/html.rb

@@ -6,16 +6,13 @@
 class Spidy::Connector::Html
   include Spidy::Connector::StaticAccessor
 
-  def initialize(wait_time:, user_agent:, logger: nil)
-    @wait_time = wait_time
-    @logger = logger || proc { |values| STDERR.puts(values.to_json) }
+  def initialize(user_agent:)
     @agent = Mechanize.new
     @user_agent = user_agent
     @agent.user_agent = user_agent
   end
 
   attr_reader :agent
-  attr_reader :logger
 
   def call(url, encoding: nil, retry_count: 5, &yielder)
     fail 'url is not specified' if url.blank?
@@ -23,42 +20,27 @@ class Spidy::Connector::Html
       agent.default_encoding = encoding
       agent.force_default_encoding = true
     end
-    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-    get(url, retry_count, yielder)
-  end
-
-  private
-
-  def get(url, retry_count, yielder)
     connect(url, retry_count, yielder)
-  rescue Spidy::Connector::Retry => e
-    logger.call('retry.accessed': Time.current,
-                'retry.uri': url,
-                'retry.response_code': e.response_code,
-                'retry.rest_count': retry_count)
+  end
 
+  def refresh!
     @agent = Mechanize.new
     @agent.user_agent = @user_agent
-
-    retry_count -= 1
-    if retry_count.positive?
-      sleep e.wait_time
-      retry
-    end
-    raise e
   end
 
+  private
+
   def connect(url, retry_count, yielder)
     result = nil
     agent.get(url) do |page|
-      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
+      fail Spidy::Connector::Retry, object: page, response_code: page.try(:response_code) if page.title == 'Sorry, unable to access page...'
 
       result = yielder.call(page)
     end
     result
   rescue Mechanize::ResponseCodeError => e
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
-    raise e
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '502'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code)
   end
 end

data/lib/spidy/connector/json.rb

@@ -6,7 +6,9 @@
 class Spidy::Connector::Json
   include Spidy::Connector::StaticAccessor
 
-  def initialize(wait_time: nil, user_agent: nil)
+  attr_reader :logger
+
+  def initialize(user_agent: nil)
     @user_agent = user_agent
   end
 
@@ -15,7 +17,9 @@ class Spidy::Connector::Json
     connect(url, &block)
   end
 
-  def connect(url)
+  def connect(url, retry_count: 5)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 end

data/lib/spidy/connector/xml.rb

@@ -6,15 +6,21 @@
 class Spidy::Connector::Xml
   include Spidy::Connector::StaticAccessor
 
-  def call(url)
+  def call(url, &block)
     fail 'URL is undefined' if url.blank?
 
+    connect(url, &block)
+  end
+
+  def connect(url, &block)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
-      yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
+      block.call Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent:)
     @user_agent = user_agent
   end
 end

data/lib/spidy/definition.rb

@@ -28,10 +28,6 @@ module Spidy::Definition
     @socks_proxy = { host: host, port: port }
   end
 
-  def tor?
-    Tor.running?
-  end
-
   def each(source = nil, name: :default, &yielder)
     name = name.presence || :default
     spidy = @namespace[:"#{name}_spider"]

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.7'
+  VERSION = '0.3.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.3.2
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-01 00:00:00.000000000 Z
+date: 2020-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -187,6 +187,9 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- example/master_detail.rb
+- example/proxy.rb
+- example/retry.rb
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb