spidy 0.2.5 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f123b0f2bb611ea41bafe6a9cb5a7bb7564ba6be7044127e77fddf34dc3faed1
-  data.tar.gz: 87f0079575c62dac9a43d6900b88da45cce77a37f5b994376b0d31dd24b03fa5
+  metadata.gz: f87cda14101ec7c184d3e3134f25601cf3c0d2ef2d22b9766f5ce2b417734212
+  data.tar.gz: 803f7012304d475280949f742fb5da7af588dc9f2ab85e3991973b4c60a81a46
 SHA512:
-  metadata.gz: b6554befde64315d01d2db11ba22b479629ae602ca4acddee51c941bd091144849ad15f2b98c220d29bd97c70c02fc2726061db429db2140a50bbfe0fce850a2
-  data.tar.gz: 511b7dccf5f93e4cf8a652d3039ec852c556cbd49e238c676029d1b0e611dce669ac84140c8b9be3f861426e8f248f6452698bf7319f5493cb7ecccb3cd1428d
+  metadata.gz: cd725d2f1a697d4f912e5b37113de561bb002423db61a666cf06404a4bcb3c46765ef1b54dd024f6dd4760f73dc75e285e828e2672348d70e3cb8d71861f5c5f
+  data.tar.gz: ba4550f6871bad2c10f8ad1964e6fd30534f66e02d6f034f916e1afa5247a0f23249fa91842b60477f6f2cbbe2829d8f7edf279655b020eeeb4d242b26d81970

data/Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    spidy (0.2.3)
+    spidy (0.2.9)
       activesupport
       mechanize
       pry
+      socksify
+      tor
 
 GEM
   remote: https://rubygems.org/
@@ -16,7 +18,7 @@ GEM
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2, >= 2.2.2)
     coderay (1.1.2)
-    concurrent-ruby (1.1.6)
+    concurrent-ruby (1.1.7)
     connection_pool (2.2.3)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
@@ -40,7 +42,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
-    minitest (5.14.1)
+    minitest (5.14.2)
     mixlib-shellout (2.4.4)
     net-http-digest_auth (1.4.1)
     net-http-persistent (4.0.0)
@@ -72,7 +74,9 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.2)
+    socksify (1.7.1)
     thread_safe (0.3.6)
+    tor (0.1.4)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unf (0.1.4)

data/example/master_detail.rb

@@ -0,0 +1,73 @@
+
+
+Spidy.define do
+  url_to_params = ->(url) {
+    uri = URI.parse(url)
+    params = URI.decode_www_form(uri.query).to_h if uri.query.present?
+    params if params.present?
+  }
+
+  master_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    page = params&.dig('page')&.to_i || 0
+
+    limit_page = 3
+    per_page = 25
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.main {
+            (page * per_page + 1).upto((page + 1) * per_page).each do |i|
+              doc.a("page #{i}", href: "http://localhost/?id=#{i}")
+            end
+          }
+          doc.a('NEXT', href: "http://localhost/?page=#{page + 1}", class: 'next') if page < limit_page
+        }
+      }
+    }.doc)
+  }
+
+  detail_page = proc { |url, &yielder|
+    params = url_to_params.call(url)
+    id = params['id']
+
+    yielder.call(Nokogiri::HTML::Builder.new { |doc|
+      doc.html {
+        doc.body {
+          doc.span.bold {
+            doc.text "Hello world"
+          }
+          doc.h1("title_#{id}", id: 'title')
+          doc.main("body_#{id}", id: 'body')
+          doc.div.sub do
+            doc.span.name('testtest')
+          end
+        }
+      }
+    }.doc)
+  }
+
+  define(as: :html, connector: detail_page) do
+    let(:title, '#title')
+    let(:body, '#body')
+  end
+
+  define(:sub, as: :html, connector: :direct) do
+    let(:name, '.name')
+  end
+
+  spider(as: :html, connector: master_page) do |yielder, connector|
+    next_url = 'http://localhost'
+    while next_url.present?
+      connector.call(next_url) do |page|
+        page.search('main a').each do |a|
+          yielder.call(a.attr('href'))
+        end
+        next_url = page.at('a.next')&.attr('href')
+      end
+    end
+  end
+end

data/example/proxy.rb

@@ -0,0 +1,10 @@
+Spidy.define do
+  user_agent 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0'
+  socks_proxy '127.0.0.1', 9050
+
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/ip') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/example/retry.rb

@@ -0,0 +1,7 @@
+Spidy.define do
+  spider(as: :json) do |yielder, connector|
+    connector.call('https://httpbin.org/status/500') do |json|
+      yielder.call(json[:origin])
+    end
+  end
+end

data/lib/spidy.rb

@@ -5,6 +5,7 @@ require 'active_support/all'
 require 'mechanize'
 require 'open-uri'
 require 'socksify'
+require 'tor'
 
 #
 # web spider dsl engine

data/lib/spidy/connector.rb

@@ -10,25 +10,12 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
-  #
-  # retry class
-  #
-  class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
-
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
-      @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
-    end
-  end
+  DEFAULT_WAIT_TIME = 5
 
   #
   # default user agent
   #
-  USER_AGENT = [
+  DEFAULT_USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
     'AppleWebKit/537.36',
@@ -38,18 +25,121 @@ module Spidy::Connector
   ].join(' ')
 
   #
-  # get connection handller
+  # error output logger
   #
-  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
-    return value if value.respond_to?(:call)
+  DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
 
-    connector = const_get(value.to_s.classify).new(wait_time: wait_time || 5, user_agent: user_agent || USER_AGENT)
-    return connector if socks_proxy.nil?
+  #
+  # static method
+  #
+  module StaticAccessor
+    extend ActiveSupport::Concern
+    class_methods do
+      def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
+        new(wait_time: wait_time, user_agent: user_agent, logger: logger).call(url, &block)
+      end
+    end
+  end
+
+  #
+  # retry class
+  #
+  class Retry < StandardError
+    attr_reader :object, :response_code, :error
+
+    def initialize(object: nil, error: nil, response_code: nil)
+      @object = object
+      @response_code = response_code
+      @error = error
+    end
+  end
+
+  #
+  # retry
+  #
+  class Retryable
+    attr_reader :origin_connector
+
+    def initialize(connector, logger:, wait_time:)
+      @origin_connector = connector
+      @logger = logger
+      @wait_time = wait_time
+      @retry_attempt_count = 5
+    end
+
+    def call(url, &block)
+      connect(url, &block)
+    end
+
+    def connect(url, retry_attempt_count: @retry_attempt_count, &block)
+      @logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+      @origin_connector.call(url, &block)
+    rescue Spidy::Connector::Retry => e
+      @logger.call('retry.accessed': Time.current,
+                   'retry.uri': url,
+                   'retry.response_code': e.response_code,
+                   'retry.attempt_count': retry_attempt_count)
+
+      retry_attempt_count -= 1
+      if retry_attempt_count.positive?
+        sleep @wait_time
+        @origin_connector.refresh! if @origin_connector.respond_to?(:refresh!)
+        retry
+      end
+      raise e.error
+    end
+  end
 
-    lambda do |url, &block|
+  #
+  # tor proxy
+  #
+  class TorConnector
+    attr_reader :connector, :socks_proxy
+
+    def initialize(connector, socks_proxy)
+      @connector = connector
+      @socks_proxy = socks_proxy
+    end
+
+    def call(url, &block)
       Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
         connector.call(url, &block)
       end
     end
+
+    def try_connection?
+      try_connection!
+      true
+    rescue Errno::ECONNREFUSED
+      false
+    end
+
+    def try_connection!
+      Tor::Controller.new(host: @socks_proxy[:host], port: @socks_proxy[:port]).close
+    end
+  end
+
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
+    logger ||= DEFAULT_LOGGER
+    user_agent ||= DEFAULT_USER_AGENT
+    wait_time ||= DEFAULT_WAIT_TIME
+
+    connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
+    Retryable.new(connector, wait_time: wait_time, logger: logger)
+  end
+
+  #
+  # get connection handller
+  #
+  def self.get_connector(value, user_agent: nil, socks_proxy: nil)
+    return value if value.respond_to?(:call)
+
+    connector = const_get(value.to_s.classify).new(user_agent: user_agent)
+    fail "Not defined connnector[#{value}]" if connector.nil?
+    return connector if socks_proxy.nil?
+
+    tor = TorConnector.new(connnector, socks_proxy)
+    tor.try_connection!
+    tor
   end
 end

data/lib/spidy/connector/direct.rb

@@ -8,6 +8,6 @@ class Spidy::Connector::Direct
     yielder.call(resource)
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent:)
   end
 end

data/lib/spidy/connector/html.rb

@@ -4,16 +4,15 @@
 # Mechanize wrapper
 #
 class Spidy::Connector::Html
-  def initialize(wait_time:, user_agent:, logger: nil)
-    @wait_time = wait_time
-    @logger = logger || proc { |values| STDERR.puts(values.to_json) }
+  include Spidy::Connector::StaticAccessor
+
+  def initialize(user_agent:)
     @agent = Mechanize.new
     @user_agent = user_agent
     @agent.user_agent = user_agent
   end
 
   attr_reader :agent
-  attr_reader :logger
 
   def call(url, encoding: nil, retry_count: 5, &yielder)
     fail 'url is not specified' if url.blank?
@@ -21,42 +20,27 @@ class Spidy::Connector::Html
       agent.default_encoding = encoding
       agent.force_default_encoding = true
     end
-    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-    get(url, retry_count, yielder)
-  end
-
-  private
-
-  def get(url, retry_count, yielder)
     connect(url, retry_count, yielder)
-  rescue Spidy::Connector::Retry => e
-    logger.call('retry.accessed': Time.current,
-                'retry.uri': url,
-                'retry.response_code': e.response_code,
-                'retry.rest_count': retry_count)
+  end
 
+  def refresh!
     @agent = Mechanize.new
     @agent.user_agent = @user_agent
-
-    retry_count -= 1
-    if retry_count.positive?
-      sleep e.wait_time
-      retry
-    end
-    raise e
   end
 
+  private
+
   def connect(url, retry_count, yielder)
     result = nil
     agent.get(url) do |page|
-      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
+      fail Spidy::Connector::Retry, object: page, response_code: page.try(:response_code) if page.title == 'Sorry, unable to access page...'
 
       result = yielder.call(page)
     end
     result
   rescue Mechanize::ResponseCodeError => e
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
-    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
-    raise e
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '502'
+    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code)
   end
 end

data/lib/spidy/connector/json.rb

@@ -4,7 +4,11 @@
 # OpenURI to JSON.parse
 #
 class Spidy::Connector::Json
-  def initialize(wait_time: nil, user_agent: nil)
+  include Spidy::Connector::StaticAccessor
+
+  attr_reader :logger
+
+  def initialize(user_agent: nil)
     @user_agent = user_agent
   end
 
@@ -13,7 +17,9 @@ class Spidy::Connector::Json
     connect(url, &block)
   end
 
-  def connect(url)
+  def connect(url, retry_count: 5)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 end

data/lib/spidy/connector/xml.rb

@@ -4,15 +4,23 @@
 # xml
 #
 class Spidy::Connector::Xml
-  def call(url)
+  include Spidy::Connector::StaticAccessor
+
+  def call(url, &block)
     fail 'URL is undefined' if url.blank?
 
+    connect(url, &block)
+  end
+
+  def connect(url, &block)
     OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
-      yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
+      block.call Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
+  rescue OpenURI::HTTPError => e
+    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
   end
 
-  def initialize(wait_time: nil, user_agent: nil)
+  def initialize(user_agent:)
     @user_agent = user_agent
   end
 end

data/lib/spidy/definition.rb

@@ -46,7 +46,7 @@ module Spidy::Definition
 
   def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
     binder = Spidy::Binder.get(self, binder || as)
     @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
   end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.5'
+  VERSION = '0.3.0'
 end

data/spidy.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'
 
+  spec.add_runtime_dependency 'tor'
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'
   spec.add_runtime_dependency 'socksify'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-31 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -173,6 +187,9 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- example/master_detail.rb
+- example/proxy.rb
+- example/retry.rb
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb