spidy 0.2.4 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d95a2c20a93c9c60969c4a738200391443f2e797273988d7aaa9a7abc812350
-  data.tar.gz: 789cbc87df6997b5fbdb53db4b3fe7c91b6e899b995d232a227a2395c53f67c6
+  metadata.gz: 44d16509821e1779d821effb7c571d2a11cbb79166168943242628b2e53457ec
+  data.tar.gz: 5b52389a8042c9e69aaeead0d96b81553da2e5f5e51ad16696f87249e38067bd
 SHA512:
-  metadata.gz: 4e811d34caebb96b1b1d0247ff1a547ed10e5e8a27d82df105c8a809a4b96011e0660805075e8864100d8df29545537c2dde3d3b742e5c35fae33e3a847ad5e9
-  data.tar.gz: fa2f8c4039b009dfc746384dac40028956b880efa96d479485e85b702aad718f4b90e8b44b04912b8aa2bb94d54f27f50570978e2d07078943120bce206e38be
+  metadata.gz: 3475b05ea1235b388960416a03ab96adb64608ce4046d1812b4d80f038a91b99343882e45413ee6a2f74e29626869aca8a48d34f8e57c64b480c8893efcda123
+  data.tar.gz: 7e4fd1abf5898a7a7273d4eaf53021e4c1898d995e8e0d23abfcbf80c3e1b45b2f059c95e48707577bae72a4f158d7cba7f9be097e7e8a277d89e683386c32d6

data/Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    spidy (0.2.3)
+    spidy (0.2.9)
       activesupport
       mechanize
       pry
+      socksify
+      tor
 
 GEM
   remote: https://rubygems.org/
@@ -16,7 +18,7 @@ GEM
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2, >= 2.2.2)
     coderay (1.1.2)
-    concurrent-ruby (1.1.6)
+    concurrent-ruby (1.1.7)
     connection_pool (2.2.3)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
@@ -40,7 +42,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
-    minitest (5.14.1)
+    minitest (5.14.2)
     mixlib-shellout (2.4.4)
     net-http-digest_auth (1.4.1)
     net-http-persistent (4.0.0)
@@ -72,7 +74,9 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.2)
+    socksify (1.7.1)
     thread_safe (0.3.6)
+    tor (0.1.4)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unf (0.1.4)

data/lib/spidy.rb

@@ -4,6 +4,8 @@ require 'spidy/version'
 require 'active_support/all'
 require 'mechanize'
 require 'open-uri'
+require 'socksify'
+require 'tor'
 
 #
 # web spider dsl engine
@@ -36,6 +38,9 @@ module Spidy
     spidy.instance_eval do
       undef :spider
       undef :define
+      undef :wait_time
+      undef :user_agent
+      undef :socks_proxy
     end
     spidy
   end

data/lib/spidy/binder.rb

@@ -37,10 +37,6 @@ module Spidy::Binder
       @url = url
     end
 
-    def scraper(name, source)
-      lambda { |&block| @spidy.call(source, name: name, &block) }
-    end
-
     def to_s
       to_h.to_json
     end

data/lib/spidy/connector.rb

@@ -10,6 +10,9 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
+  #
+  # default user agent
+  #
   USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
@@ -19,9 +22,77 @@ module Spidy::Connector
     'Safari/537.36'
   ].join(' ')
 
-  def self.get(value)
-    return const_get(value.to_s.classify) if value.is_a?(String) || value.is_a?(Symbol)
+  #
+  # error output logger
+  #
+  DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
 
-    value
+  #
+  # static method
+  #
+  module StaticAccessor
+    extend ActiveSupport::Concern
+    class_methods do
+      def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
+        new(wait_time: wait_time, user_agent: user_agent, logger: logger).call(url, &block)
+      end
+    end
+  end
+
+  #
+  # retry class
+  #
+  class Retry < StandardError
+    attr_reader :page
+    attr_reader :response_code
+    attr_reader :wait_time
+
+    def initialize(wait_time: 2, page: nil, error: nil)
+      @page = page
+      @wait_time = wait_time
+      @response_code = error.try(:response_code) || page.try(:response_code)
+    end
+  end
+
+  class Builder
+    attr_reader :origin_connector, :proxy_connector
+
+    def initialize(connector, socks_proxy)
+      @socks_proxy = socks_proxy
+      @origin_connector = connector
+      @proxy_connector =
+        lambda do |url, &block|
+          Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
+            connector.call(url, &block)
+          end
+        end
+    end
+
+    def proxy_disabled?
+      !tor?
+    end
+
+    def tor?
+      Tor::Controller.new(host: @socks_proxy[:host], port: @socks_proxy[:port]).close
+      true
+    rescue Errno::ECONNREFUSED
+      false
+    end
+  end
+
+  #
+  # get connection handller
+  #
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
+    return value if value.respond_to?(:call)
+
+    builder = Builder.new(const_get(value.to_s.classify).new(
+      wait_time: wait_time || 5,
+      user_agent: user_agent || USER_AGENT,
+      logger: logger || DEFAULT_LOGGER,
+    ), socks_proxy)
+    return builder.origin_connector if socks_proxy.nil? || builder.proxy_disabled?
+
+    builder.proxy_connector
   end
 end

data/lib/spidy/connector/direct.rb

@@ -3,8 +3,11 @@
 #
 # Direct resource ( not network resource  )
 #
-module Spidy::Connector::Direct
-  def self.call(resource, &yielder)
+class Spidy::Connector::Direct
+  def call(resource, &yielder)
     yielder.call(resource)
   end
+
+  def initialize(wait_time: nil, user_agent: nil, logger: nil)
+  end
 end

data/lib/spidy/connector/html.rb

@@ -3,74 +3,62 @@
 #
 # Mechanize wrapper
 #
-module Spidy::Connector::Html
-  #
-  # retry class
-  #
-  class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
+class Spidy::Connector::Html
+  include Spidy::Connector::StaticAccessor
 
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
-      @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
-    end
+  def initialize(wait_time:, user_agent:, logger: nil)
+    @wait_time = wait_time
+    @logger = logger
+    @agent = Mechanize.new
+    @user_agent = user_agent
+    @agent.user_agent = user_agent
   end
 
-  @logger = proc { |values| STDERR.puts(values.to_json) }
-  @agent = Mechanize.new
-  @agent.user_agent = Spidy::Connector::USER_AGENT
-
-  class << self
-    attr_reader :agent
-    attr_accessor :logger
+  attr_reader :agent
+  attr_reader :logger
 
-    def call(url, encoding: nil, retry_count: 5, &yielder)
-      fail 'url is not specified' if url.blank?
-      if encoding
-        agent.default_encoding = encoding
-        agent.force_default_encoding = true
-      end
-      logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-      get(url, retry_count, yielder)
+  def call(url, encoding: nil, retry_count: 5, &yielder)
+    fail 'url is not specified' if url.blank?
+    if encoding
+      agent.default_encoding = encoding
+      agent.force_default_encoding = true
     end
+    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+    get(url, retry_count, yielder)
+  end
 
-    private
+  private
 
-    def get(url, retry_count, yielder)
-      connect(url, retry_count, yielder)
-    rescue Retry => e
-      logger.call('retry.accessed': Time.current,
-                  'retry.uri': url,
-                  'retry.response_code': e.response_code,
-                  'retry.rest_count': retry_count)
+  def get(url, retry_count, yielder)
+    connect(url, retry_count, yielder)
+  rescue Spidy::Connector::Retry => e
+    logger.call('retry.accessed': Time.current,
+                'retry.uri': url,
+                'retry.response_code': e.response_code,
+                'retry.rest_count': retry_count)
 
-      @agent = Mechanize.new
-      @agent.user_agent = Spidy::Connector::USER_AGENT
+    @agent = Mechanize.new
+    @agent.user_agent = @user_agent
 
-      retry_count -= 1
-      if retry_count.positive?
-        sleep e.wait_time
-        retry
-      end
-      raise e
+    retry_count -= 1
+    if retry_count.positive?
+      sleep e.wait_time
+      retry
     end
+    raise e
+  end
 
-    def connect(url, retry_count, yielder)
-      result = nil
-      agent.get(url) do |page|
-        fail Retry, page: page, wait_time: 5 if page.title == 'Sorry, unable to access page...'
+  def connect(url, retry_count, yielder)
+    result = nil
+    agent.get(url) do |page|
+      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
 
-        result = yielder.call(page)
-      end
-      result
-    rescue Mechanize::ResponseCodeError => e
-      raise Retry, error: e if e.response_code == '429'
-      raise Retry, error: e if e.response_code == '502'
-      raise e
+      result = yielder.call(page)
     end
-
+    result
+  rescue Mechanize::ResponseCodeError => e
+    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
+    raise e
   end
 end

data/lib/spidy/connector/json.rb

@@ -3,9 +3,35 @@
 #
 # OpenURI to JSON.parse
 #
-module Spidy::Connector::Json
-  def self.call(url)
+class Spidy::Connector::Json
+  include Spidy::Connector::StaticAccessor
+
+  attr_reader :logger
+
+  def initialize(wait_time: nil, user_agent: nil, logger: nil)
+    @wait_time = wait_time
+    @user_agent = user_agent
+    @logger = logger
+  end
+
+  def call(url, &block)
     fail 'url is not specified' if url.blank?
-    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+    connect(url, &block)
+  end
+
+  def connect(url, retry_count: 5)
+    OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+  rescue OpenURI::HTTPError => e
+    logger.call('retry.accessed': Time.current,
+                'retry.uri': url,
+                'retry.response_code': e.message,
+                'retry.rest_count': retry_count)
+
+    retry_count -= 1
+    if retry_count.positive?
+      sleep @wait_time
+      retry
+    end
+    raise e
   end
 end

data/lib/spidy/connector/xml.rb

@@ -3,12 +3,18 @@
 #
 # xml
 #
-module Spidy::Connector::Xml
-  def self.call(url)
+class Spidy::Connector::Xml
+  include Spidy::Connector::StaticAccessor
+
+  def call(url)
     fail 'URL is undefined' if url.blank?
 
-    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) do |body|
+    OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
       yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
   end
+
+  def initialize(user_agent: nil, logger: nil)
+    @user_agent = user_agent
+  end
 end

data/lib/spidy/console.rb

@@ -18,4 +18,9 @@ class Spidy::Console
   def reload!
     @definition_file.eval_definition
   end
+
+  def connector(url, as:)
+    connector = Spidy::Connector.get(as)
+    connector.call(url) { |page| break page }
+  end
 end

data/lib/spidy/definition.rb

@@ -16,6 +16,18 @@ module Spidy::Definition
     spidy.call(source, &yielder)
   end
 
+  def user_agent(user_agent)
+    @user_agent = user_agent
+  end
+
+  def wait_time(wait_time)
+    @wait_time = wait_time
+  end
+
+  def socks_proxy(host, port)
+    @socks_proxy = { host: host, port: port }
+  end
+
   def each(source = nil, name: :default, &yielder)
     name = name.presence || :default
     spidy = @namespace[:"#{name}_spider"]
@@ -26,7 +38,7 @@ module Spidy::Definition
 
   def spider(name = :default, connector: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
     @namespace[:"#{name}_spider"] = proc do |source, &yielder|
       define_block.call(yielder, connector, source)
     end
@@ -34,7 +46,7 @@ module Spidy::Definition
 
   def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
     binder = Spidy::Binder.get(self, binder || as)
     @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
   end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.4'
+  VERSION = '0.2.9'
 end

data/spidy.gemspec

@@ -31,7 +31,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'
 
+  spec.add_runtime_dependency 'tor'
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'
+  spec.add_runtime_dependency 'socksify'
   spec.add_runtime_dependency 'pry'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.9
 platform: ruby
 authors:
 - aileron
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-31 00:00:00.000000000 Z
+date: 2020-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: socksify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +164,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - aileron.cc@gmail.com
 executables:
@@ -183,7 +211,7 @@ homepage: https://github.com/aileron-inc/spidy
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -199,7 +227,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: web spider dsl
 test_files: []