spidy 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d95a2c20a93c9c60969c4a738200391443f2e797273988d7aaa9a7abc812350
-  data.tar.gz: 789cbc87df6997b5fbdb53db4b3fe7c91b6e899b995d232a227a2395c53f67c6
+  metadata.gz: f123b0f2bb611ea41bafe6a9cb5a7bb7564ba6be7044127e77fddf34dc3faed1
+  data.tar.gz: 87f0079575c62dac9a43d6900b88da45cce77a37f5b994376b0d31dd24b03fa5
 SHA512:
-  metadata.gz: 4e811d34caebb96b1b1d0247ff1a547ed10e5e8a27d82df105c8a809a4b96011e0660805075e8864100d8df29545537c2dde3d3b742e5c35fae33e3a847ad5e9
-  data.tar.gz: fa2f8c4039b009dfc746384dac40028956b880efa96d479485e85b702aad718f4b90e8b44b04912b8aa2bb94d54f27f50570978e2d07078943120bce206e38be
+  metadata.gz: b6554befde64315d01d2db11ba22b479629ae602ca4acddee51c941bd091144849ad15f2b98c220d29bd97c70c02fc2726061db429db2140a50bbfe0fce850a2
+  data.tar.gz: 511b7dccf5f93e4cf8a652d3039ec852c556cbd49e238c676029d1b0e611dce669ac84140c8b9be3f861426e8f248f6452698bf7319f5493cb7ecccb3cd1428d

data/lib/spidy.rb

@@ -4,6 +4,7 @@ require 'spidy/version'
 require 'active_support/all'
 require 'mechanize'
 require 'open-uri'
+require 'socksify'
 
 #
 # web spider dsl engine
@@ -36,6 +37,9 @@ module Spidy
     spidy.instance_eval do
       undef :spider
       undef :define
+      undef :wait_time
+      undef :user_agent
+      undef :socks_proxy
     end
     spidy
   end

data/lib/spidy/binder.rb

@@ -37,10 +37,6 @@ module Spidy::Binder
       @url = url
     end
 
-    def scraper(name, source)
-      lambda { |&block| @spidy.call(source, name: name, &block) }
-    end
-
     def to_s
       to_h.to_json
     end

data/lib/spidy/connector.rb

@@ -10,6 +10,24 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
+  #
+  # retry class
+  #
+  class Retry < StandardError
+    attr_reader :page
+    attr_reader :response_code
+    attr_reader :wait_time
+
+    def initialize(wait_time: 2, page: nil, error: nil)
+      @page = page
+      @wait_time = wait_time
+      @response_code = error.try(:response_code) || page.try(:response_code)
+    end
+  end
+
+  #
+  # default user agent
+  #
   USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
@@ -19,9 +37,19 @@ module Spidy::Connector
     'Safari/537.36'
   ].join(' ')
 
-  def self.get(value)
-    return const_get(value.to_s.classify) if value.is_a?(String) || value.is_a?(Symbol)
+  #
+  # get connection handller
+  #
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
+    return value if value.respond_to?(:call)
+
+    connector = const_get(value.to_s.classify).new(wait_time: wait_time || 5, user_agent: user_agent || USER_AGENT)
+    return connector if socks_proxy.nil?
 
-    value
+    lambda do |url, &block|
+      Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
+        connector.call(url, &block)
+      end
+    end
   end
 end

data/lib/spidy/connector/direct.rb

@@ -3,8 +3,11 @@
 #
 # Direct resource ( not network resource  )
 #
-module Spidy::Connector::Direct
-  def self.call(resource, &yielder)
+class Spidy::Connector::Direct
+  def call(resource, &yielder)
     yielder.call(resource)
   end
+
+  def initialize(wait_time: nil, user_agent: nil)
+  end
 end

data/lib/spidy/connector/html.rb

@@ -3,74 +3,60 @@
 #
 # Mechanize wrapper
 #
-module Spidy::Connector::Html
-  #
-  # retry class
-  #
-  class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
-
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
-      @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
-    end
+class Spidy::Connector::Html
+  def initialize(wait_time:, user_agent:, logger: nil)
+    @wait_time = wait_time
+    @logger = logger || proc { |values| STDERR.puts(values.to_json) }
+    @agent = Mechanize.new
+    @user_agent = user_agent
+    @agent.user_agent = user_agent
   end
 
-  @logger = proc { |values| STDERR.puts(values.to_json) }
-  @agent = Mechanize.new
-  @agent.user_agent = Spidy::Connector::USER_AGENT
-
-  class << self
-    attr_reader :agent
-    attr_accessor :logger
+  attr_reader :agent
+  attr_reader :logger
 
-    def call(url, encoding: nil, retry_count: 5, &yielder)
-      fail 'url is not specified' if url.blank?
-      if encoding
-        agent.default_encoding = encoding
-        agent.force_default_encoding = true
-      end
-      logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-      get(url, retry_count, yielder)
+  def call(url, encoding: nil, retry_count: 5, &yielder)
+    fail 'url is not specified' if url.blank?
+    if encoding
+      agent.default_encoding = encoding
+      agent.force_default_encoding = true
     end
+    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+    get(url, retry_count, yielder)
+  end
 
-    private
+  private
 
-    def get(url, retry_count, yielder)
-      connect(url, retry_count, yielder)
-    rescue Retry => e
-      logger.call('retry.accessed': Time.current,
-                  'retry.uri': url,
-                  'retry.response_code': e.response_code,
-                  'retry.rest_count': retry_count)
+  def get(url, retry_count, yielder)
+    connect(url, retry_count, yielder)
+  rescue Spidy::Connector::Retry => e
+    logger.call('retry.accessed': Time.current,
+                'retry.uri': url,
+                'retry.response_code': e.response_code,
+                'retry.rest_count': retry_count)
 
-      @agent = Mechanize.new
-      @agent.user_agent = Spidy::Connector::USER_AGENT
+    @agent = Mechanize.new
+    @agent.user_agent = @user_agent
 
-      retry_count -= 1
-      if retry_count.positive?
-        sleep e.wait_time
-        retry
-      end
-      raise e
+    retry_count -= 1
+    if retry_count.positive?
+      sleep e.wait_time
+      retry
     end
+    raise e
+  end
 
-    def connect(url, retry_count, yielder)
-      result = nil
-      agent.get(url) do |page|
-        fail Retry, page: page, wait_time: 5 if page.title == 'Sorry, unable to access page...'
+  def connect(url, retry_count, yielder)
+    result = nil
+    agent.get(url) do |page|
+      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
 
-        result = yielder.call(page)
-      end
-      result
-    rescue Mechanize::ResponseCodeError => e
-      raise Retry, error: e if e.response_code == '429'
-      raise Retry, error: e if e.response_code == '502'
-      raise e
+      result = yielder.call(page)
     end
-
+    result
+  rescue Mechanize::ResponseCodeError => e
+    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
+    raise e
   end
 end

data/lib/spidy/connector/json.rb

@@ -3,9 +3,17 @@
 #
 # OpenURI to JSON.parse
 #
-module Spidy::Connector::Json
-  def self.call(url)
+class Spidy::Connector::Json
+  def initialize(wait_time: nil, user_agent: nil)
+    @user_agent = user_agent
+  end
+
+  def call(url, &block)
     fail 'url is not specified' if url.blank?
-    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+    connect(url, &block)
+  end
+
+  def connect(url)
+    OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
   end
 end

data/lib/spidy/connector/xml.rb

@@ -3,12 +3,16 @@
 #
 # xml
 #
-module Spidy::Connector::Xml
-  def self.call(url)
+class Spidy::Connector::Xml
+  def call(url)
     fail 'URL is undefined' if url.blank?
 
-    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) do |body|
+    OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
       yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
   end
+
+  def initialize(wait_time: nil, user_agent: nil)
+    @user_agent = user_agent
+  end
 end

data/lib/spidy/console.rb

@@ -18,4 +18,9 @@ class Spidy::Console
   def reload!
     @definition_file.eval_definition
   end
+
+  def connector(url, as:)
+    connector = Spidy::Connector.get(as)
+    connector.call(url) { |page| break page }
+  end
 end

data/lib/spidy/definition.rb

@@ -16,6 +16,18 @@ module Spidy::Definition
     spidy.call(source, &yielder)
   end
 
+  def user_agent(user_agent)
+    @user_agent = user_agent
+  end
+
+  def wait_time(wait_time)
+    @wait_time = wait_time
+  end
+
+  def socks_proxy(host, port)
+    @socks_proxy = { host: host, port: port }
+  end
+
   def each(source = nil, name: :default, &yielder)
     name = name.presence || :default
     spidy = @namespace[:"#{name}_spider"]
@@ -26,7 +38,7 @@ module Spidy::Definition
 
   def spider(name = :default, connector: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
     @namespace[:"#{name}_spider"] = proc do |source, &yielder|
       define_block.call(yielder, connector, source)
     end
@@ -34,7 +46,7 @@ module Spidy::Definition
 
   def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent)
     binder = Spidy::Binder.get(self, binder || as)
     @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
   end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.4'
+  VERSION = '0.2.5'
 end

data/spidy.gemspec

@@ -33,5 +33,6 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'
+  spec.add_runtime_dependency 'socksify'
   spec.add_runtime_dependency 'pry'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - aileron
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-31 00:00:00.000000000 Z
+date: 2020-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: socksify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +150,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - aileron.cc@gmail.com
 executables:
@@ -183,7 +197,7 @@ homepage: https://github.com/aileron-inc/spidy
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -199,7 +213,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: web spider dsl
 test_files: []