spidy 0.2.2 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c032c78b7580bada781db301697b264b337d7c829bc5df4c3810a95770f56ee
-  data.tar.gz: 4b282b06cc91b3fcdef8187381ded42d01a689720ebfede4fb135b1639d47f11
+  metadata.gz: 177e6b5e3b4f864251d7071d23b2b42b219d8193673a5b57ee9b376e393092a9
+  data.tar.gz: 92b39a277d752eb5b1944f822702eae803356f7f5e3211b1f91e2d9ad635090e
 SHA512:
-  metadata.gz: de20181e766b5b8d9189e657bfa20f4a4720e2c0a0adb7ece522309433ef5d93325d98d63a1b31304bd9a2575391318c3330c01f66356ae4110b7005c6560253
-  data.tar.gz: 5cf49000bb2f234c111216f9a554267df18a360c4b0511685c000b8e22312feaf3ba5b0b53b56612f0653157cd51e1da46e6140d7e4902b8a4a062894256b1d1
+  metadata.gz: b4078e3d1ad9813b0273f267d99bf7a9bf4473bb86b6cfa4fa283e6c5b060fa248e0b6fb517bede448f94e0dcebbb90ee71ef4eeb4bd5a45b1405c50005985fe
+  data.tar.gz: 00aa2cd512cf3a083082227234f59148d86e8ee0e4f602e07d1c67cbf9f3e8160b9144ebd29f3a67efded607c1705c2ce3008df235325afc7f3a715a6dd7527d

data/.ruby-version

@@ -1 +1 @@
-2.6.5
+2.6.6

data/Gemfile.lock

@@ -1,30 +1,32 @@
 PATH
   remote: .
   specs:
-    spidy (0.1.6)
+    spidy (0.2.6)
       activesupport
       mechanize
       pry
+      socksify
+      tor
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.2.1)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.2)
+      zeitwerk (~> 2.2, >= 2.2.2)
     coderay (1.1.2)
-    concurrent-ruby (1.1.5)
-    connection_pool (2.2.2)
+    concurrent-ruby (1.1.7)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     ffaker (2.10.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    i18n (1.8.2)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
     mechanize (2.7.6)
       domain_name (~> 0.5, >= 0.5.1)
@@ -38,14 +40,14 @@ GEM
     method_source (0.9.2)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
-    minitest (5.14.0)
+    minitest (5.14.2)
     mixlib-shellout (2.4.4)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (3.1.0)
+    net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
-    nokogiri (1.10.7)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     ntlm-http (0.1.1)
     pry (0.12.2)
@@ -72,14 +74,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.2)
+    socksify (1.7.1)
     thread_safe (0.3.6)
-    tzinfo (1.2.6)
+    tor (0.1.4)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
+    unf_ext (0.0.7.7)
     webrobots (0.1.2)
-    zeitwerk (2.2.2)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
@@ -94,4 +98,4 @@ DEPENDENCIES
   spidy!
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -60,7 +60,7 @@ call('http://example.com') { |html| break html } # html as nokogiri object ( mec
 ```
 
 ### When used from the ruby code
-``
+```rb
 a = Spidy.define do
   # Implementing spiders and scrapers
 end

data/lib/spidy.rb

@@ -4,13 +4,14 @@ require 'spidy/version'
 require 'active_support/all'
 require 'mechanize'
 require 'open-uri'
+require 'socksify'
+require 'tor'
 
 #
 # web spider dsl engine
 #
 module Spidy
   extend ActiveSupport::Autoload
-  autoload :Interface
   autoload :Shell
   autoload :CommandLine
   autoload :Console
@@ -34,6 +35,13 @@ module Spidy
         module_eval(&block)
       end
     end
-    Spidy::Interface.new(spidy)
+    spidy.instance_eval do
+      undef :spider
+      undef :define
+      undef :wait_time
+      undef :user_agent
+      undef :socks_proxy
+    end
+    spidy
   end
 end

data/lib/spidy/binder.rb

@@ -37,10 +37,6 @@ module Spidy::Binder
       @url = url
     end
 
-    def scraper(name, source)
-      lambda { |&block| @spidy.call(source, name: name, &block) }
-    end
-
     def to_s
       to_h.to_json
     end

data/lib/spidy/connector.rb

@@ -10,6 +10,33 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
+  module StaticAccessor
+    extend ActiveSupport::Concern
+    class_methods do
+      def call(url, wait_time: nil, user_agent: Spidy::Connector::USER_AGENT, &block)
+        new(wait_time: wait_time, user_agent: user_agent).call(url, &block)
+      end
+    end
+  end
+
+  #
+  # retry class
+  #
+  class Retry < StandardError
+    attr_reader :page
+    attr_reader :response_code
+    attr_reader :wait_time
+
+    def initialize(wait_time: 2, page: nil, error: nil)
+      @page = page
+      @wait_time = wait_time
+      @response_code = error.try(:response_code) || page.try(:response_code)
+    end
+  end
+
+  #
+  # default user agent
+  #
   USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
@@ -19,9 +46,19 @@ module Spidy::Connector
     'Safari/537.36'
   ].join(' ')
 
-  def self.get(value)
-    return const_get(value.to_s.classify) if value.is_a?(String) || value.is_a?(Symbol)
+  #
+  # get connection handller
+  #
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
+    return value if value.respond_to?(:call)
+
+    connector = const_get(value.to_s.classify).new(wait_time: wait_time || 5, user_agent: user_agent || USER_AGENT)
+    return connector if socks_proxy.nil?
 
-    value
+    lambda do |url, &block|
+      Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
+        connector.call(url, &block)
+      end
+    end
   end
 end

data/lib/spidy/connector/direct.rb

@@ -3,8 +3,11 @@
 #
 # Direct resource ( not network resource  )
 #
-module Spidy::Connector::Direct
-  def self.call(resource, &yielder)
+class Spidy::Connector::Direct
+  def call(resource, &yielder)
     yielder.call(resource)
   end
+
+  def initialize(wait_time: nil, user_agent: nil)
+  end
 end

data/lib/spidy/connector/html.rb

@@ -3,73 +3,62 @@
 #
 # Mechanize wrapper
 #
-module Spidy::Connector::Html
-  #
-  # retry class
-  #
-  class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
+class Spidy::Connector::Html
+  include Spidy::Connector::StaticAccessor
 
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
-      @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
-    end
+  def initialize(wait_time:, user_agent:, logger: nil)
+    @wait_time = wait_time
+    @logger = logger || proc { |values| STDERR.puts(values.to_json) }
+    @agent = Mechanize.new
+    @user_agent = user_agent
+    @agent.user_agent = user_agent
   end
 
-  @logger = proc { |values| STDERR.puts(values.to_json) }
-  @agent = Mechanize.new
-  @agent.user_agent = Spidy::Connector::USER_AGENT
-
-  class << self
-    attr_reader :agent
-    attr_accessor :logger
+  attr_reader :agent
+  attr_reader :logger
 
-    def call(url, encoding: nil, retry_count: 5, &yielder)
-      fail 'url is not specified' if url.blank?
-      if encoding
-        agent.default_encoding = encoding
-        agent.force_default_encoding = true
-      end
-      logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-      get(url, retry_count, yielder)
+  def call(url, encoding: nil, retry_count: 5, &yielder)
+    fail 'url is not specified' if url.blank?
+    if encoding
+      agent.default_encoding = encoding
+      agent.force_default_encoding = true
     end
+    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+    get(url, retry_count, yielder)
+  end
 
-    private
+  private
 
-    def get(url, retry_count, yielder)
-      connect(url, retry_count, yielder)
-    rescue Retry => e
-      logger.call('retry.accessed': Time.current,
-                  'retry.uri': url,
-                  'retry.response_code': e.response_code,
-                  'retry.rest_count': retry_count)
+  def get(url, retry_count, yielder)
+    connect(url, retry_count, yielder)
+  rescue Spidy::Connector::Retry => e
+    logger.call('retry.accessed': Time.current,
+                'retry.uri': url,
+                'retry.response_code': e.response_code,
+                'retry.rest_count': retry_count)
 
-      @agent = Mechanize.new
-      @agent.user_agent = Spidy::Connector::USER_AGENT
+    @agent = Mechanize.new
+    @agent.user_agent = @user_agent
 
-      retry_count -= 1
-      if retry_count.positive?
-        sleep e.wait_time
-        retry
-      end
-      raise e
+    retry_count -= 1
+    if retry_count.positive?
+      sleep e.wait_time
+      retry
     end
+    raise e
+  end
 
-    def connect(url, retry_count, yielder)
-      result = nil
-      agent.get(url) do |page|
-        fail Retry, page: page, wait_time: 5 if page.title == 'Sorry, unable to access page...'
+  def connect(url, retry_count, yielder)
+    result = nil
+    agent.get(url) do |page|
+      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
 
-        result = yielder.call(page)
-      end
-      result
-    rescue Mechanize::ResponseCodeError => e
-      raise Retry, error: e if e.response_code == '429'
-      raise e
+      result = yielder.call(page)
     end
-
+    result
+  rescue Mechanize::ResponseCodeError => e
+    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
+    raise e
   end
 end

data/lib/spidy/connector/json.rb

@@ -3,9 +3,19 @@
 #
 # OpenURI to JSON.parse
 #
-module Spidy::Connector::Json
-  def self.call(url)
+class Spidy::Connector::Json
+  include Spidy::Connector::StaticAccessor
+
+  def initialize(wait_time: nil, user_agent: nil)
+    @user_agent = user_agent
+  end
+
+  def call(url, &block)
     fail 'url is not specified' if url.blank?
-    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+    connect(url, &block)
+  end
+
+  def connect(url)
+    OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
   end
 end

data/lib/spidy/connector/xml.rb

@@ -3,12 +3,18 @@
 #
 # xml
 #
-module Spidy::Connector::Xml
-  def self.call(url)
+class Spidy::Connector::Xml
+  include Spidy::Connector::StaticAccessor
+
+  def call(url)
     fail 'URL is undefined' if url.blank?
 
-    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) do |body|
+    OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
       yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
   end
+
+  def initialize(wait_time: nil, user_agent: nil)
+    @user_agent = user_agent
+  end
 end

data/lib/spidy/console.rb

@@ -18,4 +18,9 @@ class Spidy::Console
   def reload!
     @definition_file.eval_definition
   end
+
+  def connector(url, as:)
+    connector = Spidy::Connector.get(as)
+    connector.call(url) { |page| break page }
+  end
 end

data/lib/spidy/definition.rb

@@ -16,6 +16,22 @@ module Spidy::Definition
     spidy.call(source, &yielder)
   end
 
+  def user_agent(user_agent)
+    @user_agent = user_agent
+  end
+
+  def wait_time(wait_time)
+    @wait_time = wait_time
+  end
+
+  def socks_proxy(host, port)
+    @socks_proxy = { host: host, port: port }
+  end
+
+  def tor?
+    Tor.running?
+  end
+
   def each(source = nil, name: :default, &yielder)
     name = name.presence || :default
     spidy = @namespace[:"#{name}_spider"]
@@ -26,7 +42,7 @@ module Spidy::Definition
 
   def spider(name = :default, connector: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
     @namespace[:"#{name}_spider"] = proc do |source, &yielder|
       define_block.call(yielder, connector, source)
     end
@@ -34,7 +50,7 @@ module Spidy::Definition
 
   def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
     binder = Spidy::Binder.get(self, binder || as)
     @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
   end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.2'
+  VERSION = '0.2.7'
 end

data/spidy.gemspec

@@ -31,7 +31,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'
 
+  spec.add_runtime_dependency 'tor'
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'
+  spec.add_runtime_dependency 'socksify'
   spec.add_runtime_dependency 'pry'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.7
 platform: ruby
 authors:
 - aileron
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-03-09 00:00:00.000000000 Z
+date: 2020-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: socksify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +164,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - aileron.cc@gmail.com
 executables:
@@ -174,7 +202,6 @@ files:
 - lib/spidy/console.rb
 - lib/spidy/definition.rb
 - lib/spidy/definition_file.rb
-- lib/spidy/interface.rb
 - lib/spidy/shell.rb
 - lib/spidy/spider.rb
 - lib/spidy/version.rb
@@ -184,7 +211,7 @@ homepage: https://github.com/aileron-inc/spidy
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -199,8 +226,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: web spider dsl
 test_files: []

data/lib/spidy/interface.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-#
-# spidy interface
-#
-class Spidy::Interface
-  delegate :call, :each, :namespace, to: :@spidy
-
-  def initialize(spidy)
-    @spidy = spidy
-  end
-end