spidy 0.2.1 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8027a668da137f729aebe8b67e3053fc1631e723580ba7126f5b6983bb234471
-  data.tar.gz: 764df48ccba3dc32db8a022bb16576e9e5177a0835a84f25eee4294a745840ab
+  metadata.gz: c6c2e2e85979c5d5492564fec243c23c44ef1d762230cd1a59400fcc46bf42a3
+  data.tar.gz: d551534240f528923ccb862e17ae521aaccdc5596e45c09e1cee0e90f0c990c5
 SHA512:
-  metadata.gz: 7c640710ee7cd4f439c831c763d531d5c855992c0381fbf6be22a86689d90e3036837c6c93e4525f7efca87a36f18987f7473473a48729416d73412ad4e28c7d
-  data.tar.gz: e971c17e48e4693310c48392346c9d259a79a8f8d2284f3e4bc05a5c7bad584d187192713a88923ef88023732129284c28781f4d8e438062708d038d621f04d9
+  metadata.gz: 3348c6243500f90f7157435bf36f1675744f46c0025dc4bab81fad4ccfb65c4e2ac0d029a837fb589c768b4868fbaf8ab7c258ad67aa4e1a9f33f0949701b2da
+  data.tar.gz: ef8d804eae97e2300747abb1b9e97f84e100b6b151e01b587fff95c98327ce21e0024456c6d8208ecd81ceaead8a0353d251f15a4ef4deff7e3b077e13c872cb

data/.ruby-version

@@ -1 +1 @@
-2.6.5
+2.6.6

data/Gemfile.lock

@@ -1,30 +1,32 @@
 PATH
   remote: .
   specs:
-    spidy (0.1.6)
+    spidy (0.2.6)
       activesupport
       mechanize
       pry
+      socksify
+      tor
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.2.1)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.2)
+      zeitwerk (~> 2.2, >= 2.2.2)
     coderay (1.1.2)
-    concurrent-ruby (1.1.5)
-    connection_pool (2.2.2)
+    concurrent-ruby (1.1.7)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     ffaker (2.10.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    i18n (1.8.2)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
     mechanize (2.7.6)
       domain_name (~> 0.5, >= 0.5.1)
@@ -38,14 +40,14 @@ GEM
     method_source (0.9.2)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
-    minitest (5.14.0)
+    minitest (5.14.2)
     mixlib-shellout (2.4.4)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (3.1.0)
+    net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
-    nokogiri (1.10.7)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     ntlm-http (0.1.1)
     pry (0.12.2)
@@ -72,14 +74,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.2)
+    socksify (1.7.1)
     thread_safe (0.3.6)
-    tzinfo (1.2.6)
+    tor (0.1.4)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
+    unf_ext (0.0.7.7)
     webrobots (0.1.2)
-    zeitwerk (2.2.2)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
@@ -94,4 +98,4 @@ DEPENDENCIES
   spidy!
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -60,7 +60,7 @@ call('http://example.com') { |html| break html } # html as nokogiri object ( mec
 ```
 
 ### When used from the ruby code
-``
+```rb
 a = Spidy.define do
   # Implementing spiders and scrapers
 end

data/lib/spidy.rb

@@ -4,13 +4,14 @@ require 'spidy/version'
 require 'active_support/all'
 require 'mechanize'
 require 'open-uri'
+require 'socksify'
+require 'tor'
 
 #
 # web spider dsl engine
 #
 module Spidy
   extend ActiveSupport::Autoload
-  autoload :Interface
   autoload :Shell
   autoload :CommandLine
   autoload :Console
@@ -34,6 +35,13 @@ module Spidy
         module_eval(&block)
       end
     end
-    Spidy::Interface.new(spidy)
+    spidy.instance_eval do
+      undef :spider
+      undef :define
+      undef :wait_time
+      undef :user_agent
+      undef :socks_proxy
+    end
+    spidy
   end
 end

data/lib/spidy/binder.rb

@@ -9,17 +9,44 @@ module Spidy::Binder
   autoload :Html
   autoload :Xml
 
+  class Error < StandardError
+  end
+
   class Caller
     def initialize(spidy, binder)
       @spidy = spidy
       @binder = binder
     end
 
-    def call(source, url: nil, define: nil)
-      yield Class.new(@binder, &define).new(@spidy, source, url)
+    def call(source, url: nil, define: nil, define_name: nil)
+      yield Class.new(@binder, &define).new(define_name, @spidy, source, url)
+    end
+  end
+
+  class Base
+    class << self
+      attr_reader :attribute_names
+    end
+
+    attr_reader :resource, :url
+
+    def initialize(define_name, spidy, resource, url)
+      @define_name = define_name
+      @spidy = spidy
+      @resource = resource
+      @url = url
+    end
+
+    def to_s
+      to_h.to_json
+    end
+
+    def to_h
+      self.class.attribute_names.map { |name| [name, send(name)] }.to_h
     end
   end
 
+
   def self.get(spidy, value)
     return Caller.new(spidy, const_get(value.to_s.classify)) if name.is_a?(String) || name.is_a?(Symbol)
 

data/lib/spidy/binder/html.rb

@@ -3,40 +3,23 @@
 #
 # Bind html and convert to object
 #
-class Spidy::Binder::Html
-  class << self
-    attr_reader :attribute_names
+class Spidy::Binder::Html < Spidy::Binder::Base
+  def self.let(name, query = nil, &block)
+    @attribute_names ||= []
+    @attribute_names << name
 
-    def let(name, query = nil, &block)
-      @attribute_names ||= []
-      @attribute_names << name
-      define_method(name) do
-        return html.at(query)&.text if block.nil?
-        return instance_exec(&block) if query.blank?
+    return define_method(name) { html.at(query)&.text } if block.nil?
 
+    define_method(name) do
+      if query.present?
         instance_exec(html.at(query), &block)
+      else
+        instance_exec(&block)
       end
+    rescue StandardError => e
+      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  attr_reader :html, :url
-  alias_method :resource, :html
-
-  def initialize(spidy, html, url)
-    @spidy = spidy
-    @url = url
-    @html = html
-  end
-
-  def scraper(name, source)
-    lambda { |&block| @spidy.call(source, name: name, &block) }
-  end
-
-  def to_s
-    to_h.to_json
-  end
-
-  def to_h
-    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-  end
+  alias_method :html, :resource
 end

data/lib/spidy/binder/json.rb

@@ -3,40 +3,23 @@
 #
 # Bind json and convert to object
 #
-class Spidy::Binder::Json
-  class << self
-    attr_reader :attribute_names
-
-    def let(name, *query, &block)
-      @attribute_names ||= []
-      @attribute_names << name
-      define_method(name) do
-        result = json.dig(*query) if query.present?
-        return result if block.nil?
-
-        instance_exec(result, &block)
+class Spidy::Binder::Json < Spidy::Binder::Base
+  def self.let(name, *query, &block)
+    @attribute_names ||= []
+    @attribute_names << name
+
+    return define_method(name) { json.dig(*query) } if block.nil?
+
+    define_method(name) do
+      if query.present?
+        instance_exec(json.dig(*query), &block)
+      else
+        instance_exec(&block)
       end
+    rescue StandardError => e
+      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  attr_reader :json, :url
-  alias_method :resource, :json
-
-  def initialize(spidy, json, url)
-    @spidy = spidy
-    @json = json
-    @url = url
-  end
-
-  def scraper(name, source)
-    lambda { |&block| @spidy.call(source, name: name, &block) }
-  end
-
-  def to_s
-    to_h.to_json
-  end
-
-  def to_h
-    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-  end
+  alias_method :json, :resource
 end

data/lib/spidy/binder/xml.rb

@@ -3,40 +3,23 @@
 #
 # Bind xml and convert to object
 #
-class Spidy::Binder::Xml
-  class << self
-    attr_reader :attribute_names
+class Spidy::Binder::Xml < Spidy::Binder::Base
+  def self.let(name, query = nil, &block)
+    @attribute_names ||= []
+    @attribute_names << name
 
-    def let(name, query = nil, &block)
-      @attribute_names ||= []
-      @attribute_names << name
-      define_method(name) do
-        return xml.at(query)&.text if block.nil?
-        return instance_exec(&block) if query.blank?
+    return define_method(name) { xml.at(query)&.text } if block.nil?
 
+    define_method(name) do
+      if query.present?
         instance_exec(xml.at(query), &block)
+      else
+        instance_exec(&block)
       end
+    rescue StandardError => e
+      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  attr_reader :xml, :url
-  alias_method :resource, :xml
-
-  def initialize(spidy, xml, url)
-    @spidy = spidy
-    @xml = xml
-    @url = url
-  end
-
-  def scraper(name, source)
-    lambda { |&block| @spidy.call(source, name: name, &block) }
-  end
-
-  def to_s
-    to_h.to_json
-  end
-
-  def to_h
-    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-  end
+  alias_method :xml, :resource
 end

data/lib/spidy/connector.rb

@@ -10,6 +10,24 @@ module Spidy::Connector
   autoload :Json
   autoload :Xml
 
+  #
+  # retry class
+  #
+  class Retry < StandardError
+    attr_reader :page
+    attr_reader :response_code
+    attr_reader :wait_time
+
+    def initialize(wait_time: 2, page: nil, error: nil)
+      @page = page
+      @wait_time = wait_time
+      @response_code = error.try(:response_code) || page.try(:response_code)
+    end
+  end
+
+  #
+  # default user agent
+  #
   USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
@@ -19,9 +37,19 @@ module Spidy::Connector
     'Safari/537.36'
   ].join(' ')
 
-  def self.get(value)
-    return const_get(value.to_s.classify) if value.is_a?(String) || value.is_a?(Symbol)
+  #
+  # get connection handller
+  #
+  def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil)
+    return value if value.respond_to?(:call)
+
+    connector = const_get(value.to_s.classify).new(wait_time: wait_time || 5, user_agent: user_agent || USER_AGENT)
+    return connector if socks_proxy.nil?
 
-    value
+    lambda do |url, &block|
+      Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
+        connector.call(url, &block)
+      end
+    end
   end
 end

data/lib/spidy/connector/direct.rb

@@ -3,8 +3,11 @@
 #
 # Direct resource ( not network resource  )
 #
-module Spidy::Connector::Direct
-  def self.call(resource, &yielder)
+class Spidy::Connector::Direct
+  def call(resource, &yielder)
     yielder.call(resource)
   end
+
+  def initialize(wait_time: nil, user_agent: nil)
+  end
 end

data/lib/spidy/connector/html.rb

@@ -3,73 +3,60 @@
 #
 # Mechanize wrapper
 #
-module Spidy::Connector::Html
-  #
-  # retry class
-  #
-  class Retry < StandardError
-    attr_reader :page
-    attr_reader :response_code
-    attr_reader :wait_time
-
-    def initialize(wait_time: 2, page: nil, error: nil)
-      @page = page
-      @wait_time = wait_time
-      @response_code = error.try(:response_code) || page.try(:response_code)
-    end
+class Spidy::Connector::Html
+  def initialize(wait_time:, user_agent:, logger: nil)
+    @wait_time = wait_time
+    @logger = logger || proc { |values| STDERR.puts(values.to_json) }
+    @agent = Mechanize.new
+    @user_agent = user_agent
+    @agent.user_agent = user_agent
   end
 
-  @logger = proc { |values| STDERR.puts(values.to_json) }
-  @agent = Mechanize.new
-  @agent.user_agent = Spidy::Connector::USER_AGENT
-
-  class << self
-    attr_reader :agent
-    attr_accessor :logger
+  attr_reader :agent
+  attr_reader :logger
 
-    def call(url, encoding: nil, retry_count: 5, &yielder)
-      fail 'url is not specified' if url.blank?
-      if encoding
-        agent.default_encoding = encoding
-        agent.force_default_encoding = true
-      end
-      logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-      get(url, retry_count, yielder)
+  def call(url, encoding: nil, retry_count: 5, &yielder)
+    fail 'url is not specified' if url.blank?
+    if encoding
+      agent.default_encoding = encoding
+      agent.force_default_encoding = true
     end
+    logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+    get(url, retry_count, yielder)
+  end
 
-    private
+  private
 
-    def get(url, retry_count, yielder)
-      connect(url, retry_count, yielder)
-    rescue Retry => e
-      logger.call('retry.accessed': Time.current,
-                  'retry.uri': url,
-                  'retry.response_code': e.response_code,
-                  'retry.rest_count': retry_count)
+  def get(url, retry_count, yielder)
+    connect(url, retry_count, yielder)
+  rescue Spidy::Connector::Retry => e
+    logger.call('retry.accessed': Time.current,
+                'retry.uri': url,
+                'retry.response_code': e.response_code,
+                'retry.rest_count': retry_count)
 
-      @agent = Mechanize.new
-      @agent.user_agent = Spidy::Connector::USER_AGENT
+    @agent = Mechanize.new
+    @agent.user_agent = @user_agent
 
-      retry_count -= 1
-      if retry_count.positive?
-        sleep e.wait_time
-        retry
-      end
-      raise e
+    retry_count -= 1
+    if retry_count.positive?
+      sleep e.wait_time
+      retry
     end
+    raise e
+  end
 
-    def connect(url, retry_count, yielder)
-      result = nil
-      agent.get(url) do |page|
-        fail Retry, page: page, wait_time: 5 if page.title == 'Sorry, unable to access page...'
+  def connect(url, retry_count, yielder)
+    result = nil
+    agent.get(url) do |page|
+      fail Spidy::Connector::Retry, page: page, wait_time: @wait_time if page.title == 'Sorry, unable to access page...'
 
-        result = yielder.call(page)
-      end
-      result
-    rescue Mechanize::ResponseCodeError => e
-      raise Retry, error: e if e.response_code == '429'
-      raise e
+      result = yielder.call(page)
     end
-
+    result
+  rescue Mechanize::ResponseCodeError => e
+    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '429'
+    raise Spidy::Connector::Retry, error: e, wait_time: @wait_time if e.response_code == '502'
+    raise e
   end
 end

data/lib/spidy/connector/json.rb

@@ -3,9 +3,17 @@
 #
 # OpenURI to JSON.parse
 #
-module Spidy::Connector::Json
-  def self.call(url)
+class Spidy::Connector::Json
+  def initialize(wait_time: nil, user_agent: nil)
+    @user_agent = user_agent
+  end
+
+  def call(url, &block)
     fail 'url is not specified' if url.blank?
-    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+    connect(url, &block)
+  end
+
+  def connect(url)
+    OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
   end
 end

data/lib/spidy/connector/xml.rb

@@ -3,12 +3,16 @@
 #
 # xml
 #
-module Spidy::Connector::Xml
-  def self.call(url)
+class Spidy::Connector::Xml
+  def call(url)
     fail 'URL is undefined' if url.blank?
 
-    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) do |body|
+    OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
       yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
   end
+
+  def initialize(wait_time: nil, user_agent: nil)
+    @user_agent = user_agent
+  end
 end

data/lib/spidy/console.rb

@@ -18,4 +18,9 @@ class Spidy::Console
   def reload!
     @definition_file.eval_definition
   end
+
+  def connector(url, as:)
+    connector = Spidy::Connector.get(as)
+    connector.call(url) { |page| break page }
+  end
 end

data/lib/spidy/definition.rb

@@ -16,6 +16,22 @@ module Spidy::Definition
     spidy.call(source, &yielder)
   end
 
+  def user_agent(user_agent)
+    @user_agent = user_agent
+  end
+
+  def wait_time(wait_time)
+    @wait_time = wait_time
+  end
+
+  def socks_proxy(host, port)
+    @socks_proxy = { host: host, port: port }
+  end
+
+  def tor?
+    Tor.running?
+  end
+
   def each(source = nil, name: :default, &yielder)
     name = name.presence || :default
     spidy = @namespace[:"#{name}_spider"]
@@ -26,7 +42,7 @@ module Spidy::Definition
 
   def spider(name = :default, connector: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
     @namespace[:"#{name}_spider"] = proc do |source, &yielder|
       define_block.call(yielder, connector, source)
     end
@@ -34,18 +50,18 @@ module Spidy::Definition
 
   def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent)
     binder = Spidy::Binder.get(self, binder || as)
-    @namespace[:"#{name}_scraper"] = define_proc(connector, binder, define_block)
+    @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
   end
 
   private
 
-  def define_proc(connector, binder, define_block)
+  def define_proc(name, connector, binder, define_block)
     proc do |source, &yielder|
       yielder = lambda { |result| break result } if yielder.nil?
       connection_yielder = lambda do |page|
-        binder.call(page, url: source, define: define_block) { |object| yielder.call(object) }
+        binder.call(page, url: source, define: define_block, define_name: name) { |object| yielder.call(object) }
       end
       connector.call(source, &connection_yielder)
     end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.2.1'
+  VERSION = '0.2.6'
 end

data/spidy.gemspec

@@ -31,7 +31,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'
 
+  spec.add_runtime_dependency 'tor'
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'
+  spec.add_runtime_dependency 'socksify'
   spec.add_runtime_dependency 'pry'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.6
 platform: ruby
 authors:
 - aileron
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2020-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: socksify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +164,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - aileron.cc@gmail.com
 executables:
@@ -174,7 +202,6 @@ files:
 - lib/spidy/console.rb
 - lib/spidy/definition.rb
 - lib/spidy/definition_file.rb
-- lib/spidy/interface.rb
 - lib/spidy/shell.rb
 - lib/spidy/spider.rb
 - lib/spidy/version.rb
@@ -184,7 +211,7 @@ homepage: https://github.com/aileron-inc/spidy
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -199,8 +226,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: web spider dsl
 test_files: []

data/lib/spidy/interface.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-#
-# spidy interface
-#
-class Spidy::Interface
-  delegate :call, :each, :namespace, to: :@spidy
-
-  def initialize(spidy)
-    @spidy = spidy
-  end
-end