spidy 0.3.6 → 0.3.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38936d0a025edde8a0ecb15a164c36fbccf1346f9bee9ec7d01a43931eaeb163
-  data.tar.gz: 71c02d424da0b18cfad383035c78465b1c81ccdfe41399baa511a7bae3675662
+  metadata.gz: 76cb60ea985d1a663f24b7b024198d222756376bd9dd979a032c46ba39b16548
+  data.tar.gz: ff2e7f056f7ad5afe06df90adf0bb2e438c696472cde50c8d5758b2f9801684e
 SHA512:
-  metadata.gz: d97c61f76fb9dde1d5693c7d33830c6df7605930408c42666ffe31a6d46262d1203cfed710d2112f2c396cdcbc3e40f1149d9fc005c5947af3851946bd6f487e
-  data.tar.gz: ffe6d6362f6bb50c11e54cb398730b0a649ac832b5fdd048d61675dce7a17ec58993900cbaa250238c0cc702cbe649e901cab660c265adb2ef8207e2704ea48c
+  metadata.gz: a721848978135752ddcfe3da30a293317a4852b41dc99209019ae71960538fe448ec4ad54da661e0b99edef3fcb85a84b095b99ddbbba9b628fdd4ac1be2f23c
+  data.tar.gz: a156f47f317cd4f1f0a66a13ac5102073723f139c5b797c8dc56d7dbdd41e342cb1ad1a6814812563033c68f448c4d57775c0edf300456dd164b90211632737e

data/.rubocop.yml

@@ -1,7 +1,8 @@
 inherit_from: .rubocop_todo.yml
 AllCops:
+  TargetRubyVersion: 3.0.2
+  NewCops: enable
   DisplayCopNames: true
-  TargetRubyVersion: 2.6
 
 Style/ClassAndModuleChildren:
   Enabled: false
@@ -9,7 +10,7 @@ Style/ClassAndModuleChildren:
 Style/SignalException:
   EnforcedStyle: semantic
 
-Naming/UncommunicativeMethodParamName:
+Naming/MethodParameterName:
   AllowedNames:
     - as
 
@@ -17,8 +18,11 @@ Metrics/AbcSize:
   Max: 21
   Exclude:
 
+Metrics/MethodLength:
+  Max: 15
+
 Metrics/LineLength:
-  Max: 120
+  Max: 130
 
 Metrics/BlockLength:
   Max: 120

data/.ruby-version

@@ -1 +1 @@
-2.6.6
+3.0.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.3.4)
+    spidy (0.3.10)
       activesupport
       mechanize
       pry
@@ -11,124 +11,126 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.3)
+    activesupport (7.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
-    addressable (2.7.0)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    capybara (3.33.0)
+    capybara (3.36.0)
       addressable
+      matrix
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
-      regexp_parser (~> 1.5)
+      regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
     capybara_discoball (0.1.0)
       capybara (>= 2.7, < 4)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.7)
-    connection_pool (2.2.3)
-    diff-lcs (1.3)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    diff-lcs (1.5.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffaker (2.10.0)
-    http-cookie (1.0.3)
+    ffaker (2.20.0)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
-    i18n (1.8.5)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    mechanize (2.7.6)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (>= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (>= 0.0.9, < 0.2)
-    method_source (0.9.2)
-    mime-types (3.3.1)
+    matrix (0.4.2)
+    mechanize (2.8.3)
+      addressable (~> 2.8)
+      domain_name (~> 0.5, >= 0.5.20190701)
+      http-cookie (~> 1.0, >= 1.0.3)
+      mime-types (~> 3.0)
+      net-http-digest_auth (~> 1.4, >= 1.4.1)
+      net-http-persistent (>= 2.5.2, < 5.0.dev)
+      nokogiri (~> 1.11, >= 1.11.2)
+      rubyntlm (~> 0.6, >= 0.6.3)
+      webrick (~> 1.7)
+      webrobots (~> 0.1.2)
+    method_source (1.0.0)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
-    mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
-    minitest (5.14.2)
+    mime-types-data (3.2021.1115)
+    mini_mime (1.1.2)
+    minitest (5.15.0)
     mixlib-shellout (2.4.4)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.0)
+    net-http-persistent (4.0.1)
       connection_pool (~> 2.2)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    ntlm-http (0.1.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    nokogiri (1.12.5-arm64-darwin)
+      racc (~> 1.4)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     public_suffix (4.0.6)
+    racc (1.6.0)
     rack (2.2.3)
-    rack-protection (2.0.8.1)
+    rack-protection (2.1.0)
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rake (10.5.0)
-    regexp_parser (1.8.1)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
+    rake (13.0.6)
+    regexp_parser (2.2.0)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
     rspec-command (1.0.3)
       mixlib-shellout (~> 2.0)
       rspec (~> 3.2)
       rspec-its (~> 1.2)
-    rspec-core (3.8.2)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.4)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
+      rspec-support (~> 3.10.0)
     rspec-its (1.3.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
-    rspec-mocks (3.8.1)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.2)
-    ruby2_keywords (0.0.2)
-    sinatra (2.0.8.1)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    ruby2_keywords (0.0.5)
+    rubyntlm (0.6.3)
+    sinatra (2.1.0)
       mustermann (~> 1.0)
-      rack (~> 2.0)
-      rack-protection (= 2.0.8.1)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
       tilt (~> 2.0)
     socksify (1.7.1)
-    thread_safe (0.3.6)
     tilt (2.0.10)
-    tor (0.1.4)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
+    tor (0.1.5)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
+    unf_ext (0.0.8)
+    webrick (1.7.0)
     webrobots (0.1.2)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.4.0)
 
 PLATFORMS
-  ruby
+  arm64-darwin-20
 
 DEPENDENCIES
   bundler (~> 2.0)
   capybara_discoball
   ffaker
   pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   rspec-command
   sinatra
   spidy!
 
 BUNDLED WITH
-   2.1.4
+   2.2.22

data/example/master_detail.rb

@@ -1,7 +1,7 @@
-
+# frozen_string_literal: true
 
 Spidy.define do
-  url_to_params = ->(url) {
+  url_to_params = lambda { |url|
     uri = URI.parse(url)
     params = URI.decode_www_form(uri.query).to_h if uri.query.present?
     params if params.present?
@@ -13,41 +13,41 @@ Spidy.define do
 
     limit_page = 3
     per_page = 25
-    yielder.call(Nokogiri::HTML::Builder.new { |doc|
-      doc.html {
-        doc.body {
-          doc.span.bold {
-            doc.text "Hello world"
-          }
-          doc.main {
-            (page * per_page + 1).upto((page + 1) * per_page).each do |i|
+    yielder.call(Nokogiri::HTML::Builder.new do |doc|
+      doc.html do
+        doc.body do
+          doc.span.bold do
+            doc.text 'Hello world'
+          end
+          doc.main do
+            ((page * per_page) + 1).upto((page + 1) * per_page).each do |i|
               doc.a("page #{i}", href: "http://localhost/?id=#{i}")
             end
-          }
+          end
           doc.a('NEXT', href: "http://localhost/?page=#{page + 1}", class: 'next') if page < limit_page
-        }
-      }
-    }.doc)
+        end
+      end
+    end.doc)
   }
 
   detail_page = proc { |url, &yielder|
     params = url_to_params.call(url)
     id = params['id']
 
-    yielder.call(Nokogiri::HTML::Builder.new { |doc|
-      doc.html {
-        doc.body {
-          doc.span.bold {
-            doc.text "Hello world"
-          }
+    yielder.call(Nokogiri::HTML::Builder.new do |doc|
+      doc.html do
+        doc.body do
+          doc.span.bold do
+            doc.text 'Hello world'
+          end
           doc.h1("title_#{id}", id: 'title')
           doc.main("body_#{id}", id: 'body')
           doc.div.sub do
             doc.span.name('testtest')
           end
-        }
-      }
-    }.doc)
+        end
+      end
+    end.doc)
   }
 
   define(as: :html, connector: detail_page) do

data/example/proxy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Spidy.define do
   user_agent 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0'
   socks_proxy '127.0.0.1', 9050

data/example/retry.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Spidy.define do
   spider(as: :json) do |yielder, connector|
     connector.call('https://httpbin.org/status/500') do |json|

data/example/wikip.rb

@@ -11,11 +11,8 @@ Spidy.define do
 
   define(:infobox, as: :html, connector: :direct) do
     let(:columns) do
-      html.search('tr').each do |tr|
-        {
-          name: tr.at('th')&.text,
-          value: tr.at('td')&.text
-        }
+      html.search('tr').map do |tr|
+        { name: tr.at('th')&.text, value: tr.at('td')&.text }
       end
     end
   end

data/exe/spidy

@@ -6,10 +6,10 @@ require 'pry'
 
 if ARGV[1].blank?
   case ARGV[0]
-  when 'version' then STDOUT.puts(Spidy::VERSION)
+  when 'version' then $stdout.puts(Spidy::VERSION)
   when 'console' then Spidy.shell.interactive
   else
-    STDOUT.puts 'usage: spidy [version console]'
+    $stdout.puts 'usage: spidy [version console]'
   end
 else
   case ARGV[0]
@@ -17,7 +17,8 @@ else
   when 'function' then Spidy.shell(ARGV[1]).function
   when 'call' then Spidy.shell(ARGV[1]).call(ARGV[2])
   when 'each' then Spidy.shell(ARGV[1]).each(ARGV[2])
+  when 'eval' then Spidy.shell(ARGV[1]).eval_call(ARGV[2])
   else
-    STDOUT.puts 'usage: spidy [console function call each] [file]'
+    $stdout.puts 'usage: spidy [console function call each run] [file]'
   end
 end

data/lib/spidy/binder/error.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class Spidy::Binder::Error < StandardError
+end

data/lib/spidy/binder/html.rb

@@ -3,12 +3,12 @@
 #
 # Bind html and convert to object
 #
-class Spidy::Binder::Html < Spidy::Binder::Base
-  def self.let(name, query = nil, &block)
+module Spidy::Binder::Html
+  def let(name, query = nil, &block)
     @attribute_names ||= []
     @attribute_names << name
 
-    return define_method(name) { html.at(query)&.text } if block.nil?
+    return define_method(name) { html.at(query)&.text&.strip } if block.nil?
 
     define_method(name) do
       if query.present?
@@ -17,9 +17,11 @@ class Spidy::Binder::Html < Spidy::Binder::Base
         instance_exec(&block)
       end
     rescue StandardError => e
-      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
+      raise Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  alias_method :html, :resource
+  def self.extended(obj)
+    obj.alias_method :html, :resource
+  end
 end

data/lib/spidy/binder/json.rb

@@ -3,8 +3,8 @@
 #
 # Bind json and convert to object
 #
-class Spidy::Binder::Json < Spidy::Binder::Base
-  def self.let(name, *query, &block)
+module Spidy::Binder::Json
+  def let(name, *query, &block)
     @attribute_names ||= []
     @attribute_names << name
 
@@ -17,9 +17,11 @@ class Spidy::Binder::Json < Spidy::Binder::Base
         instance_exec(&block)
       end
     rescue StandardError => e
-      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
+      raise Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  alias_method :json, :resource
+  def self.extended(obj)
+    obj.alias_method :json, :resource
+  end
 end

data/lib/spidy/binder/xml.rb

@@ -3,12 +3,12 @@
 #
 # Bind xml and convert to object
 #
-class Spidy::Binder::Xml < Spidy::Binder::Base
-  def self.let(name, query = nil, &block)
+module Spidy::Binder::Xml
+  def let(name, query = nil, &block)
     @attribute_names ||= []
     @attribute_names << name
 
-    return define_method(name) { xml.at(query)&.text } if block.nil?
+    return define_method(name) { xml.at(query)&.text&.strip } if block.nil?
 
     define_method(name) do
       if query.present?
@@ -17,9 +17,11 @@ class Spidy::Binder::Xml < Spidy::Binder::Base
         instance_exec(&block)
       end
     rescue StandardError => e
-      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
+      raise Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  alias_method :xml, :resource
+  def self.extended(obj)
+    obj.alias_method :xml, :resource
+  end
 end

data/lib/spidy/binder.rb

@@ -5,51 +5,8 @@
 #
 module Spidy::Binder
   extend ActiveSupport::Autoload
+  autoload :Error
   autoload :Json
   autoload :Html
   autoload :Xml
-
-  class Error < StandardError
-  end
-
-  class Caller
-    def initialize(spidy, binder)
-      @spidy = spidy
-      @binder = binder
-    end
-
-    def call(source, url: nil, define: nil, define_name: nil)
-      yield Class.new(@binder, &define).new(define_name, @spidy, source, url)
-    end
-  end
-
-  class Base
-    class << self
-      attr_reader :attribute_names
-    end
-
-    attr_reader :resource, :url
-
-    def initialize(define_name, spidy, resource, url)
-      @define_name = define_name
-      @spidy = spidy
-      @resource = resource
-      @url = url
-    end
-
-    def to_s
-      to_h.to_json
-    end
-
-    def to_h
-      self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-    end
-  end
-
-
-  def self.get(spidy, value)
-    return Caller.new(spidy, const_get(value.to_s.classify)) if name.is_a?(String) || name.is_a?(Symbol)
-
-    value
-  end
 end

data/lib/spidy/command_line.rb

@@ -5,45 +5,47 @@
 #
 class Spidy::CommandLine
   delegate :spidy, to: :@definition_file
-  class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
-  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
+  class_attribute :output, default: (proc { |result| $stdout.puts(result.to_s) })
+  class_attribute :error_handler, default: (proc { |e, url|
+                                              warn({ url: url, message: e.message, backtrace: e.backtrace }.to_json)
+                                            })
+
+  def eval_call(script)
+    @definition_file.spidy.instance_eval(script)
+  end
 
   def initialize(definition_file)
     @definition_file = definition_file
-    raise 'unloaded spidy' if definition_file.spidy.nil?
+    fail 'unloaded spidy' if definition_file.spidy.nil?
   end
 
   def each_stdin_lines(name)
-    STDIN.each_line do |url|
-      begin
-        spidy.each(url.strip, name: name, &output)
-      rescue => e
-        error_handler.call(e, url)
-      end
+    $stdin.each_line do |url|
+      spidy.each(url.strip, name: name, &output)
+    rescue StandardError => e
+      error_handler.call(e, url)
     end
   end
 
   def call_stdin_lines(name)
-    STDIN.each_line do |url|
-      begin
-        spidy.call(url.strip, name: name, &output)
-      rescue => e
-        error_handler.call(e, url)
-      end
+    $stdin.each_line do |url|
+      spidy.call(url.strip, name: name, &output)
+    rescue StandardError => e
+      error_handler.call(e, url)
     end
   end
 
   def call(name)
-    return call_stdin_lines(name) if FileTest.pipe?(STDIN)
-    spidy.call(name: name, &output) unless FileTest.pipe?(STDIN)
-  rescue => e
+    return call_stdin_lines(name) if FileTest.pipe?($stdin)
+    spidy.call(name: name, &output) unless FileTest.pipe?($stdin)
+  rescue StandardError => e
     error_handler.call(e, nil)
   end
 
   def each(name)
-    return each_stdin_lines(name) if FileTest.pipe?(STDIN)
+    return each_stdin_lines(name) if FileTest.pipe?($stdin)
     spidy.each(name: name, &output)
-  rescue => e
+  rescue StandardError => e
     error_handler.call(e, nil)
   end
 
@@ -59,36 +61,32 @@ class Spidy::CommandLine
   end
 
   def build(name)
-    build_shell(name)
-    build_ruby(name)
+    File.write("#{name}.sh", build_shell_script(name))
+    File.write("#{name}.rb", build_ruby_script)
   end
 
   def build_shell(name)
-    File.open("#{name}.sh", 'w') do |f|
-      f.write <<~SHELL
-        #!/bin/bash
-        eval "$(spidy $(dirname "${0}")/#{name}.rb shell)"
-        spider example
-      SHELL
-    end
+    <<~SHELL
+      #!/bin/bash
+      eval "$(spidy $(dirname "${0}")/#{name}.rb shell)"
+      spider
+    SHELL
   end
 
-  def build_ruby(name)
-    File.open("#{name}.rb", 'w') do |f|
-      f.write <<~RUBY
-        # frozen_string_literal: true
+  def build_ruby
+    <<~RUBY
+      # frozen_string_literal: true
 
-        Spidy.define do
-          spider(:example) do |yielder, connector|
-            # connector.call(url) do |resource|
-            #   yielder.call(url or resource)
-            # end
-          end
+      Spidy.define do
+        spider(as: :html) do |yielder, connector|
+          # connector.call(url) do |resource|
+          #   yielder.call(url or resource)
+          # end
+        end
 
-          define(:example) do
-          end
+        define(as: :html) do
         end
-      RUBY
-    end
+      end
+    RUBY
   end
 end

data/lib/spidy/connector/direct.rb

@@ -4,7 +4,7 @@
 # Direct resource ( not network resource  )
 #
 class Spidy::Connector::Direct
-  def call(resource, &yielder)
+  def call(resource)
     if block_given?
       yield resource
     else
@@ -12,6 +12,5 @@ class Spidy::Connector::Direct
     end
   end
 
-  def initialize(user_agent:)
-  end
+  def initialize(user_agent:); end
 end

data/lib/spidy/connector/html.rb

@@ -14,13 +14,13 @@ class Spidy::Connector::Html
 
   attr_reader :agent
 
-  def call(url, encoding: nil, retry_count: 5, &yielder)
+  def call(url, encoding: nil, &yielder)
     fail 'url is not specified' if url.blank?
     if encoding
       agent.default_encoding = encoding
       agent.force_default_encoding = true
     end
-    connect(url, retry_count, yielder)
+    connect(url, yielder)
   end
 
   def refresh!
@@ -30,17 +30,19 @@ class Spidy::Connector::Html
 
   private
 
-  def connect(url, retry_count, yielder)
+  def connect(url, yielder)
     result = nil
     agent.get(url) do |page|
-      fail Spidy::Connector::Retry, object: page, response_code: page.try(:response_code) if page.title == 'Sorry, unable to access page...'
+      if page.title == 'Sorry, unable to access page...'
+        fail Spidy::Connector::Retry.new(object: page, response_code: page.try(:response_code))
+      end
 
       result = yielder.call(page)
     end
     result
   rescue Mechanize::ResponseCodeError => e
-    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '429'
-    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '502'
-    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code)
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.try(:response_code)) if e.response_code == '429'
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.try(:response_code)) if e.response_code == '502'
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.try(:response_code))
   end
 end

data/lib/spidy/connector/json.rb

@@ -17,9 +17,9 @@ class Spidy::Connector::Json
     connect(url, &block)
   end
 
-  def connect(url, retry_count: 5)
-    OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+  def connect(url)
+    OpenURI.open_uri(url, 'User-Agent' => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
   rescue OpenURI::HTTPError => e
-    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.io.status[0])
   end
 end

data/lib/spidy/connector/xml.rb

@@ -13,11 +13,11 @@ class Spidy::Connector::Xml
   end
 
   def connect(url, &block)
-    OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
+    OpenURI.open_uri(url, 'User-Agent' => @user_agent) do |body|
       block.call Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
   rescue OpenURI::HTTPError => e
-    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.io.status[0])
   end
 
   def initialize(user_agent:)

data/lib/spidy/connector.rb

@@ -27,7 +27,7 @@ module Spidy::Connector
   #
   # error output logger
   #
-  DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
+  DEFAULT_LOGGER = proc { |values| warn(values.to_json) }
 
   #
   # static method
@@ -36,7 +36,9 @@ module Spidy::Connector
     extend ActiveSupport::Concern
     class_methods do
       def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
-        ::Spidy::Connector::RetryableCaller.new(new(user_agent: user_agent), wait_time: wait_time, logger: logger).call(url, &block)
+        ::Spidy::Connector::RetryableCaller.new(new(user_agent: user_agent), wait_time: wait_time, logger: logger).call(
+          url, &block
+        )
       end
     end
   end
@@ -51,6 +53,7 @@ module Spidy::Connector
       @object = object
       @response_code = response_code
       @error = error
+      super(error)
     end
   end
 
@@ -58,13 +61,13 @@ module Spidy::Connector
   # retry
   #
   class RetryableCaller
-    attr_reader :origin_connector
+    attr_reader :origin_connector, :logger, :wait_time
 
-    def initialize(connector, logger:, wait_time:)
+    def initialize(connector, logger:, wait_time:, retry_attempt_count: 5)
       @origin_connector = connector
       @logger = logger
       @wait_time = wait_time
-      @retry_attempt_count = 5
+      @retry_attempt_count = retry_attempt_count
     end
 
     def call(url, &block)
@@ -73,18 +76,18 @@ module Spidy::Connector
     end
 
     def connect(url, retry_attempt_count: @retry_attempt_count, &block)
-      @logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-      @origin_connector.call(url, &block)
+      logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+      origin_connector.call(url, &block)
     rescue Spidy::Connector::Retry => e
-      @logger.call('retry.accessed': Time.current,
-                   'retry.uri': url,
-                   'retry.response_code': e.response_code,
-                   'retry.attempt_count': retry_attempt_count)
+      logger.call('retry.accessed': Time.current,
+                  'retry.uri': url,
+                  'retry.response_code': e.response_code,
+                  'retry.attempt_count': retry_attempt_count)
 
       retry_attempt_count -= 1
       if retry_attempt_count.positive?
-        sleep @wait_time
-        @origin_connector.refresh! if @origin_connector.respond_to?(:refresh!)
+        sleep wait_time
+        origin_connector.refresh! if origin_connector.respond_to?(:refresh!)
         retry
       end
       raise e.error
@@ -103,7 +106,7 @@ module Spidy::Connector
     end
 
     def call(url, &block)
-      Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
+      Socksify.proxy(socks_proxy[:host], socks_proxy[:port]) do
         connector.call(url, &block)
       end
     end
@@ -141,7 +144,6 @@ module Spidy::Connector
     fail "Not defined connnector[#{value}]" if connector.nil?
     return connector if socks_proxy.nil?
 
-    tor = TorConnector.new(connector, socks_proxy)
-    tor
+    TorConnector.new(connector, socks_proxy)
   end
 end

data/lib/spidy/definition.rb

@@ -33,33 +33,39 @@ module Spidy::Definition
     spidy = @namespace[:"#{name}_spider"]
     fail "undefined spidy [#{name}]" if spidy.nil?
 
-    spidy.call(source, &yielder)
+    if yielder
+      spidy.call(source, &yielder)
+    else
+      Enumerator.new do |enumerate_yielder|
+        spidy.call(source, &enumerate_yielder)
+      end
+    end
   end
 
   def spider(name = :default, connector: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent,
+                                                      socks_proxy: @socks_proxy)
     @namespace[:"#{name}_spider"] = proc do |source, &yielder|
       define_block.call(yielder, connector, source)
     end
   end
 
-  def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
+  def define(name = :default, connector: nil, as: nil, &define_block)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent,
+                                                      socks_proxy: @socks_proxy)
+    binder_base = Spidy::Binder.const_get(as.to_s.classify)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
-    binder = Spidy::Binder.get(self, binder || as)
-    @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
-  end
-
-  private
-
-  def define_proc(name, connector, binder, define_block)
-    proc do |source, &yielder|
-      yielder = lambda { |result| break result } if yielder.nil?
-      connection_yielder = lambda do |page|
-        binder.call(page, url: source, define: define_block, define_name: name) { |object| yielder.call(object) }
+    @namespace[:"#{name}_scraper"] = Class.new(Spidy::DefinitionObject) do
+      extend binder_base
+      class_eval(&define_block)
+      define_singleton_method(:call) do |source, &yielder|
+        yielder = ->(result) { break result } if yielder.nil?
+        connection_yielder = lambda do |page|
+          yielder.call(new(page, source))
+        end
+        connector.call(source, &connection_yielder)
       end
-      connector.call(source, &connection_yielder)
     end
   end
 end

data/lib/spidy/definition_file.rb

@@ -4,8 +4,7 @@
 # spidy interface binding
 #
 class Spidy::DefinitionFile
-  attr_reader :path
-  attr_reader :spidy
+  attr_reader :path, :spidy
 
   def self.open(filepath)
     object = new(filepath)
@@ -15,7 +14,7 @@ class Spidy::DefinitionFile
 
   # rubocop:disable Security/Eval
   def eval_definition
-    @spidy = eval(File.open(path).read) if path
+    @spidy = eval(File.read(path)) if path
   end
   # rubocop:enable Security/Eval
 

data/lib/spidy/definition_object.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+#
+# An object that represents the scraper defined by define block.
+#
+class Spidy::DefinitionObject
+  class << self
+    attr_reader :attribute_names
+  end
+  attr_reader :resource, :url
+
+  def initialize(resource, url)
+    @resource = resource
+    @url = url
+  end
+
+  def to_s
+    to_h.to_json
+  end
+
+  def to_h
+    self.class.attribute_names.to_h { |name| [name, send(name)] }
+  end
+end

data/lib/spidy/shell.rb

@@ -16,5 +16,5 @@ class Spidy::Shell
     Spidy::CommandLine.new(@definition_file)
   end
 
-  delegate :function, :each, :call, to: :command_line
+  delegate :function, :each, :call, :eval_call, to: :command_line
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.3.6'
+  VERSION = '0.3.12'
 end

data/lib/spidy.rb

@@ -17,6 +17,7 @@ module Spidy
   autoload :Console
   autoload :Definition
   autoload :DefinitionFile
+  autoload :DefinitionObject
   autoload :Binder
   autoload :Connector
 

data/spidy.gemspec

@@ -25,17 +25,20 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'capybara_discoball'
+  spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'
-  spec.add_development_dependency 'capybara_discoball'
   spec.add_development_dependency 'sinatra'
 
-  spec.add_runtime_dependency 'tor'
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'
-  spec.add_runtime_dependency 'socksify'
   spec.add_runtime_dependency 'pry'
+  spec.add_runtime_dependency 'socksify'
+  spec.add_runtime_dependency 'tor'
+  spec.metadata = {
+    'rubygems_mfa_required' => 'true'
+  }
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.12
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-09 00:00:00.000000000 Z
+date: 2022-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: capybara_discoball
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,63 +39,63 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: ffaker
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: ffaker
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: rspec-command
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: capybara_discoball
+  name: rspec-command
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: tor
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -151,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -179,7 +179,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: tor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -222,6 +222,7 @@ files:
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb
+- lib/spidy/binder/error.rb
 - lib/spidy/binder/html.rb
 - lib/spidy/binder/json.rb
 - lib/spidy/binder/xml.rb
@@ -234,6 +235,7 @@ files:
 - lib/spidy/console.rb
 - lib/spidy/definition.rb
 - lib/spidy/definition_file.rb
+- lib/spidy/definition_object.rb
 - lib/spidy/shell.rb
 - lib/spidy/spider.rb
 - lib/spidy/version.rb
@@ -242,7 +244,8 @@ files:
 homepage: https://github.com/aileron-inc/spidy
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -258,7 +261,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: web spider dsl