spidy 0.3.9 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75215453c834a8e481b27cf4377235cc97ce6a6e4eff142a11743e68ee4982b4
-  data.tar.gz: 25f3b14ad6f31b580396458c8075167f24fd4a2f6e7ff98947338ecd0588eb9d
+  metadata.gz: b6a7d70df09642e17d34cc85e1973914b8b7e151c34670526cb4d6b2d3589227
+  data.tar.gz: 012b7def5510c16d68676bada533d452315244fc5d47d7f26a9bd71068a3f9a3
 SHA512:
-  metadata.gz: 447b7152b807c7985e16b7b403d27f9f7b949264577e8e4dc11a52358cb9af49510696d29166adb60bb5b87158aa8d2c10faf7c810f8eced4c29f9eed8bb493a
-  data.tar.gz: 55d82e5c495a7e5a0fd57b466e08ea072e3a712829b2d83d550b107147f30e3eaff54ed8dcec69163bf6f74e0c32990051af9de0d044cc19eadf39ecc749d003
+  metadata.gz: 8b6682cd3d1499b115cdfba3964cab62ae65f3a7943fb87f53d8613c2a6553b4f2a4a0728f2af07990dc90ae03c07df4d33914739ab4df462a37c2b04f5efdc5
+  data.tar.gz: a2680dd41fb1a6dead95ecd20742560d749f5c3f27367baac7d3f1294d4c6ee7de946cb173e1f2750169e6a341915197e375d1e56ca0faf7c5be4491f5a55ae9

data/.rubocop.yml

@@ -1,7 +1,8 @@
 inherit_from: .rubocop_todo.yml
 AllCops:
+  TargetRubyVersion: 3.0.2
+  NewCops: enable
   DisplayCopNames: true
-  TargetRubyVersion: 2.6
 
 Style/ClassAndModuleChildren:
   Enabled: false
@@ -9,7 +10,7 @@ Style/ClassAndModuleChildren:
 Style/SignalException:
   EnforcedStyle: semantic
 
-Naming/UncommunicativeMethodParamName:
+Naming/MethodParameterName:
   AllowedNames:
     - as
 
@@ -17,8 +18,11 @@ Metrics/AbcSize:
   Max: 21
   Exclude:
 
+Metrics/MethodLength:
+  Max: 15
+
 Metrics/LineLength:
-  Max: 120
+  Max: 130
 
 Metrics/BlockLength:
   Max: 120

data/.ruby-version

@@ -1 +1 @@
-3.0.2
+3.4.2

data/CLAUDE.md

@@ -0,0 +1,28 @@
+# Claude Helper for Spidy
+
+## Build/Test/Lint Commands
+- Install dependencies: `bundle install`
+- Run all tests: `bundle exec rake spec`
+- Run single test: `bundle exec rspec spec/path/to_spec.rb:LINE_NUMBER`
+- Install gem locally: `bundle exec rake install`
+- Release gem: `bundle exec rake release`
+
+## Code Style Guidelines
+- **Naming Conventions**: 
+  - snake_case for methods/variables/files
+  - CamelCase for classes/modules
+  - SCREAMING_SNAKE_CASE for constants
+- **File Organization**: Match file paths to module/class hierarchy
+- **Imports**: 
+  - Add `# frozen_string_literal: true` at file start
+  - Use `extend ActiveSupport::Autoload` for modules with sub-modules
+- **Error Handling**: Create custom error classes inheriting from StandardError
+- **Documentation**: Add brief comments before classes and methods
+- **Testing**: 
+  - Use RSpec with `expect` syntax 
+  - Organize with `describe` and `specify` blocks
+  - Name test files with `_spec.rb` suffix
+
+## Dependencies
+- Runtime: activesupport, mechanize, socksify, tor
+- Development: bundler, capybara_discoball, ffaker, rake, rspec, sinatra

data/Gemfile

@@ -4,3 +4,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in crawler.gemspec
 gemspec
+
+gem 'webrick'
+gem 'rackup'
+gem 'irb'

data/Gemfile.lock

@@ -1,138 +1,193 @@
 PATH
   remote: .
   specs:
-    spidy (0.3.9)
-      activesupport
+    spidy (0.3.12)
+      activesupport (~> 7.1)
       mechanize
-      pry
       socksify
       tor
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.0)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+    activesupport (7.2.2.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      tzinfo (~> 2.0)
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
-    capybara (3.36.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
+    capybara (3.40.0)
       addressable
       matrix
       mini_mime (>= 0.1.3)
-      nokogiri (~> 1.8)
+      nokogiri (~> 1.11)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
     capybara_discoball (0.1.0)
       capybara (>= 2.7, < 4)
-    coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    connection_pool (2.2.5)
-    diff-lcs (1.4.4)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
-    ffaker (2.20.0)
-    http-cookie (1.0.4)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
+    date (3.4.1)
+    diff-lcs (1.6.0)
+    domain_name (0.6.20240107)
+    drb (2.2.1)
+    ffaker (2.24.0)
+    http-cookie (1.0.8)
       domain_name (~> 0.5)
-    i18n (1.8.11)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
+    io-console (0.8.0)
+    irb (1.15.1)
+      pp (>= 0.6.0)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
+    logger (1.6.6)
     matrix (0.4.2)
-    mechanize (2.8.3)
+    mechanize (2.14.0)
       addressable (~> 2.8)
+      base64
       domain_name (~> 0.5, >= 0.5.20190701)
       http-cookie (~> 1.0, >= 1.0.3)
-      mime-types (~> 3.0)
+      mime-types (~> 3.3)
       net-http-digest_auth (~> 1.4, >= 1.4.1)
       net-http-persistent (>= 2.5.2, < 5.0.dev)
+      nkf
       nokogiri (~> 1.11, >= 1.11.2)
       rubyntlm (~> 0.6, >= 0.6.3)
       webrick (~> 1.7)
       webrobots (~> 0.1.2)
-    method_source (1.0.0)
-    mime-types (3.4.1)
+    mime-types (3.6.1)
+      logger
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.1115)
-    mini_mime (1.1.2)
-    mini_portile2 (2.6.1)
-    minitest (5.15.0)
+    mime-types-data (3.2025.0318)
+    mini_mime (1.1.5)
+    minitest (5.25.5)
     mixlib-shellout (2.4.4)
-    mustermann (1.1.1)
+    mustermann (3.0.3)
       ruby2_keywords (~> 0.0.1)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.1)
+    net-http-persistent (4.0.5)
       connection_pool (~> 2.2)
-    nokogiri (1.12.5)
-      mini_portile2 (~> 2.6.1)
+    nkf (0.2.0)
+    nokogiri (1.18.5-aarch64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.18.5-aarch64-linux-musl)
+      racc (~> 1.4)
+    nokogiri (1.18.5-arm-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.18.5-arm-linux-musl)
+      racc (~> 1.4)
+    nokogiri (1.18.5-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.18.5-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.18.5-x86_64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.18.5-x86_64-linux-musl)
       racc (~> 1.4)
-    pry (0.14.1)
-      coderay (~> 1.1)
-      method_source (~> 1.0)
-    public_suffix (4.0.6)
-    racc (1.6.0)
-    rack (2.2.3)
-    rack-protection (2.1.0)
-      rack
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
-    rake (13.0.6)
-    regexp_parser (2.2.0)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
+    pp (0.6.2)
+      prettyprint
+    prettyprint (0.2.0)
+    psych (5.2.3)
+      date
+      stringio
+    public_suffix (6.0.1)
+    racc (1.8.1)
+    rack (3.1.12)
+    rack-protection (4.1.1)
+      base64 (>= 0.1.0)
+      logger (>= 1.6.0)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.1.0)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0)
+    rack-test (2.2.0)
+      rack (>= 1.3)
+    rackup (2.2.1)
+      rack (>= 3)
+    rake (13.2.1)
+    rdoc (6.12.0)
+      psych (>= 4.0.0)
+    regexp_parser (2.10.0)
+    reline (0.6.0)
+      io-console (~> 0.5)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
     rspec-command (1.0.3)
       mixlib-shellout (~> 2.0)
       rspec (~> 3.2)
       rspec-its (~> 1.2)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec-core (3.13.3)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-its (1.3.0)
+      rspec-support (~> 3.13.0)
+    rspec-its (1.3.1)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
-    rspec-mocks (3.10.2)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.3)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.2)
     ruby2_keywords (0.0.5)
-    rubyntlm (0.6.3)
-    sinatra (2.1.0)
-      mustermann (~> 1.0)
-      rack (~> 2.2)
-      rack-protection (= 2.1.0)
+    rubyntlm (0.6.5)
+      base64
+    securerandom (0.4.1)
+    sinatra (4.1.1)
+      logger (>= 1.6.0)
+      mustermann (~> 3.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.1.1)
+      rack-session (>= 2.0.0, < 3)
       tilt (~> 2.0)
     socksify (1.7.1)
-    tilt (2.0.10)
-    tor (0.1.5)
-    tzinfo (2.0.4)
+    stringio (3.1.5)
+    tilt (2.6.0)
+    tor (0.1.7)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.8)
-    webrick (1.7.0)
+    webrick (1.9.1)
     webrobots (0.1.2)
     xpath (3.2.0)
       nokogiri (~> 1.8)
 
 PLATFORMS
-  ruby
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
+  x86_64-darwin
+  x86_64-linux-gnu
+  x86_64-linux-musl
 
 DEPENDENCIES
   bundler (~> 2.0)
   capybara_discoball
   ffaker
-  pry
+  irb
+  rackup
   rake (~> 13.0)
   rspec (~> 3.0)
   rspec-command
   sinatra
   spidy!
+  webrick
 
 BUNDLED WITH
-   2.2.22
+   2.6.5

data/README.md

@@ -51,7 +51,7 @@ spidy console website.rb
 
 ### reload source code
 ```
-pry(#<Spidy::Console>)> reload!
+irb(#<Spidy::Console>)> reload!
 ```
 
 ```rb

data/bin/console

@@ -3,12 +3,11 @@
 
 require 'bundler/setup'
 require 'spidy'
+require 'irb'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-# (If you use this, don't forget to add pry to your Gemfile!)
-require 'pry'
 def reload!
   ActiveSupport::Dependencies.clear
   ActiveSupport::DescendantsTracker.clear
@@ -18,5 +17,5 @@ end
 if ARGV[0]
   Spidy.open(ARGV[0]).console
 else
-  Pry.start
+  IRB.start
 end

data/example/master_detail.rb

@@ -1,7 +1,7 @@
-
+# frozen_string_literal: true
 
 Spidy.define do
-  url_to_params = ->(url) {
+  url_to_params = lambda { |url|
     uri = URI.parse(url)
     params = URI.decode_www_form(uri.query).to_h if uri.query.present?
     params if params.present?
@@ -13,41 +13,41 @@ Spidy.define do
 
     limit_page = 3
     per_page = 25
-    yielder.call(Nokogiri::HTML::Builder.new { |doc|
-      doc.html {
-        doc.body {
-          doc.span.bold {
-            doc.text "Hello world"
-          }
-          doc.main {
-            (page * per_page + 1).upto((page + 1) * per_page).each do |i|
+    yielder.call(Nokogiri::HTML::Builder.new do |doc|
+      doc.html do
+        doc.body do
+          doc.span.bold do
+            doc.text 'Hello world'
+          end
+          doc.main do
+            ((page * per_page) + 1).upto((page + 1) * per_page).each do |i|
               doc.a("page #{i}", href: "http://localhost/?id=#{i}")
             end
-          }
+          end
           doc.a('NEXT', href: "http://localhost/?page=#{page + 1}", class: 'next') if page < limit_page
-        }
-      }
-    }.doc)
+        end
+      end
+    end.doc)
   }
 
   detail_page = proc { |url, &yielder|
     params = url_to_params.call(url)
     id = params['id']
 
-    yielder.call(Nokogiri::HTML::Builder.new { |doc|
-      doc.html {
-        doc.body {
-          doc.span.bold {
-            doc.text "Hello world"
-          }
+    yielder.call(Nokogiri::HTML::Builder.new do |doc|
+      doc.html do
+        doc.body do
+          doc.span.bold do
+            doc.text 'Hello world'
+          end
           doc.h1("title_#{id}", id: 'title')
           doc.main("body_#{id}", id: 'body')
           doc.div.sub do
             doc.span.name('testtest')
           end
-        }
-      }
-    }.doc)
+        end
+      end
+    end.doc)
   }
 
   define(as: :html, connector: detail_page) do

data/example/proxy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Spidy.define do
   user_agent 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0'
   socks_proxy '127.0.0.1', 9050

data/example/retry.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Spidy.define do
   spider(as: :json) do |yielder, connector|
     connector.call('https://httpbin.org/status/500') do |json|

data/example/wikip.rb

@@ -11,11 +11,8 @@ Spidy.define do
 
   define(:infobox, as: :html, connector: :direct) do
     let(:columns) do
-      html.search('tr').each do |tr|
-        {
-          name: tr.at('th')&.text,
-          value: tr.at('td')&.text
-        }
+      html.search('tr').map do |tr|
+        { name: tr.at('th')&.text, value: tr.at('td')&.text }
       end
     end
   end

data/exe/spidy

@@ -2,14 +2,13 @@
 # frozen_string_literal: true
 
 require 'spidy'
-require 'pry'
 
 if ARGV[1].blank?
   case ARGV[0]
-  when 'version' then STDOUT.puts(Spidy::VERSION)
+  when 'version' then $stdout.puts(Spidy::VERSION)
   when 'console' then Spidy.shell.interactive
   else
-    STDOUT.puts 'usage: spidy [version console]'
+    $stdout.puts 'usage: spidy [version console]'
   end
 else
   case ARGV[0]
@@ -19,6 +18,6 @@ else
   when 'each' then Spidy.shell(ARGV[1]).each(ARGV[2])
   when 'eval' then Spidy.shell(ARGV[1]).eval_call(ARGV[2])
   else
-    STDOUT.puts 'usage: spidy [console function call each run] [file]'
+    $stdout.puts 'usage: spidy [console function call each run] [file]'
   end
 end

data/lib/spidy/binder/error.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class Spidy::Binder::Error < StandardError
+end

data/lib/spidy/binder/html.rb

@@ -17,9 +17,10 @@ module Spidy::Binder::Html
         instance_exec(&block)
       end
     rescue StandardError => e
-      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
+      raise Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
+
   def self.extended(obj)
     obj.alias_method :html, :resource
   end

data/lib/spidy/binder/json.rb

@@ -17,9 +17,10 @@ module Spidy::Binder::Json
         instance_exec(&block)
       end
     rescue StandardError => e
-      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
+      raise Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
+
   def self.extended(obj)
     obj.alias_method :json, :resource
   end

data/lib/spidy/binder/xml.rb

@@ -17,9 +17,10 @@ module Spidy::Binder::Xml
         instance_exec(&block)
       end
     rescue StandardError => e
-      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
+      raise Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
+
   def self.extended(obj)
     obj.alias_method :xml, :resource
   end

data/lib/spidy/binder.rb

@@ -5,6 +5,7 @@
 #
 module Spidy::Binder
   extend ActiveSupport::Autoload
+  autoload :Error
   autoload :Json
   autoload :Html
   autoload :Xml

data/lib/spidy/command_line.rb

@@ -5,8 +5,10 @@
 #
 class Spidy::CommandLine
   delegate :spidy, to: :@definition_file
-  class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
-  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
+  class_attribute :output, default: (proc { |result| $stdout.puts(result.to_s) })
+  class_attribute :error_handler, default: (proc { |e, url|
+                                              warn({ url: url, message: e.message, backtrace: e.backtrace }.to_json)
+                                            })
 
   def eval_call(script)
     @definition_file.spidy.instance_eval(script)
@@ -14,40 +16,36 @@ class Spidy::CommandLine
 
   def initialize(definition_file)
     @definition_file = definition_file
-    raise 'unloaded spidy' if definition_file.spidy.nil?
+    fail 'unloaded spidy' if definition_file.spidy.nil?
   end
 
   def each_stdin_lines(name)
-    STDIN.each_line do |url|
-      begin
-        spidy.each(url.strip, name: name, &output)
-      rescue => e
-        error_handler.call(e, url)
-      end
+    $stdin.each_line do |url|
+      spidy.each(url.strip, name: name, &output)
+    rescue StandardError => e
+      error_handler.call(e, url)
     end
   end
 
   def call_stdin_lines(name)
-    STDIN.each_line do |url|
-      begin
-        spidy.call(url.strip, name: name, &output)
-      rescue => e
-        error_handler.call(e, url)
-      end
+    $stdin.each_line do |url|
+      spidy.call(url.strip, name: name, &output)
+    rescue StandardError => e
+      error_handler.call(e, url)
     end
   end
 
   def call(name)
-    return call_stdin_lines(name) if FileTest.pipe?(STDIN)
-    spidy.call(name: name, &output) unless FileTest.pipe?(STDIN)
-  rescue => e
+    return call_stdin_lines(name) if FileTest.pipe?($stdin)
+    spidy.call(name: name, &output) unless FileTest.pipe?($stdin)
+  rescue StandardError => e
     error_handler.call(e, nil)
   end
 
   def each(name)
-    return each_stdin_lines(name) if FileTest.pipe?(STDIN)
+    return each_stdin_lines(name) if FileTest.pipe?($stdin)
     spidy.each(name: name, &output)
-  rescue => e
+  rescue StandardError => e
     error_handler.call(e, nil)
   end
 
@@ -63,36 +61,32 @@ class Spidy::CommandLine
   end
 
   def build(name)
-    build_shell(name)
-    build_ruby(name)
+    File.write("#{name}.sh", build_shell_script(name))
+    File.write("#{name}.rb", build_ruby_script)
   end
 
   def build_shell(name)
-    File.open("#{name}.sh", 'w') do |f|
-      f.write <<~SHELL
-        #!/bin/bash
-        eval "$(spidy $(dirname "${0}")/#{name}.rb shell)"
-        spider example
-      SHELL
-    end
+    <<~SHELL
+      #!/bin/bash
+      eval "$(spidy $(dirname "${0}")/#{name}.rb shell)"
+      spider
+    SHELL
   end
 
-  def build_ruby(name)
-    File.open("#{name}.rb", 'w') do |f|
-      f.write <<~RUBY
-        # frozen_string_literal: true
+  def build_ruby
+    <<~RUBY
+      # frozen_string_literal: true
 
-        Spidy.define do
-          spider(:example) do |yielder, connector|
-            # connector.call(url) do |resource|
-            #   yielder.call(url or resource)
-            # end
-          end
+      Spidy.define do
+        spider(as: :html) do |yielder, connector|
+          # connector.call(url) do |resource|
+          #   yielder.call(url or resource)
+          # end
+        end
 
-          define(:example) do
-          end
+        define(as: :html) do
         end
-      RUBY
-    end
+      end
+    RUBY
   end
 end

data/lib/spidy/connector/direct.rb

@@ -4,7 +4,7 @@
 # Direct resource ( not network resource  )
 #
 class Spidy::Connector::Direct
-  def call(resource, &yielder)
+  def call(resource)
     if block_given?
       yield resource
     else
@@ -12,6 +12,5 @@ class Spidy::Connector::Direct
     end
   end
 
-  def initialize(user_agent:)
-  end
+  def initialize(user_agent:); end
 end

data/lib/spidy/connector/html.rb

@@ -14,13 +14,13 @@ class Spidy::Connector::Html
 
   attr_reader :agent
 
-  def call(url, encoding: nil, retry_count: 5, &yielder)
+  def call(url, encoding: nil, &yielder)
     fail 'url is not specified' if url.blank?
     if encoding
       agent.default_encoding = encoding
       agent.force_default_encoding = true
     end
-    connect(url, retry_count, yielder)
+    connect(url, yielder)
   end
 
   def refresh!
@@ -30,17 +30,19 @@ class Spidy::Connector::Html
 
   private
 
-  def connect(url, retry_count, yielder)
+  def connect(url, yielder)
     result = nil
     agent.get(url) do |page|
-      fail Spidy::Connector::Retry, object: page, response_code: page.try(:response_code) if page.title == 'Sorry, unable to access page...'
+      if page.title == 'Sorry, unable to access page...'
+        fail Spidy::Connector::Retry.new(object: page, response_code: page.try(:response_code))
+      end
 
       result = yielder.call(page)
     end
     result
   rescue Mechanize::ResponseCodeError => e
-    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '429'
-    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code) if e.response_code == '502'
-    raise Spidy::Connector::Retry, error: e, response_code: e.try(:response_code)
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.try(:response_code)) if e.response_code == '429'
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.try(:response_code)) if e.response_code == '502'
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.try(:response_code))
   end
 end

data/lib/spidy/connector/json.rb

@@ -17,9 +17,9 @@ class Spidy::Connector::Json
     connect(url, &block)
   end
 
-  def connect(url, retry_count: 5)
-    OpenURI.open_uri(url, "User-Agent" => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
+  def connect(url)
+    OpenURI.open_uri(url, 'User-Agent' => @user_agent) { |body| yield JSON.parse(body.read, symbolize_names: true) }
   rescue OpenURI::HTTPError => e
-    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.io.status[0])
   end
 end

data/lib/spidy/connector/xml.rb

@@ -13,11 +13,11 @@ class Spidy::Connector::Xml
   end
 
   def connect(url, &block)
-    OpenURI.open_uri(url, "User-Agent" => @user_agent) do |body|
+    OpenURI.open_uri(url, 'User-Agent' => @user_agent) do |body|
       block.call Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
   rescue OpenURI::HTTPError => e
-    raise Spidy::Connector::Retry, error: e, response_code: e.io.status[0]
+    raise Spidy::Connector::Retry.new(error: e, response_code: e.io.status[0])
   end
 
   def initialize(user_agent:)

data/lib/spidy/connector.rb

@@ -27,7 +27,7 @@ module Spidy::Connector
   #
   # error output logger
   #
-  DEFAULT_LOGGER = proc { |values| STDERR.puts(values.to_json) }
+  DEFAULT_LOGGER = proc { |values| warn(values.to_json) }
 
   #
   # static method
@@ -36,7 +36,9 @@ module Spidy::Connector
     extend ActiveSupport::Concern
     class_methods do
       def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
-        ::Spidy::Connector::RetryableCaller.new(new(user_agent: user_agent), wait_time: wait_time, logger: logger).call(url, &block)
+        ::Spidy::Connector::RetryableCaller.new(new(user_agent: user_agent), wait_time: wait_time, logger: logger).call(
+          url, &block
+        )
       end
     end
   end
@@ -51,6 +53,7 @@ module Spidy::Connector
       @object = object
       @response_code = response_code
       @error = error
+      super(error)
     end
   end
 
@@ -58,13 +61,13 @@ module Spidy::Connector
   # retry
   #
   class RetryableCaller
-    attr_reader :origin_connector
+    attr_reader :origin_connector, :logger, :wait_time
 
-    def initialize(connector, logger:, wait_time:)
+    def initialize(connector, logger:, wait_time:, retry_attempt_count: 5)
       @origin_connector = connector
       @logger = logger
       @wait_time = wait_time
-      @retry_attempt_count = 5
+      @retry_attempt_count = retry_attempt_count
     end
 
     def call(url, &block)
@@ -73,18 +76,18 @@ module Spidy::Connector
     end
 
     def connect(url, retry_attempt_count: @retry_attempt_count, &block)
-      @logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-      @origin_connector.call(url, &block)
+      logger.call('connnector.get': url, 'connnector.accessed': Time.current)
+      origin_connector.call(url, &block)
     rescue Spidy::Connector::Retry => e
-      @logger.call('retry.accessed': Time.current,
-                   'retry.uri': url,
-                   'retry.response_code': e.response_code,
-                   'retry.attempt_count': retry_attempt_count)
+      logger.call('retry.accessed': Time.current,
+                  'retry.uri': url,
+                  'retry.response_code': e.response_code,
+                  'retry.attempt_count': retry_attempt_count)
 
       retry_attempt_count -= 1
       if retry_attempt_count.positive?
-        sleep @wait_time
-        @origin_connector.refresh! if @origin_connector.respond_to?(:refresh!)
+        sleep wait_time
+        origin_connector.refresh! if origin_connector.respond_to?(:refresh!)
         retry
       end
       raise e.error
@@ -103,7 +106,7 @@ module Spidy::Connector
     end
 
     def call(url, &block)
-      Socksify::proxy(socks_proxy[:host], socks_proxy[:port]) do
+      Socksify.proxy(socks_proxy[:host], socks_proxy[:port]) do
         connector.call(url, &block)
       end
     end
@@ -141,7 +144,6 @@ module Spidy::Connector
     fail "Not defined connnector[#{value}]" if connector.nil?
     return connector if socks_proxy.nil?
 
-    tor = TorConnector.new(connector, socks_proxy)
-    tor
+    TorConnector.new(connector, socks_proxy)
   end
 end

data/lib/spidy/definition.rb

@@ -33,26 +33,34 @@ module Spidy::Definition
     spidy = @namespace[:"#{name}_spider"]
     fail "undefined spidy [#{name}]" if spidy.nil?
 
-    spidy.call(source, &yielder)
+    if yielder
+      spidy.call(source, &yielder)
+    else
+      Enumerator.new do |enumerate_yielder|
+        spidy.call(source, &enumerate_yielder)
+      end
+    end
   end
 
   def spider(name = :default, connector: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent,
+                                                      socks_proxy: @socks_proxy)
     @namespace[:"#{name}_spider"] = proc do |source, &yielder|
       define_block.call(yielder, connector, source)
     end
   end
 
   def define(name = :default, connector: nil, as: nil, &define_block)
-    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
+    connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent,
+                                                      socks_proxy: @socks_proxy)
     binder_base = Spidy::Binder.const_get(as.to_s.classify)
     @namespace ||= {}
-    @namespace[:"#{name}_scraper"] = Class.new(Spidy::DefineObject) do
+    @namespace[:"#{name}_scraper"] = Class.new(Spidy::DefinitionObject) do
       extend binder_base
       class_eval(&define_block)
       define_singleton_method(:call) do |source, &yielder|
-        yielder = lambda { |result| break result } if yielder.nil?
+        yielder = ->(result) { break result } if yielder.nil?
         connection_yielder = lambda do |page|
           yielder.call(new(page, source))
         end

data/lib/spidy/definition_file.rb

@@ -4,8 +4,7 @@
 # spidy interface binding
 #
 class Spidy::DefinitionFile
-  attr_reader :path
-  attr_reader :spidy
+  attr_reader :path, :spidy
 
   def self.open(filepath)
     object = new(filepath)
@@ -15,7 +14,7 @@ class Spidy::DefinitionFile
 
   # rubocop:disable Security/Eval
   def eval_definition
-    @spidy = eval(File.open(path).read) if path
+    @spidy = eval(File.read(path)) if path
   end
   # rubocop:enable Security/Eval
 

data/lib/spidy/{define_object.rb → definition_object.rb}

@@ -1,4 +1,9 @@
-class Spidy::DefineObject
+# frozen_string_literal: true
+
+#
+# An object that represents the scraper defined by define block.
+#
+class Spidy::DefinitionObject
   class << self
     attr_reader :attribute_names
   end
@@ -14,6 +19,6 @@ class Spidy::DefineObject
   end
 
   def to_h
-    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
+    self.class.attribute_names.to_h { |name| [name, send(name)] }
   end
 end

data/lib/spidy/shell.rb

@@ -9,7 +9,12 @@ class Spidy::Shell
   end
 
   def interactive
-    Pry.start(Spidy::Console.new(@definition_file))
+    console = Spidy::Console.new(@definition_file)
+    require 'irb'
+    IRB.setup(nil)
+    irb = IRB::Irb.new(IRB::WorkSpace.new(console))
+    IRB.conf[:MAIN_CONTEXT] = irb.context
+    irb.eval_input
   end
 
   def command_line

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.3.9'
+  VERSION = '0.4.0'
 end

data/lib/spidy.rb

@@ -17,9 +17,9 @@ module Spidy
   autoload :Console
   autoload :Definition
   autoload :DefinitionFile
+  autoload :DefinitionObject
   autoload :Binder
   autoload :Connector
-  autoload :DefineObject
 
   def self.shell(filepath = nil)
     Spidy::Shell.new(filepath)

data/spidy.gemspec

@@ -25,17 +25,18 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'capybara_discoball'
+  spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'
-  spec.add_development_dependency 'capybara_discoball'
   spec.add_development_dependency 'sinatra'
 
-  spec.add_runtime_dependency 'tor'
-  spec.add_runtime_dependency 'activesupport'
+  spec.add_runtime_dependency 'activesupport', '~> 7.1'
   spec.add_runtime_dependency 'mechanize'
   spec.add_runtime_dependency 'socksify'
-  spec.add_runtime_dependency 'pry'
+  spec.add_runtime_dependency 'tor'
+  spec.metadata = {
+    'rubygems_mfa_required' => 'true'
+  }
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.4.0
 platform: ruby
 authors:
 - aileron
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-12-23 00:00:00.000000000 Z
+date: 2025-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -25,7 +24,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: capybara_discoball
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -38,34 +37,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: ffaker
   requirement: !ruby/object:Gem::Requirement
@@ -81,35 +52,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec-command
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: capybara_discoball
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: sinatra
+  name: rspec-command
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,13 +94,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: tor
+  name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -140,16 +111,16 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
@@ -179,7 +150,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: tor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -192,7 +163,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email:
 - aileron.cc@gmail.com
 executables:
@@ -207,6 +177,7 @@ files:
 - ".ruby-version"
 - ".travis.yml"
 - CHANGELOG.md
+- CLAUDE.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -222,6 +193,7 @@ files:
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb
+- lib/spidy/binder/error.rb
 - lib/spidy/binder/html.rb
 - lib/spidy/binder/json.rb
 - lib/spidy/binder/xml.rb
@@ -232,9 +204,9 @@ files:
 - lib/spidy/connector/json.rb
 - lib/spidy/connector/xml.rb
 - lib/spidy/console.rb
-- lib/spidy/define_object.rb
 - lib/spidy/definition.rb
 - lib/spidy/definition_file.rb
+- lib/spidy/definition_object.rb
 - lib/spidy/shell.rb
 - lib/spidy/spider.rb
 - lib/spidy/version.rb
@@ -243,8 +215,8 @@ files:
 homepage: https://github.com/aileron-inc/spidy
 licenses:
 - MIT
-metadata: {}
-post_install_message:
+metadata:
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -259,8 +231,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
-signing_key:
+rubygems_version: 3.6.5
 specification_version: 4
 summary: web spider dsl
 test_files: []