spidy 0.3.3 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a70bf5d610f60d0b71c719cf870995a7e93b6e9abd71ef9823a71e2ed506f190
-  data.tar.gz: 203bc7721020e244b9ad3ecc526b27e1c8d83a807cc2798bcb8930b1e17d5277
+  metadata.gz: 75215453c834a8e481b27cf4377235cc97ce6a6e4eff142a11743e68ee4982b4
+  data.tar.gz: 25f3b14ad6f31b580396458c8075167f24fd4a2f6e7ff98947338ecd0588eb9d
 SHA512:
-  metadata.gz: 9977aeb13ff786bd8fbeb7d8ca0ee3ef7b67dfb577739c9aba0a310c46741fa48d86596de6f69ad05eaf2d64f9de4259bea84d4939ee2c7288781106fb25f2b3
-  data.tar.gz: d64e9e66b25d8985f2c009abddd6f6b862aa8f533b67e125fa35008d308424e766d3ad4a8a64c16a0c9ed3448cdad14a09c80f6f77437e604826808286270d4f
+  metadata.gz: 447b7152b807c7985e16b7b403d27f9f7b949264577e8e4dc11a52358cb9af49510696d29166adb60bb5b87158aa8d2c10faf7c810f8eced4c29f9eed8bb493a
+  data.tar.gz: 55d82e5c495a7e5a0fd57b466e08ea072e3a712829b2d83d550b107147f30e3eaff54ed8dcec69163bf6f74e0c32990051af9de0d044cc19eadf39ecc749d003

data/.ruby-version

@@ -1 +1 @@
-2.6.6
+3.0.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.3.3)
+    spidy (0.3.9)
       activesupport
       mechanize
       pry
@@ -11,110 +11,114 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.3)
+    activesupport (7.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
-    addressable (2.7.0)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    capybara (3.33.0)
+    capybara (3.36.0)
       addressable
+      matrix
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
-      regexp_parser (~> 1.5)
+      regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
     capybara_discoball (0.1.0)
       capybara (>= 2.7, < 4)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.7)
-    connection_pool (2.2.3)
-    diff-lcs (1.3)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffaker (2.10.0)
-    http-cookie (1.0.3)
+    ffaker (2.20.0)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
-    i18n (1.8.5)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    mechanize (2.7.6)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (>= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (>= 0.0.9, < 0.2)
-    method_source (0.9.2)
-    mime-types (3.3.1)
+    matrix (0.4.2)
+    mechanize (2.8.3)
+      addressable (~> 2.8)
+      domain_name (~> 0.5, >= 0.5.20190701)
+      http-cookie (~> 1.0, >= 1.0.3)
+      mime-types (~> 3.0)
+      net-http-digest_auth (~> 1.4, >= 1.4.1)
+      net-http-persistent (>= 2.5.2, < 5.0.dev)
+      nokogiri (~> 1.11, >= 1.11.2)
+      rubyntlm (~> 0.6, >= 0.6.3)
+      webrick (~> 1.7)
+      webrobots (~> 0.1.2)
+    method_source (1.0.0)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
-    mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
-    minitest (5.14.2)
+    mime-types-data (3.2021.1115)
+    mini_mime (1.1.2)
+    mini_portile2 (2.6.1)
+    minitest (5.15.0)
     mixlib-shellout (2.4.4)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.0)
+    net-http-persistent (4.0.1)
       connection_pool (~> 2.2)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    ntlm-http (0.1.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    nokogiri (1.12.5)
+      mini_portile2 (~> 2.6.1)
+      racc (~> 1.4)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     public_suffix (4.0.6)
+    racc (1.6.0)
     rack (2.2.3)
-    rack-protection (2.0.8.1)
+    rack-protection (2.1.0)
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rake (10.5.0)
-    regexp_parser (1.8.1)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
+    rake (13.0.6)
+    regexp_parser (2.2.0)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
     rspec-command (1.0.3)
       mixlib-shellout (~> 2.0)
       rspec (~> 3.2)
       rspec-its (~> 1.2)
-    rspec-core (3.8.2)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.4)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
+      rspec-support (~> 3.10.0)
     rspec-its (1.3.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
-    rspec-mocks (3.8.1)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.2)
-    ruby2_keywords (0.0.2)
-    sinatra (2.0.8.1)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    ruby2_keywords (0.0.5)
+    rubyntlm (0.6.3)
+    sinatra (2.1.0)
       mustermann (~> 1.0)
-      rack (~> 2.0)
-      rack-protection (= 2.0.8.1)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
       tilt (~> 2.0)
     socksify (1.7.1)
-    thread_safe (0.3.6)
     tilt (2.0.10)
-    tor (0.1.4)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
+    tor (0.1.5)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
+    unf_ext (0.0.8)
+    webrick (1.7.0)
     webrobots (0.1.2)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
@@ -124,11 +128,11 @@ DEPENDENCIES
   capybara_discoball
   ffaker
   pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   rspec-command
   sinatra
   spidy!
 
 BUNDLED WITH
-   2.1.4
+   2.2.22

data/example/wikip.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+Spidy.define do
+  def self.infobox_scrape(params, &block)
+    call(params.html.at('.infobox'), name: :infobox, &block)
+  end
+
+  define(as: :html) do
+    let(:title, 'h1')
+  end
+
+  define(:infobox, as: :html, connector: :direct) do
+    let(:columns) do
+      html.search('tr').each do |tr|
+        {
+          name: tr.at('th')&.text,
+          value: tr.at('td')&.text
+        }
+      end
+    end
+  end
+end

data/exe/spidy

@@ -17,7 +17,8 @@ else
   when 'function' then Spidy.shell(ARGV[1]).function
   when 'call' then Spidy.shell(ARGV[1]).call(ARGV[2])
   when 'each' then Spidy.shell(ARGV[1]).each(ARGV[2])
+  when 'eval' then Spidy.shell(ARGV[1]).eval_call(ARGV[2])
   else
-    STDOUT.puts 'usage: spidy [console function call each] [file]'
+    STDOUT.puts 'usage: spidy [console function call each run] [file]'
   end
 end

data/lib/spidy/binder/html.rb

@@ -3,12 +3,12 @@
 #
 # Bind html and convert to object
 #
-class Spidy::Binder::Html < Spidy::Binder::Base
-  def self.let(name, query = nil, &block)
+module Spidy::Binder::Html
+  def let(name, query = nil, &block)
     @attribute_names ||= []
     @attribute_names << name
 
-    return define_method(name) { html.at(query)&.text } if block.nil?
+    return define_method(name) { html.at(query)&.text&.strip } if block.nil?
 
     define_method(name) do
       if query.present?
@@ -20,6 +20,7 @@ class Spidy::Binder::Html < Spidy::Binder::Base
       fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
-
-  alias_method :html, :resource
+  def self.extended(obj)
+    obj.alias_method :html, :resource
+  end
 end

data/lib/spidy/binder/json.rb

@@ -3,8 +3,8 @@
 #
 # Bind json and convert to object
 #
-class Spidy::Binder::Json < Spidy::Binder::Base
-  def self.let(name, *query, &block)
+module Spidy::Binder::Json
+  def let(name, *query, &block)
     @attribute_names ||= []
     @attribute_names << name
 
@@ -20,6 +20,7 @@ class Spidy::Binder::Json < Spidy::Binder::Base
       fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
-
-  alias_method :json, :resource
+  def self.extended(obj)
+    obj.alias_method :json, :resource
+  end
 end

data/lib/spidy/binder/xml.rb

@@ -3,12 +3,12 @@
 #
 # Bind xml and convert to object
 #
-class Spidy::Binder::Xml < Spidy::Binder::Base
-  def self.let(name, query = nil, &block)
+module Spidy::Binder::Xml
+  def let(name, query = nil, &block)
     @attribute_names ||= []
     @attribute_names << name
 
-    return define_method(name) { xml.at(query)&.text } if block.nil?
+    return define_method(name) { xml.at(query)&.text&.strip } if block.nil?
 
     define_method(name) do
       if query.present?
@@ -20,6 +20,7 @@ class Spidy::Binder::Xml < Spidy::Binder::Base
       fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
-
-  alias_method :xml, :resource
+  def self.extended(obj)
+    obj.alias_method :xml, :resource
+  end
 end

data/lib/spidy/binder.rb

@@ -8,48 +8,4 @@ module Spidy::Binder
   autoload :Json
   autoload :Html
   autoload :Xml
-
-  class Error < StandardError
-  end
-
-  class Caller
-    def initialize(spidy, binder)
-      @spidy = spidy
-      @binder = binder
-    end
-
-    def call(source, url: nil, define: nil, define_name: nil)
-      yield Class.new(@binder, &define).new(define_name, @spidy, source, url)
-    end
-  end
-
-  class Base
-    class << self
-      attr_reader :attribute_names
-    end
-
-    attr_reader :resource, :url
-
-    def initialize(define_name, spidy, resource, url)
-      @define_name = define_name
-      @spidy = spidy
-      @resource = resource
-      @url = url
-    end
-
-    def to_s
-      to_h.to_json
-    end
-
-    def to_h
-      self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-    end
-  end
-
-
-  def self.get(spidy, value)
-    return Caller.new(spidy, const_get(value.to_s.classify)) if name.is_a?(String) || name.is_a?(Symbol)
-
-    value
-  end
 end

data/lib/spidy/command_line.rb

@@ -8,6 +8,10 @@ class Spidy::CommandLine
   class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
   class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
 
+  def eval_call(script)
+    @definition_file.spidy.instance_eval(script)
+  end
+
   def initialize(definition_file)
     @definition_file = definition_file
     raise 'unloaded spidy' if definition_file.spidy.nil?

data/lib/spidy/connector/direct.rb

@@ -5,7 +5,11 @@
 #
 class Spidy::Connector::Direct
   def call(resource, &yielder)
-    yielder.call(resource)
+    if block_given?
+      yield resource
+    else
+      resource
+    end
   end
 
   def initialize(user_agent:)

data/lib/spidy/connector.rb

@@ -126,6 +126,8 @@ module Spidy::Connector
     wait_time ||= DEFAULT_WAIT_TIME
 
     connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
+    return connector if connector.is_a?(Spidy::Connector::Direct)
+
     RetryableCaller.new(connector, wait_time: wait_time, logger: logger)
   end
 

data/lib/spidy/define_object.rb

@@ -0,0 +1,19 @@
+class Spidy::DefineObject
+  class << self
+    attr_reader :attribute_names
+  end
+  attr_reader :resource, :url
+
+  def initialize(resource, url)
+    @resource = resource
+    @url = url
+  end
+
+  def to_s
+    to_h.to_json
+  end
+
+  def to_h
+    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
+  end
+end

data/lib/spidy/definition.rb

@@ -44,22 +44,20 @@ module Spidy::Definition
     end
   end
 
-  def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
-    @namespace ||= {}
+  def define(name = :default, connector: nil, as: nil, &define_block)
     connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
-    binder = Spidy::Binder.get(self, binder || as)
-    @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
-  end
-
-  private
-
-  def define_proc(name, connector, binder, define_block)
-    proc do |source, &yielder|
-      yielder = lambda { |result| break result } if yielder.nil?
-      connection_yielder = lambda do |page|
-        binder.call(page, url: source, define: define_block, define_name: name) { |object| yielder.call(object) }
+    binder_base = Spidy::Binder.const_get(as.to_s.classify)
+    @namespace ||= {}
+    @namespace[:"#{name}_scraper"] = Class.new(Spidy::DefineObject) do
+      extend binder_base
+      class_eval(&define_block)
+      define_singleton_method(:call) do |source, &yielder|
+        yielder = lambda { |result| break result } if yielder.nil?
+        connection_yielder = lambda do |page|
+          yielder.call(new(page, source))
+        end
+        connector.call(source, &connection_yielder)
       end
-      connector.call(source, &connection_yielder)
     end
   end
 end

data/lib/spidy/shell.rb

@@ -16,5 +16,5 @@ class Spidy::Shell
     Spidy::CommandLine.new(@definition_file)
   end
 
-  delegate :function, :each, :call, to: :command_line
+  delegate :function, :each, :call, :eval_call, to: :command_line
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.3.3'
+  VERSION = '0.3.9'
 end

data/lib/spidy.rb

@@ -19,6 +19,7 @@ module Spidy
   autoload :DefinitionFile
   autoload :Binder
   autoload :Connector
+  autoload :DefineObject
 
   def self.shell(filepath = nil)
     Spidy::Shell.new(filepath)

data/spidy.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.9
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2021-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -218,6 +218,7 @@ files:
 - example/master_detail.rb
 - example/proxy.rb
 - example/retry.rb
+- example/wikip.rb
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb
@@ -231,6 +232,7 @@ files:
 - lib/spidy/connector/json.rb
 - lib/spidy/connector/xml.rb
 - lib/spidy/console.rb
+- lib/spidy/define_object.rb
 - lib/spidy/definition.rb
 - lib/spidy/definition_file.rb
 - lib/spidy/shell.rb
@@ -257,7 +259,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: web spider dsl