spidy 0.3.8 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 666e9b2dcad792e940aa9e6b6cb99eaf759b9e6af1b187519cbdb8bd573a2861
-  data.tar.gz: 8477c2617d04470030ff95673033382996f2988d5f95ebacb5f94272309ad851
+  metadata.gz: 75215453c834a8e481b27cf4377235cc97ce6a6e4eff142a11743e68ee4982b4
+  data.tar.gz: 25f3b14ad6f31b580396458c8075167f24fd4a2f6e7ff98947338ecd0588eb9d
 SHA512:
-  metadata.gz: 422486fe8f5e0207e3f55832051e04e22dd07cbbc6e1efa12a6db57bbb0103244f2e41516018da0597bdd785a7d3a8171d4c9bb32d3a503c358502a6f3133d44
-  data.tar.gz: a841cc76004f5a020c4c3c343504be1448fbbcebd90fc1a06c3d34a9657acbe1d39a5c5c5be9b08433a34be4875675e62250578d2bb8d2b7e68a20150c2e3506
+  metadata.gz: 447b7152b807c7985e16b7b403d27f9f7b949264577e8e4dc11a52358cb9af49510696d29166adb60bb5b87158aa8d2c10faf7c810f8eced4c29f9eed8bb493a
+  data.tar.gz: 55d82e5c495a7e5a0fd57b466e08ea072e3a712829b2d83d550b107147f30e3eaff54ed8dcec69163bf6f74e0c32990051af9de0d044cc19eadf39ecc749d003

data/.ruby-version

@@ -1 +1 @@
-2.6.6
+3.0.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.3.7)
+    spidy (0.3.9)
       activesupport
       mechanize
       pry
@@ -11,99 +11,101 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3)
+    activesupport (7.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    capybara (3.33.0)
+    capybara (3.36.0)
       addressable
+      matrix
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
-      regexp_parser (~> 1.5)
+      regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
     capybara_discoball (0.1.0)
       capybara (>= 2.7, < 4)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.8)
-    connection_pool (2.2.3)
-    diff-lcs (1.3)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffaker (2.10.0)
-    http-cookie (1.0.3)
+    ffaker (2.20.0)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
-    i18n (1.8.9)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    mechanize (2.7.7)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (>= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
+    matrix (0.4.2)
+    mechanize (2.8.3)
+      addressable (~> 2.8)
+      domain_name (~> 0.5, >= 0.5.20190701)
+      http-cookie (~> 1.0, >= 1.0.3)
+      mime-types (~> 3.0)
+      net-http-digest_auth (~> 1.4, >= 1.4.1)
+      net-http-persistent (>= 2.5.2, < 5.0.dev)
+      nokogiri (~> 1.11, >= 1.11.2)
+      rubyntlm (~> 0.6, >= 0.6.3)
       webrick (~> 1.7)
-      webrobots (>= 0.0.9, < 0.2)
-    method_source (0.9.2)
-    mime-types (3.3.1)
+      webrobots (~> 0.1.2)
+    method_source (1.0.0)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0225)
-    mini_mime (1.0.2)
-    mini_portile2 (2.5.0)
-    minitest (5.14.4)
+    mime-types-data (3.2021.1115)
+    mini_mime (1.1.2)
+    mini_portile2 (2.6.1)
+    minitest (5.15.0)
     mixlib-shellout (2.4.4)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     net-http-digest_auth (1.4.1)
     net-http-persistent (4.0.1)
       connection_pool (~> 2.2)
-    nokogiri (1.11.2)
-      mini_portile2 (~> 2.5.0)
+    nokogiri (1.12.5)
+      mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
-    ntlm-http (0.1.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     public_suffix (4.0.6)
-    racc (1.5.2)
+    racc (1.6.0)
     rack (2.2.3)
-    rack-protection (2.0.8.1)
+    rack-protection (2.1.0)
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rake (13.0.3)
-    regexp_parser (1.8.1)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
+    rake (13.0.6)
+    regexp_parser (2.2.0)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
     rspec-command (1.0.3)
       mixlib-shellout (~> 2.0)
       rspec (~> 3.2)
       rspec-its (~> 1.2)
-    rspec-core (3.8.2)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.4)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
+      rspec-support (~> 3.10.0)
     rspec-its (1.3.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
-    rspec-mocks (3.8.1)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.2)
-    ruby2_keywords (0.0.2)
-    sinatra (2.0.8.1)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    ruby2_keywords (0.0.5)
+    rubyntlm (0.6.3)
+    sinatra (2.1.0)
       mustermann (~> 1.0)
-      rack (~> 2.0)
-      rack-protection (= 2.0.8.1)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
       tilt (~> 2.0)
     socksify (1.7.1)
     tilt (2.0.10)
@@ -112,12 +114,11 @@ GEM
       concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
+    unf_ext (0.0.8)
     webrick (1.7.0)
     webrobots (0.1.2)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
@@ -134,4 +135,4 @@ DEPENDENCIES
   spidy!
 
 BUNDLED WITH
-   2.2.14
+   2.2.22

data/exe/spidy

@@ -17,7 +17,8 @@ else
   when 'function' then Spidy.shell(ARGV[1]).function
   when 'call' then Spidy.shell(ARGV[1]).call(ARGV[2])
   when 'each' then Spidy.shell(ARGV[1]).each(ARGV[2])
+  when 'eval' then Spidy.shell(ARGV[1]).eval_call(ARGV[2])
   else
-    STDOUT.puts 'usage: spidy [console function call each] [file]'
+    STDOUT.puts 'usage: spidy [console function call each run] [file]'
   end
 end

data/lib/spidy/binder/html.rb

@@ -3,8 +3,8 @@
 #
 # Bind html and convert to object
 #
-class Spidy::Binder::Html < Spidy::Binder::Base
-  def self.let(name, query = nil, &block)
+module Spidy::Binder::Html
+  def let(name, query = nil, &block)
     @attribute_names ||= []
     @attribute_names << name
 
@@ -20,6 +20,7 @@ class Spidy::Binder::Html < Spidy::Binder::Base
       fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
-
-  alias_method :html, :resource
+  def self.extended(obj)
+    obj.alias_method :html, :resource
+  end
 end

data/lib/spidy/binder/json.rb

@@ -3,8 +3,8 @@
 #
 # Bind json and convert to object
 #
-class Spidy::Binder::Json < Spidy::Binder::Base
-  def self.let(name, *query, &block)
+module Spidy::Binder::Json
+  def let(name, *query, &block)
     @attribute_names ||= []
     @attribute_names << name
 
@@ -20,6 +20,7 @@ class Spidy::Binder::Json < Spidy::Binder::Base
       fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
-
-  alias_method :json, :resource
+  def self.extended(obj)
+    obj.alias_method :json, :resource
+  end
 end

data/lib/spidy/binder/xml.rb

@@ -3,8 +3,8 @@
 #
 # Bind xml and convert to object
 #
-class Spidy::Binder::Xml < Spidy::Binder::Base
-  def self.let(name, query = nil, &block)
+module Spidy::Binder::Xml
+  def let(name, query = nil, &block)
     @attribute_names ||= []
     @attribute_names << name
 
@@ -20,6 +20,7 @@ class Spidy::Binder::Xml < Spidy::Binder::Base
       fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
-
-  alias_method :xml, :resource
+  def self.extended(obj)
+    obj.alias_method :xml, :resource
+  end
 end

data/lib/spidy/binder.rb

@@ -8,48 +8,4 @@ module Spidy::Binder
   autoload :Json
   autoload :Html
   autoload :Xml
-
-  class Error < StandardError
-  end
-
-  class Caller
-    def initialize(spidy, binder)
-      @spidy = spidy
-      @binder = binder
-    end
-
-    def call(source, url: nil, define: nil, define_name: nil)
-      yield Class.new(@binder, &define).new(define_name, @spidy, source, url)
-    end
-  end
-
-  class Base
-    class << self
-      attr_reader :attribute_names
-    end
-
-    attr_reader :resource, :url
-
-    def initialize(define_name, spidy, resource, url)
-      @define_name = define_name
-      @spidy = spidy
-      @resource = resource
-      @url = url
-    end
-
-    def to_s
-      to_h.to_json
-    end
-
-    def to_h
-      self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-    end
-  end
-
-
-  def self.get(spidy, value)
-    return Caller.new(spidy, const_get(value.to_s.classify)) if name.is_a?(String) || name.is_a?(Symbol)
-
-    value
-  end
 end

data/lib/spidy/command_line.rb

@@ -8,6 +8,10 @@ class Spidy::CommandLine
   class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
   class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
 
+  def eval_call(script)
+    @definition_file.spidy.instance_eval(script)
+  end
+
   def initialize(definition_file)
     @definition_file = definition_file
     raise 'unloaded spidy' if definition_file.spidy.nil?

data/lib/spidy/define_object.rb

@@ -0,0 +1,19 @@
+class Spidy::DefineObject
+  class << self
+    attr_reader :attribute_names
+  end
+  attr_reader :resource, :url
+
+  def initialize(resource, url)
+    @resource = resource
+    @url = url
+  end
+
+  def to_s
+    to_h.to_json
+  end
+
+  def to_h
+    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
+  end
+end

data/lib/spidy/definition.rb

@@ -44,22 +44,20 @@ module Spidy::Definition
     end
   end
 
-  def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
-    @namespace ||= {}
+  def define(name = :default, connector: nil, as: nil, &define_block)
     connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
-    binder = Spidy::Binder.get(self, binder || as)
-    @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
-  end
-
-  private
-
-  def define_proc(name, connector, binder, define_block)
-    proc do |source, &yielder|
-      yielder = lambda { |result| break result } if yielder.nil?
-      connection_yielder = lambda do |page|
-        binder.call(page, url: source, define: define_block, define_name: name) { |object| yielder.call(object) }
+    binder_base = Spidy::Binder.const_get(as.to_s.classify)
+    @namespace ||= {}
+    @namespace[:"#{name}_scraper"] = Class.new(Spidy::DefineObject) do
+      extend binder_base
+      class_eval(&define_block)
+      define_singleton_method(:call) do |source, &yielder|
+        yielder = lambda { |result| break result } if yielder.nil?
+        connection_yielder = lambda do |page|
+          yielder.call(new(page, source))
+        end
+        connector.call(source, &connection_yielder)
       end
-      connector.call(source, &connection_yielder)
     end
   end
 end

data/lib/spidy/shell.rb

@@ -16,5 +16,5 @@ class Spidy::Shell
     Spidy::CommandLine.new(@definition_file)
   end
 
-  delegate :function, :each, :call, to: :command_line
+  delegate :function, :each, :call, :eval_call, to: :command_line
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.3.8'
+  VERSION = '0.3.9'
 end

data/lib/spidy.rb

@@ -19,6 +19,7 @@ module Spidy
   autoload :DefinitionFile
   autoload :Binder
   autoload :Connector
+  autoload :DefineObject
 
   def self.shell(filepath = nil)
     Spidy::Shell.new(filepath)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.9
 platform: ruby
 authors:
 - aileron
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-03-24 00:00:00.000000000 Z
+date: 2021-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -232,6 +232,7 @@ files:
 - lib/spidy/connector/json.rb
 - lib/spidy/connector/xml.rb
 - lib/spidy/console.rb
+- lib/spidy/define_object.rb
 - lib/spidy/definition.rb
 - lib/spidy/definition_file.rb
 - lib/spidy/shell.rb
@@ -258,7 +259,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.13
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: web spider dsl