spidy 0.1.5 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9eec9ae1800b7cb73cbb2808ee27f66db2c1615f952e62e87aadc84e58b3b981
-  data.tar.gz: f15145d956caa391f018a67695ea474a8585988f908f05f8999aadd0360a6920
+  metadata.gz: 6d95a2c20a93c9c60969c4a738200391443f2e797273988d7aaa9a7abc812350
+  data.tar.gz: 789cbc87df6997b5fbdb53db4b3fe7c91b6e899b995d232a227a2395c53f67c6
 SHA512:
-  metadata.gz: 72bffd2a6af128cb85eae78d7bd428caa5ce444658fc727de83e910f2c7e9c30ef153200e87a2f15b287e5b4eb3d8327b11134a2588d7b8dfcb1d433065e8a71
-  data.tar.gz: f4fc253d784ac3ca22dd57097c4aaaf0e9c7697704f6888ef7dad0e8230eeafcb9ddd655f5e20b568a2b3f8c0dab9f030f20c5b5e023b59abc9482ee69fb5029
+  metadata.gz: 4e811d34caebb96b1b1d0247ff1a547ed10e5e8a27d82df105c8a809a4b96011e0660805075e8864100d8df29545537c2dde3d3b742e5c35fae33e3a847ad5e9
+  data.tar.gz: fa2f8c4039b009dfc746384dac40028956b880efa96d479485e85b702aad718f4b90e8b44b04912b8aa2bb94d54f27f50570978e2d07078943120bce206e38be

data/.ruby-version

@@ -1 +1 @@
-2.7.0
+2.6.6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.1.3)
+    spidy (0.2.3)
       activesupport
       mechanize
       pry
@@ -9,22 +9,22 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.2.1)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.2)
+      zeitwerk (~> 2.2, >= 2.2.2)
     coderay (1.1.2)
-    concurrent-ruby (1.1.5)
-    connection_pool (2.2.2)
+    concurrent-ruby (1.1.6)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     ffaker (2.10.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    i18n (1.8.2)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
     mechanize (2.7.6)
       domain_name (~> 0.5, >= 0.5.1)
@@ -38,13 +38,14 @@ GEM
     method_source (0.9.2)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
-    minitest (5.14.0)
+    minitest (5.14.1)
+    mixlib-shellout (2.4.4)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (3.1.0)
+    net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
-    nokogiri (1.10.7)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     ntlm-http (0.1.1)
     pry (0.12.2)
@@ -55,23 +56,30 @@ GEM
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
+    rspec-command (1.0.3)
+      mixlib-shellout (~> 2.0)
+      rspec (~> 3.2)
+      rspec-its (~> 1.2)
     rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
     rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
+    rspec-its (1.3.0)
+      rspec-core (>= 3.0.0)
+      rspec-expectations (>= 3.0.0)
     rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.2)
     thread_safe (0.3.6)
-    tzinfo (1.2.6)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
+    unf_ext (0.0.7.7)
     webrobots (0.1.2)
-    zeitwerk (2.2.2)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
@@ -82,7 +90,8 @@ DEPENDENCIES
   pry
   rake (~> 10.0)
   rspec (~> 3.0)
+  rspec-command
   spidy!
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -44,8 +44,23 @@ cat urls | spidy call website.rb > website.json
 echo 'http://example.com' | spidy each website.rb | spidy call website.rb | jq .
 ```
 
+### When development console
+```bash
+spidy console website.rb
+```
+
+### reload source code
+```
+pry(#<Spidy::Console>)> reload!
+```
+
+```rb
+each('http://example.com') { |url| break url }
+call('http://example.com') { |html| break html } # html as nokogiri object ( mechanize )
+```
+
 ### When used from the ruby code
-``
+```rb
 a = Spidy.define do
   # Implementing spiders and scrapers
 end

data/exe/spidy

@@ -7,17 +7,17 @@ require 'pry'
 if ARGV[1].blank?
   case ARGV[0]
   when 'version' then STDOUT.puts(Spidy::VERSION)
-  when 'console' then Spidy.console
+  when 'console' then Spidy.shell.interactive
   else
     STDOUT.puts 'usage: spidy [version console]'
   end
 else
   case ARGV[0]
-  when 'console' then Spidy.console(ARGV[1])
-  when 'shell' then Spidy.shell(ARGV[1]).function
+  when 'console' then Spidy.shell(ARGV[1]).interactive
+  when 'function' then Spidy.shell(ARGV[1]).function
   when 'call' then Spidy.shell(ARGV[1]).call(ARGV[2])
   when 'each' then Spidy.shell(ARGV[1]).each(ARGV[2])
   else
-    STDOUT.puts 'usage: spidy [call each shell] [file]'
+    STDOUT.puts 'usage: spidy [console function call each] [file]'
   end
 end

data/lib/spidy.rb

@@ -11,35 +11,32 @@ require 'open-uri'
 module Spidy
   extend ActiveSupport::Autoload
   autoload :Shell
+  autoload :CommandLine
   autoload :Console
   autoload :Definition
   autoload :DefinitionFile
   autoload :Binder
   autoload :Connector
 
-  def self.console(filepath = nil)
-    require 'pry'
-    if filepath
-      Pry.start(Spidy::Console.new(Spidy::DefinitionFile.open(filepath)))
-    else
-      Pry.start(Spidy::Console.new)
-    end
+  def self.shell(filepath = nil)
+    Spidy::Shell.new(filepath)
   end
 
   def self.open(filepath)
     Spidy::DefinitionFile.open(filepath).spidy
   end
 
-  def self.shell(filepath)
-    Spidy::Shell.new(Spidy::DefinitionFile.open(filepath))
-  end
-
   def self.define(&block)
-    Module.new do
+    spidy = Module.new do
       class_eval do
         extend ::Spidy::Definition
         module_eval(&block)
       end
     end
+    spidy.instance_eval do
+      undef :spider
+      undef :define
+    end
+    spidy
   end
 end

data/lib/spidy/binder.rb

@@ -9,17 +9,48 @@ module Spidy::Binder
   autoload :Html
   autoload :Xml
 
+  class Error < StandardError
+  end
+
   class Caller
     def initialize(spidy, binder)
       @spidy = spidy
       @binder = binder
     end
 
-    def call(source, url: nil, define: nil)
-      yield Class.new(@binder, &define).new(@spidy, source, url)
+    def call(source, url: nil, define: nil, define_name: nil)
+      yield Class.new(@binder, &define).new(define_name, @spidy, source, url)
+    end
+  end
+
+  class Base
+    class << self
+      attr_reader :attribute_names
+    end
+
+    attr_reader :resource, :url
+
+    def initialize(define_name, spidy, resource, url)
+      @define_name = define_name
+      @spidy = spidy
+      @resource = resource
+      @url = url
+    end
+
+    def scraper(name, source)
+      lambda { |&block| @spidy.call(source, name: name, &block) }
+    end
+
+    def to_s
+      to_h.to_json
+    end
+
+    def to_h
+      self.class.attribute_names.map { |name| [name, send(name)] }.to_h
     end
   end
 
+
   def self.get(spidy, value)
     return Caller.new(spidy, const_get(value.to_s.classify)) if name.is_a?(String) || name.is_a?(Symbol)
 

data/lib/spidy/binder/html.rb

@@ -3,42 +3,23 @@
 #
 # Bind html and convert to object
 #
-class Spidy::Binder::Html
-  class << self
-    attr_reader :attribute_names
+class Spidy::Binder::Html < Spidy::Binder::Base
+  def self.let(name, query = nil, &block)
+    @attribute_names ||= []
+    @attribute_names << name
 
-    def let(name, query = nil, &block)
-      @attribute_names ||= []
-      @attribute_names << name
-      define_method(name) do
-        return html.at(query)&.text if block.nil?
-        return instance_exec(&block) if query.blank?
+    return define_method(name) { html.at(query)&.text } if block.nil?
 
+    define_method(name) do
+      if query.present?
         instance_exec(html.at(query), &block)
-      rescue NoMethodError => e
-        raise "#{html.uri} ##{name} => #{e.message}"
+      else
+        instance_exec(&block)
       end
+    rescue StandardError => e
+      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  attr_reader :html, :url
-  alias_method :resource, :html
-
-  def initialize(spidy, html, url)
-    @spidy = spidy
-    @url = url
-    @html = html
-  end
-
-  def scraper(name, source)
-    lambda { |&block| @spidy.call(source, name: name, &block) }
-  end
-
-  def to_s
-    to_h.to_json
-  end
-
-  def to_h
-    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-  end
+  alias_method :html, :resource
 end

data/lib/spidy/binder/json.rb

@@ -3,40 +3,23 @@
 #
 # Bind json and convert to object
 #
-class Spidy::Binder::Json
-  class << self
-    attr_reader :attribute_names
-
-    def let(name, *query, &block)
-      @attribute_names ||= []
-      @attribute_names << name
-      define_method(name) do
-        result = json.dig(*query) if query.present?
-        return result if block.nil?
-
-        instance_exec(result, &block)
+class Spidy::Binder::Json < Spidy::Binder::Base
+  def self.let(name, *query, &block)
+    @attribute_names ||= []
+    @attribute_names << name
+
+    return define_method(name) { json.dig(*query) } if block.nil?
+
+    define_method(name) do
+      if query.present?
+        instance_exec(json.dig(*query), &block)
+      else
+        instance_exec(&block)
       end
+    rescue StandardError => e
+      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  attr_reader :json, :url
-  alias_method :resource, :json
-
-  def initialize(spidy, json, url)
-    @spidy = spidy
-    @json = json
-    @url = url
-  end
-
-  def scraper(name, source)
-    lambda { |&block| @spidy.call(source, name: name, &block) }
-  end
-
-  def to_s
-    to_h.to_json
-  end
-
-  def to_h
-    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-  end
+  alias_method :json, :resource
 end

data/lib/spidy/binder/xml.rb

@@ -3,42 +3,23 @@
 #
 # Bind xml and convert to object
 #
-class Spidy::Binder::Xml
-  class << self
-    attr_reader :attribute_names
+class Spidy::Binder::Xml < Spidy::Binder::Base
+  def self.let(name, query = nil, &block)
+    @attribute_names ||= []
+    @attribute_names << name
 
-    def let(name, query = nil, &block)
-      @attribute_names ||= []
-      @attribute_names << name
-      define_method(name) do
-        return xml.at(query)&.text if block.nil?
-        return instance_exec(&block) if query.blank?
+    return define_method(name) { xml.at(query)&.text } if block.nil?
 
+    define_method(name) do
+      if query.present?
         instance_exec(xml.at(query), &block)
-      rescue NoMethodError => e
-        raise "#{xml} ##{name} => #{e.message}"
+      else
+        instance_exec(&block)
       end
+    rescue StandardError => e
+      fail Spidy::Binder::Error, "spidy(#{@define_name})##{name} => #{e.message}"
     end
   end
 
-  attr_reader :xml, :url
-  alias_method :resource, :xml
-
-  def initialize(spidy, xml, url)
-    @spidy = spidy
-    @xml = xml
-    @url = url
-  end
-
-  def scraper(name, source)
-    lambda { |&block| @spidy.call(source, name: name, &block) }
-  end
-
-  def to_s
-    to_h.to_json
-  end
-
-  def to_h
-    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
-  end
+  alias_method :xml, :resource
 end

data/lib/spidy/command_line.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+#
+# spidy shell interface
+#
+class Spidy::CommandLine
+  delegate :spidy, to: :@definition_file
+  class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
+  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
+
+  def initialize(definition_file)
+    @definition_file = definition_file
+    raise 'unloaded spidy' if definition_file.spidy.nil?
+  end
+
+  def each_stdin_lines(name)
+    STDIN.each_line do |url|
+      begin
+        spidy.each(url.strip, name: name, &output)
+      rescue => e
+        error_handler.call(e, url)
+      end
+    end
+  end
+
+  def call_stdin_lines(name)
+    STDIN.each_line do |url|
+      begin
+        spidy.call(url.strip, name: name, &output)
+      rescue => e
+        error_handler.call(e, url)
+      end
+    end
+  end
+
+  def call(name)
+    return call_stdin_lines(name) if FileTest.pipe?(STDIN)
+    spidy.call(name: name, &output) unless FileTest.pipe?(STDIN)
+  rescue => e
+    error_handler.call(e, nil)
+  end
+
+  def each(name)
+    return each_stdin_lines(name) if FileTest.pipe?(STDIN)
+    spidy.each(name: name, &output)
+  rescue => e
+    error_handler.call(e, nil)
+  end
+
+  def function
+    print <<~SHELL
+      function spider() {
+        spidy spider #{definition_file.path} $1
+      }
+      function scraper() {
+        spidy call #{definition_file.path} $1
+      }
+    SHELL
+  end
+
+  def build(name)
+    build_shell(name)
+    build_ruby(name)
+  end
+
+  def build_shell(name)
+    File.open("#{name}.sh", 'w') do |f|
+      f.write <<~SHELL
+        #!/bin/bash
+        eval "$(spidy $(dirname "${0}")/#{name}.rb shell)"
+        spider example
+      SHELL
+    end
+  end
+
+  def build_ruby(name)
+    File.open("#{name}.rb", 'w') do |f|
+      f.write <<~RUBY
+        # frozen_string_literal: true
+
+        Spidy.define do
+          spider(:example) do |yielder, connector|
+            # connector.call(url) do |resource|
+            #   yielder.call(url or resource)
+            # end
+          end
+
+          define(:example) do
+          end
+        end
+      RUBY
+    end
+  end
+end

data/lib/spidy/connector/html.rb

@@ -68,6 +68,7 @@ module Spidy::Connector::Html
       result
     rescue Mechanize::ResponseCodeError => e
       raise Retry, error: e if e.response_code == '429'
+      raise Retry, error: e if e.response_code == '502'
       raise e
     end
 

data/lib/spidy/console.rb

@@ -4,11 +4,10 @@
 # spidy console
 #
 class Spidy::Console
-  attr_reader :definition_file
-  delegate :spidy, to: :definition_file
-  delegate :call, :each, to: :spidy
+  delegate :spidy, to: :@definition_file
+  delegate :call, :each, :namespace, allow_nil: true, to: :spidy
 
-  def initialize(definition_file = nil)
+  def initialize(definition_file)
     @definition_file = definition_file
   end
 
@@ -17,6 +16,6 @@ class Spidy::Console
   end
 
   def reload!
-    @definition_file&.eval_definition
+    @definition_file.eval_definition
   end
 end

data/lib/spidy/definition.rb

@@ -36,17 +36,16 @@ module Spidy::Definition
     @namespace ||= {}
     connector = Spidy::Connector.get(connector || as)
     binder = Spidy::Binder.get(self, binder || as)
-    @namespace[:"#{name}_scraper"] = define_proc(connector, binder, define_block)
+    @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
   end
 
   private
 
-  def define_proc(connector, binder, define_block)
+  def define_proc(name, connector, binder, define_block)
     proc do |source, &yielder|
-      fail 'block is not specified' if yielder.nil?
-
+      yielder = lambda { |result| break result } if yielder.nil?
       connection_yielder = lambda do |page|
-        binder.call(page, url: source, define: define_block) { |object| yielder.call(object) }
+        binder.call(page, url: source, define: define_block, define_name: name) { |object| yielder.call(object) }
       end
       connector.call(source, &connection_yielder)
     end

data/lib/spidy/definition_file.rb

@@ -15,7 +15,7 @@ class Spidy::DefinitionFile
 
   # rubocop:disable Security/Eval
   def eval_definition
-    @spidy = eval(File.open(path).read)
+    @spidy = eval(File.open(path).read) if path
   end
   # rubocop:enable Security/Eval
 

data/lib/spidy/shell.rb

@@ -1,96 +1,20 @@
 # frozen_string_literal: true
 
-require 'pry'
-
 #
-# spidy shell interface
+# spidy Shell
 #
 class Spidy::Shell
-  attr_reader :definition_file, :spidy
-  class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
-  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
-  delegate :spidy, to: :definition_file
-
-  def initialize(definition_file)
-    @definition_file = definition_file
-  end
-
-  def each_stdin_lines(name)
-    STDIN.each_line do |url|
-      begin
-        spidy.each(url.strip, name: name, &output)
-      rescue => e
-        error_handler.call(e, url)
-      end
-    end
-  end
-
-  def call_stdin_lines(name)
-    STDIN.each_line do |url|
-      begin
-        spidy.call(url.strip, name: name, &output)
-      rescue => e
-        error_handler.call(e, url)
-      end
-    end
-  end
-
-  def call(name)
-    return call_stdin_lines(name) if FileTest.pipe?(STDIN)
-    spidy.call(name: name, &output) unless FileTest.pipe?(STDIN)
-  rescue => e
-    error_handler.call(e, nil)
+  def initialize(path)
+    @definition_file = Spidy::DefinitionFile.open(path)
   end
 
-  def each(name)
-    return each_stdin_lines(name) if FileTest.pipe?(STDIN)
-    spidy.each(name: name, &output)
-  rescue => e
-    error_handler.call(e, nil)
+  def interactive
+    Pry.start(Spidy::Console.new(@definition_file))
   end
 
-  def function
-    print <<~SHELL
-      function spider() {
-        spidy spider #{definition_file.path} $1
-      }
-      function scraper() {
-        spidy call #{definition_file.path} $1
-      }
-    SHELL
+  def command_line
+    Spidy::CommandLine.new(@definition_file)
   end
 
-  def build(name)
-    build_shell(name)
-    build_ruby(name)
-  end
-
-  def build_shell(name)
-    File.open("#{name}.sh", 'w') do |f|
-      f.write <<~SHELL
-        #!/bin/bash
-        eval "$(spidy $(dirname "${0}")/#{name}.rb shell)"
-        spider example
-      SHELL
-    end
-  end
-
-  def build_ruby(name)
-    File.open("#{name}.rb", 'w') do |f|
-      f.write <<~RUBY
-        # frozen_string_literal: true
-
-        Spidy.define do
-          spider(:example) do |yielder, connector|
-            # connector.call(url) do |resource|
-            #   yielder.call(url or resource)
-            # end
-          end
-
-          define(:example) do
-          end
-        end
-      RUBY
-    end
-  end
+  delegate :function, :each, :call, to: :command_line
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.1.5'
+  VERSION = '0.2.4'
 end

data/spidy.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'ffaker'
+  spec.add_development_dependency 'rspec-command'
 
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.4
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-04 00:00:00.000000000 Z
+date: 2020-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-command
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -151,6 +165,7 @@ files:
 - lib/spidy/binder/html.rb
 - lib/spidy/binder/json.rb
 - lib/spidy/binder/xml.rb
+- lib/spidy/command_line.rb
 - lib/spidy/connector.rb
 - lib/spidy/connector/direct.rb
 - lib/spidy/connector/html.rb
@@ -183,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: web spider dsl