spidy 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9eec9ae1800b7cb73cbb2808ee27f66db2c1615f952e62e87aadc84e58b3b981
-  data.tar.gz: f15145d956caa391f018a67695ea474a8585988f908f05f8999aadd0360a6920
+  metadata.gz: 2ba417984708d3441c47f69574850c379e4a26dbcf810734545449173e1b886b
+  data.tar.gz: '0881f6202ffbcc8c1a7e153e79b16d7dec68694dbb915430cc3c0a8912e29817'
 SHA512:
-  metadata.gz: 72bffd2a6af128cb85eae78d7bd428caa5ce444658fc727de83e910f2c7e9c30ef153200e87a2f15b287e5b4eb3d8327b11134a2588d7b8dfcb1d433065e8a71
-  data.tar.gz: f4fc253d784ac3ca22dd57097c4aaaf0e9c7697704f6888ef7dad0e8230eeafcb9ddd655f5e20b568a2b3f8c0dab9f030f20c5b5e023b59abc9482ee69fb5029
+  metadata.gz: 18ce16b8d0d5d3b536ccba74c842063c7c86d96fa9455010e58c78de20ed37e92051909a6054b77e7022c2c3072f9402fc246daf530c00287e22adbde2d598cf
+  data.tar.gz: e15ee2bec15d64dce143bad5eb0e59f177e32f4aeafed1626a288eaff690c028de7446e40a88bd16696b381ca08e912c959f5597eea11aeede78677ef3c9ed9e

data/.ruby-version

@@ -1 +1 @@
-2.7.0
+2.6.5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.1.3)
+    spidy (0.1.6)
       activesupport
       mechanize
       pry
@@ -41,6 +41,7 @@ GEM
     mime-types-data (3.2019.1009)
     mini_portile2 (2.4.0)
     minitest (5.14.0)
+    mixlib-shellout (2.4.4)
     net-http-digest_auth (1.4.1)
     net-http-persistent (3.1.0)
       connection_pool (~> 2.2)
@@ -55,11 +56,18 @@ GEM
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
+    rspec-command (1.0.3)
+      mixlib-shellout (~> 2.0)
+      rspec (~> 3.2)
+      rspec-its (~> 1.2)
     rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
     rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
+    rspec-its (1.3.0)
+      rspec-core (>= 3.0.0)
+      rspec-expectations (>= 3.0.0)
     rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
@@ -82,6 +90,7 @@ DEPENDENCIES
   pry
   rake (~> 10.0)
   rspec (~> 3.0)
+  rspec-command
   spidy!
 
 BUNDLED WITH

data/README.md

@@ -44,6 +44,21 @@ cat urls | spidy call website.rb > website.json
 echo 'http://example.com' | spidy each website.rb | spidy call website.rb | jq .
 ```
 
+### When development console
+```bash
+spidy console website.rb
+```
+
+### reload source code
+```
+pry(#<Spidy::Console>)> reload!
+```
+
+```rb
+each('http://example.com') { |url| break url }
+call('http://example.com') { |html| break html } # html as nokogiri object ( mechanize )
+```
+
 ### When used from the ruby code
 ``
 a = Spidy.define do

data/exe/spidy

@@ -7,17 +7,17 @@ require 'pry'
 if ARGV[1].blank?
   case ARGV[0]
   when 'version' then STDOUT.puts(Spidy::VERSION)
-  when 'console' then Spidy.console
+  when 'console' then Spidy.shell.interactive
   else
     STDOUT.puts 'usage: spidy [version console]'
   end
 else
   case ARGV[0]
-  when 'console' then Spidy.console(ARGV[1])
-  when 'shell' then Spidy.shell(ARGV[1]).function
+  when 'console' then Spidy.shell(ARGV[1]).interactive
+  when 'function' then Spidy.shell(ARGV[1]).function
   when 'call' then Spidy.shell(ARGV[1]).call(ARGV[2])
   when 'each' then Spidy.shell(ARGV[1]).each(ARGV[2])
   else
-    STDOUT.puts 'usage: spidy [call each shell] [file]'
+    STDOUT.puts 'usage: spidy [console function call each] [file]'
   end
 end

data/lib/spidy.rb

@@ -10,36 +10,30 @@ require 'open-uri'
 #
 module Spidy
   extend ActiveSupport::Autoload
+  autoload :Interface
   autoload :Shell
+  autoload :CommandLine
   autoload :Console
   autoload :Definition
   autoload :DefinitionFile
   autoload :Binder
   autoload :Connector
 
-  def self.console(filepath = nil)
-    require 'pry'
-    if filepath
-      Pry.start(Spidy::Console.new(Spidy::DefinitionFile.open(filepath)))
-    else
-      Pry.start(Spidy::Console.new)
-    end
+  def self.shell(filepath = nil)
+    Spidy::Shell.new(filepath)
   end
 
   def self.open(filepath)
     Spidy::DefinitionFile.open(filepath).spidy
   end
 
-  def self.shell(filepath)
-    Spidy::Shell.new(Spidy::DefinitionFile.open(filepath))
-  end
-
   def self.define(&block)
-    Module.new do
+    spidy = Module.new do
       class_eval do
         extend ::Spidy::Definition
         module_eval(&block)
       end
     end
+    Spidy::Interface.new(spidy)
   end
 end

data/lib/spidy/command_line.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+#
+# spidy shell interface
+#
+class Spidy::CommandLine
+  delegate :spidy, to: :@definition_file
+  class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
+  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
+
+  def initialize(definition_file)
+    @definition_file = definition_file
+    raise 'unloaded spidy' if definition_file.spidy.nil?
+  end
+
+  def each_stdin_lines(name)
+    STDIN.each_line do |url|
+      begin
+        spidy.each(url.strip, name: name, &output)
+      rescue => e
+        error_handler.call(e, url)
+      end
+    end
+  end
+
+  def call_stdin_lines(name)
+    STDIN.each_line do |url|
+      begin
+        spidy.call(url.strip, name: name, &output)
+      rescue => e
+        error_handler.call(e, url)
+      end
+    end
+  end
+
+  def call(name)
+    return call_stdin_lines(name) if FileTest.pipe?(STDIN)
+    spidy.call(name: name, &output) unless FileTest.pipe?(STDIN)
+  rescue => e
+    error_handler.call(e, nil)
+  end
+
+  def each(name)
+    return each_stdin_lines(name) if FileTest.pipe?(STDIN)
+    spidy.each(name: name, &output)
+  rescue => e
+    error_handler.call(e, nil)
+  end
+
+  def function
+    print <<~SHELL
+      function spider() {
+        spidy spider #{definition_file.path} $1
+      }
+      function scraper() {
+        spidy call #{definition_file.path} $1
+      }
+    SHELL
+  end
+
+  def build(name)
+    build_shell(name)
+    build_ruby(name)
+  end
+
+  def build_shell(name)
+    File.open("#{name}.sh", 'w') do |f|
+      f.write <<~SHELL
+        #!/bin/bash
+        eval "$(spidy $(dirname "${0}")/#{name}.rb shell)"
+        spider example
+      SHELL
+    end
+  end
+
+  def build_ruby(name)
+    File.open("#{name}.rb", 'w') do |f|
+      f.write <<~RUBY
+        # frozen_string_literal: true
+
+        Spidy.define do
+          spider(:example) do |yielder, connector|
+            # connector.call(url) do |resource|
+            #   yielder.call(url or resource)
+            # end
+          end
+
+          define(:example) do
+          end
+        end
+      RUBY
+    end
+  end
+end

data/lib/spidy/console.rb

@@ -4,11 +4,10 @@
 # spidy console
 #
 class Spidy::Console
-  attr_reader :definition_file
-  delegate :spidy, to: :definition_file
-  delegate :call, :each, to: :spidy
+  delegate :spidy, to: :@definition_file
+  delegate :call, :each, :namespace, allow_nil: true, to: :spidy
 
-  def initialize(definition_file = nil)
+  def initialize(definition_file)
     @definition_file = definition_file
   end
 
@@ -17,6 +16,6 @@ class Spidy::Console
   end
 
   def reload!
-    @definition_file&.eval_definition
+    @definition_file.eval_definition
   end
 end

data/lib/spidy/definition_file.rb

@@ -15,7 +15,7 @@ class Spidy::DefinitionFile
 
   # rubocop:disable Security/Eval
   def eval_definition
-    @spidy = eval(File.open(path).read)
+    @spidy = eval(File.open(path).read) if path
   end
   # rubocop:enable Security/Eval
 

data/lib/spidy/interface.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+#
+# spidy interface
+#
+class Spidy::Interface
+  delegate :call, :each, :namespace, to: :@spidy
+
+  def initialize(spidy)
+    @spidy = spidy
+  end
+end

data/lib/spidy/shell.rb

@@ -1,96 +1,20 @@
 # frozen_string_literal: true
 
-require 'pry'
-
 #
-# spidy shell interface
+# spidy Shell
 #
 class Spidy::Shell
-  attr_reader :definition_file, :spidy
-  class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
-  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
-  delegate :spidy, to: :definition_file
-
-  def initialize(definition_file)
-    @definition_file = definition_file
-  end
-
-  def each_stdin_lines(name)
-    STDIN.each_line do |url|
-      begin
-        spidy.each(url.strip, name: name, &output)
-      rescue => e
-        error_handler.call(e, url)
-      end
-    end
-  end
-
-  def call_stdin_lines(name)
-    STDIN.each_line do |url|
-      begin
-        spidy.call(url.strip, name: name, &output)
-      rescue => e
-        error_handler.call(e, url)
-      end
-    end
-  end
-
-  def call(name)
-    return call_stdin_lines(name) if FileTest.pipe?(STDIN)
-    spidy.call(name: name, &output) unless FileTest.pipe?(STDIN)
-  rescue => e
-    error_handler.call(e, nil)
+  def initialize(path)
+    @definition_file = Spidy::DefinitionFile.open(path)
   end
 
-  def each(name)
-    return each_stdin_lines(name) if FileTest.pipe?(STDIN)
-    spidy.each(name: name, &output)
-  rescue => e
-    error_handler.call(e, nil)
+  def interactive
+    Pry.start(Spidy::Console.new(@definition_file))
   end
 
-  def function
-    print <<~SHELL
-      function spider() {
-        spidy spider #{definition_file.path} $1
-      }
-      function scraper() {
-        spidy call #{definition_file.path} $1
-      }
-    SHELL
+  def command_line
+    Spidy::CommandLine.new(@definition_file)
   end
 
-  def build(name)
-    build_shell(name)
-    build_ruby(name)
-  end
-
-  def build_shell(name)
-    File.open("#{name}.sh", 'w') do |f|
-      f.write <<~SHELL
-        #!/bin/bash
-        eval "$(spidy $(dirname "${0}")/#{name}.rb shell)"
-        spider example
-      SHELL
-    end
-  end
-
-  def build_ruby(name)
-    File.open("#{name}.rb", 'w') do |f|
-      f.write <<~RUBY
-        # frozen_string_literal: true
-
-        Spidy.define do
-          spider(:example) do |yielder, connector|
-            # connector.call(url) do |resource|
-            #   yielder.call(url or resource)
-            # end
-          end
-
-          define(:example) do
-          end
-        end
-      RUBY
-    end
-  end
+  delegate :function, :each, :call, to: :command_line
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.1.5'
+  VERSION = '0.2.0'
 end

data/spidy.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'ffaker'
+  spec.add_development_dependency 'rspec-command'
 
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-04 00:00:00.000000000 Z
+date: 2020-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-command
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -151,6 +165,7 @@ files:
 - lib/spidy/binder/html.rb
 - lib/spidy/binder/json.rb
 - lib/spidy/binder/xml.rb
+- lib/spidy/command_line.rb
 - lib/spidy/connector.rb
 - lib/spidy/connector/direct.rb
 - lib/spidy/connector/html.rb
@@ -159,6 +174,7 @@ files:
 - lib/spidy/console.rb
 - lib/spidy/definition.rb
 - lib/spidy/definition_file.rb
+- lib/spidy/interface.rb
 - lib/spidy/shell.rb
 - lib/spidy/spider.rb
 - lib/spidy/version.rb
@@ -183,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: web spider dsl