spidy 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8dd72483f6ea6a5d0d0711803df81ed67d43edc05297db6815d40f81848bbfa6
-  data.tar.gz: 8f4576bcb16dd473802eade5a5d7732745b77810dad387de359106e5f30b2da4
+  metadata.gz: 41b1f40d3ae33eeac273d40a57b956be37971bbb940280c1ca5c9faf0371fbeb
+  data.tar.gz: e55d03b8703142255caf2eb74b927f4dcfdc58b02a96bb51fcb2f1fa0e9e4167
 SHA512:
-  metadata.gz: e99a33f48a8e5ba51e1a6956cfd2bd5061b21ac6e46b79343049cf5c109fca30fa60252f5d9ca835f28bec58d3f5a1c361aaa0905cc5ded68c335602ca6c16d9
-  data.tar.gz: bfc083af4cbbef9c44546ebb8810169c776308978f52468443a7c5cd159aca2975d498f94b5da8496573fb9e437b4587acdd02d293a277e4e7b7e03588563f06
+  metadata.gz: 9577e5315da4a33a4becb1a9c3d41788729fa5dcc8d1a8f1fe774a36a9c34503c7d4f35ee9c20efb99148ac8fe59d9ed69c47e8da20d4590a153b743cc1ea7d4
+  data.tar.gz: ff454254636e05f6df17d8b35d591ad32ffb6de66bd654a2ed34006eb73a87fa9e948539a0b954ae8bf82d30860966066f3947bc34559e783f617c90fa187715

data/exe/spidy

@@ -2,22 +2,23 @@
 # frozen_string_literal: true
 
 require 'spidy'
+require 'pry'
 
-if ARGV[0]&.to_sym == :console
-  if ARGV[1].blank?
-    Spidy.console
+if ARGV[1].blank?
+  case ARGV[0]
+  when 'version' then STDOUT.puts(Spidy::VERSION)
+  when 'console' then Spidy.console
   else
-    Spidy.open(ARGV[1]).console
+    STDOUT.puts 'usage: spidy [version console]'
   end
-  return
-end
-
-shell = Spidy.open(ARGV[1]).shell
-
-case ARGV[0]&.to_sym
-when :shell then shell.function
-when :call then shell.call(ARGV[2])
-when :each then shell.each(ARGV[2])
 else
-  fail 'usage: spidy [call shell new console] [file]'
+  spidy = Spidy.open(ARGV[1])
+  case ARGV[0]
+  when 'console' then spidy.console
+  when 'shell' then spidy.shell.function
+  when 'call' then spidy.shell.call(ARGV[2])
+  when 'each' then spidy.shell.each(ARGV[2])
+  else
+    STDOUT.puts 'usage: spidy [call each shell] [file]'
+  end
 end

data/lib/spidy/connector/html.rb

@@ -13,13 +13,28 @@ module Spidy::Connector::Html
     'Safari/537.36'
   ].join(' ')
 
-  def self.call(url, encoding: nil, &yielder)
-    agent = Mechanize.new
-    if encoding
-      agent.default_encoding = encoding
-      agent.force_default_encoding = true
+  @agent = Mechanize.new
+  @agent.user_agent = USER_AGENT
+
+  class << self
+    def call(url, encoding: nil, &yielder)
+      if encoding
+        @agent.default_encoding = encoding
+        @agent.force_default_encoding = true
+      end
+      get(url, yielder)
+    end
+
+    def get(url, yielder)
+      @agent.get(url, &yielder)
+    rescue Mechanize::ResponseCodeError => e
+      case e.response_code
+      when '429'
+        sleep 2
+        @agent.get(url, &yielder)
+      else
+        raise e
+      end
     end
-    agent.user_agent = USER_AGENT
-    agent.get(url, &yielder)
   end
 end

data/lib/spidy/definition.rb

@@ -8,23 +8,29 @@ class Spidy::Definition
   class_attribute :spiders, default: {}
 
   class << self
-    def define(name, connector: nil, binder: nil, as: nil, &define_block)
+    def spider(name = :default, connector: nil, as: nil)
       connector = Spidy::Connector.get(as || connector) || connector
-      binder = Spidy::Binder.get(as || binder) || binder
-      namespace[name] = proc do |url, &yielder|
-        connection_yielder = lambda do |resource|
-          binder.call(resource, define_block) do |object|
-            yielder.call(object)
-          end
-        end
-        connector.call(url, &connection_yielder)
+      spiders[name] = proc do |url, &yielder|
+        yield(yielder, connector, url)
       end
     end
 
-    def spider(name, connector: nil, as: nil)
+    def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
       connector = Spidy::Connector.get(as || connector) || connector
-      spiders[name] = proc do |url, &yielder|
-        yield(yielder, connector, url)
+      binder = Spidy::Binder.get(as || binder) || binder
+      namespace[name] = define_proc(connector, binder, define_block)
+    end
+
+    private
+
+    def define_proc(connector, binder, define_block)
+      proc do |url, &yielder|
+        fail 'invalid argument [Required url / block]' if url.blank? || yielder.nil?
+
+        connection_yielder = lambda do |resource|
+          binder.call(resource, define_block) { |object| yielder.call(object) }
+        end
+        connector.call(url, &connection_yielder)
       end
     end
   end

data/lib/spidy/shell.rb

@@ -76,7 +76,7 @@ class Spidy::Shell
     STDIN.each do |line|
       command.call(line.strip, &output_yielder)
     rescue StandardError => e
-      STDERR.puts("#{line.strip} => \n #{e.message}")
+      STDERR.puts("#{line.strip}\n #{e.message}")
     end
   end
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-03 00:00:00.000000000 Z
+date: 2019-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler