spidy 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a6e8adabd5eef99735eb1e4ab44609e2f25a859b5604bb2512a8692b72dd0c9e
-  data.tar.gz: ab748cec4971ae35044860ddc26465aaac67d56f47677919f47a3562bd29080a
+  metadata.gz: d88e7028d82156f7710e85af609aa8c703a92b5d48f92af5047b39f64e98b4ad
+  data.tar.gz: 984a44e45d06d72a0e8eaaa3c142f13fe81c5f77b0606b4dc883b695ae024f7a
 SHA512:
-  metadata.gz: 747950afd87f68df3572aeb8859af14087a75bd1786c5183719ce94f7964c397e9b81a58c563f1a54883322620b7aa64d580144ec25e0fe4f85c20463ed21d68
-  data.tar.gz: c2d972491eedbdb5308ceda060c71c95a409f9b4968a5897baf93a137d8f17411ad1afa9bfed7e93b0b5a177a706a358da56dc28a86e7dfe3f65152174b22ce8
+  metadata.gz: e2c6dafc543a85fc7c58ca9d7b3b9dc9fe3823fbe35aad3dfa60b7fece6434e3b0c4fbef6b507bd73ed7e827611a01d065b89df0b9d33e06b203707801eddaee
+  data.tar.gz: 82466dba270ffbec392383edd7afdad91d6c742d3580b733ef489e5199fd5c44956b70bd43bcb64d9e91b161e7aedf18fef96c4965ae655cece11b9ab585ebfc

data/.rubocop.yml

@@ -25,3 +25,6 @@ Metrics/BlockLength:
 
 SignalException:
   EnforcedStyle: semantic
+
+Layout/EmptyLineAfterGuardClause:
+  Enabled: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.0.7)
+    spidy (0.0.8)
       activemodel (~> 5.2)
       activesupport (~> 5.2)
       mechanize

data/README.md

@@ -20,7 +20,44 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+### When used from the command line
+
+website.rb
+```rb
+Spidy.defin do
+  spider(as: :html) do |yielder, connector, url|
+    connector.call(url) do |html|
+      # html as nokogiri object ( mechanize )
+      yielder.call(url)
+    end
+  end
+
+  define(as: :html) do
+    let(:object_name, 'nokogiri query')
+  end
+end
+```
+```bash
+echo 'http://example.com' | spidy each website.rb > urls
+cat urls | spidy call website.rb > website.json
+# shorthands
+echo 'http://example.com' | spidy each website.rb | spidy call website.rb | jq .
+```
+
+### When used from the ruby code
+``
+a = Spidy.define do
+  # Implementing spiders and scrapers
+end
+
+a.each(url) do |url|
+  # Loop for the number of retrieved URLs
+end
+
+a.call(url) do |object|
+  # The scrape result is passed as a defined object
+end
+```
 
 ## Development
 

data/lib/spidy.rb

@@ -29,6 +29,12 @@ module Spidy
   end
 
   def self.define(&block)
-    Class.new(::Spidy::Definition, &block)
+    Module.new do
+      class_eval do
+        extend ::Spidy::Definition
+        module_eval(&block)
+      end
+    end
+    # Module.new(::Spidy::Definition, &block)
   end
 end

data/lib/spidy/connector/html.rb

@@ -4,6 +4,18 @@
 # Mechanize wrapper
 #
 module Spidy::Connector::Html
+  #
+  # retry class
+  #
+  class Retry < StandardError
+    attr_reader :response_code
+    attr_reader :wait_time
+
+    def initialize(wait_time: 2, page: nil, error: nil)
+      @wait_time = wait_time
+      @response_code = error.try(:response_code) || page.try(:response_code)
+    end
+  end
   USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
@@ -13,33 +25,51 @@ module Spidy::Connector::Html
     'Safari/537.36'
   ].join(' ')
 
+  @logger = proc { |values| STDERR.puts(values.map { |k, v| "#{k}:#{v}" }.join("\t")) }
   @agent = Mechanize.new
   @agent.user_agent = USER_AGENT
 
   class << self
     attr_reader :agent
+    attr_accessor :logger
 
     def call(url, encoding: nil, retry_count: 3, &yielder)
       if encoding
-        @agent.default_encoding = encoding
-        @agent.force_default_encoding = true
+        agent.default_encoding = encoding
+        agent.force_default_encoding = true
       end
+      logger.call('connnector.get': url, 'connnector.accessed': Time.current)
       get(url, retry_count, yielder)
     end
 
     private
 
+    # rubocop:disable Metrics/MethodLength
     def get(url, retry_count, yielder)
-      @agent.get(url, &yielder)
+      agent.get(url) do |page|
+        fail Retry, page: page, wait_time: 5 if page.title == 'Sorry, unable to access page...'
+
+        yielder.call(page)
+      end
     rescue Mechanize::ResponseCodeError => e
+      raise Retry, error: e if e.response_code == '429'
+      raise e
+    rescue Retry => e
+      logger.call('retry.accessed': Time.current,
+                  'retry.uri': url,
+                  'retry.response_code': e.response_code,
+                  'retry.rest_count': retry_count)
+
+      @agent = Mechanize.new
+      @agent.user_agent = USER_AGENT
+
       retry_count -= 1
-      case e.response_code
-      when '429'
-        sleep 2
-        retry if retry_count
-      else
-        raise e
+      if retry_count.positive?
+        sleep e.wait_time
+        retry
       end
+      raise e
     end
+    # rubocop:enable Metrics/MethodLength
   end
 end

data/lib/spidy/console.rb

@@ -5,7 +5,7 @@
 #
 class Spidy::Console
   attr_reader :definition_file
-  delegate :namespace, :spiders, to: :definition_file
+  delegate :spidy, to: :definition_file
 
   def initialize(definition_file = nil)
     @definition_file = definition_file
@@ -20,10 +20,10 @@ class Spidy::Console
   end
 
   def call(name = :default, url = nil, &block)
-    namespace[name].call(url, &block)
+    spidy.call(name, url: url, &block)
   end
 
   def each(name = :default, url = nil, &block)
-    spiders[name].call(url, &block)
+    spidy.each(name, url: url, &block)
   end
 end

data/lib/spidy/definition.rb

@@ -3,35 +3,60 @@
 #
 # Class representing a website defined by DSL
 #
-class Spidy::Definition
-  class_attribute :namespace, default: {}
-  class_attribute :spiders, default: {}
-
-  class << self
-    def spider(name = :default, connector: nil, as: nil)
-      connector = Spidy::Connector.get(as || connector) || connector
-      spiders[name] = proc do |url, &yielder|
-        yield(yielder, connector, url)
-      end
-    end
+module Spidy::Definition
+  def call(name = :default, url: nil, stream: nil, err: nil, &output)
+    name = name.presence || :default
+    spidy = @namespace["#{name}_scraper"]
+    fail "undefined spidy [#{name}]" if spidy.nil?
+
+    exec(spidy, url: url, stream: stream, err: err, &output)
+  end
 
-    def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
-      connector = Spidy::Connector.get(as || connector) || connector
-      binder = Spidy::Binder.get(as || binder) || binder
-      namespace[name] = define_proc(connector, binder, define_block)
+  def each(name = :default, url: nil, stream: nil, err: nil, &output)
+    name = name.presence || :default
+    spidy = @namespace["#{name}_spider"]
+    fail "undefined spidy [#{name}]" if spidy.nil?
+
+    exec(spidy, url: url, stream: stream, err: err, &output)
+  end
+
+  def spider(name = :default, connector: nil, as: nil)
+    @namespace ||= {}
+    connector = Spidy::Connector.get(as || connector) || connector
+    @namespace["#{name}_spider"] = proc do |url, &yielder|
+      yield(yielder, connector, url)
     end
+  end
 
-    private
+  def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
+    @namespace ||= {}
+    connector = Spidy::Connector.get(as || connector) || connector
+    binder = Spidy::Binder.get(as || binder) || binder
+    @namespace["#{name}_scraper"] = define_proc(connector, binder, define_block)
+  end
+
+  private
+
+  def exec(spidy, url: nil, stream: nil, err: nil, &output)
+    return spidy.call(url, &output) if stream.nil?
+
+    stream.each do |value|
+      spidy.call(value.strip, &output)
+    rescue StandardError => e
+      raise e if err.nil?
+
+      err.call(e, value.strip)
+    end
+  end
 
-    def define_proc(connector, binder, define_block)
-      proc do |url, &yielder|
-        fail 'invalid argument [Required url / block]' if url.blank? && yielder.nil?
+  def define_proc(connector, binder, define_block)
+    proc do |url, &yielder|
+      fail 'invalid argument [Required url / block]' if url.blank? && yielder.nil?
 
-        connection_yielder = lambda do |resource|
-          binder.call(resource, define_block) { |object| yielder.call(object) }
-        end
-        connector.call(url, &connection_yielder)
+      connection_yielder = lambda do |resource|
+        binder.call(resource, define_block) { |object| yielder.call(object) }
       end
+      connector.call(url, &connection_yielder)
     end
   end
 end

data/lib/spidy/definition_file.rb

@@ -5,14 +5,7 @@
 #
 class Spidy::DefinitionFile
   attr_reader :path
-  attr_reader :definition
-  delegate :namespace, :spiders, to: :definition
-
-  CSV = lambda do |result|
-    ::CSV.generate do |csv|
-      csv << result.definition.attributes_to_array
-    end
-  end
+  attr_reader :spidy
 
   def self.open(filepath)
     object = new(filepath)
@@ -22,7 +15,7 @@ class Spidy::DefinitionFile
 
   # rubocop:disable Security/Eval
   def eval_definition
-    @definition = eval(File.open(path).read)
+    @spidy = eval(File.open(path).read)
   end
   # rubocop:enable Security/Eval
 

data/lib/spidy/shell.rb

@@ -6,20 +6,25 @@ require 'pry'
 # spidy shell interface
 #
 class Spidy::Shell
-  attr_reader :definition_file
-  class_attribute :output_yielder, default: (proc { |result| STDOUT.puts(result.to_s) })
-  delegate :namespace, :spiders, to: :definition_file
+  attr_reader :definition_file, :spidy
+  class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
+  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts("#{url}\n #{e.message}") })
+  delegate :spidy, to: :definition_file
 
   def initialize(definition_file)
     @definition_file = definition_file
   end
 
   def call(name)
-    exec(namespace[name&.to_sym] || namespace.values.first)
+    return spidy.call(name, stream: STDIN, err: error_handler, &output) if FileTest.pipe?(STDIN)
+
+    spidy.call(name, err: error_handler, &output)
   end
 
   def each(name)
-    exec(spiders[name&.to_sym] || spiders.values.first)
+    return spidy.each(name, stream: STDIN, err: error_handler, &output) if FileTest.pipe?(STDIN)
+
+    spidy.each(name, err: error_handler, &output)
   end
 
   def function
@@ -66,17 +71,4 @@ class Spidy::Shell
       RUBY
     end
   end
-
-  private
-
-  def exec(command)
-    fail "undefined commmand[#{name}]" if command.nil?
-    return command.call(&output_yielder) unless FileTest.pipe?(STDIN)
-
-    STDIN.each do |line|
-      command.call(line.strip, &output_yielder)
-    rescue StandardError => e
-      STDERR.puts("#{line.strip}\n #{e.message}")
-    end
-  end
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.0.8'
+  VERSION = '0.0.9'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-20 00:00:00.000000000 Z
+date: 2019-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler