scruber 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6aecb299dd5c70bfe864f65d2faed704cdc3cde0
-  data.tar.gz: f0697ca03b0f1552e03fb1ee70d0265d5b8bed04
+  metadata.gz: 8cc15677a33ddfea6a5a02bc0051d16cffea104f
+  data.tar.gz: 858c94d1eab0bd11688f719f9a012ef93bf198e8
 SHA512:
-  metadata.gz: 2f52170ba8af4152b88e694537fa9e55156fa6d082c1e9ec409ae46808dc9ca28ff47ed241e8072c0edbba13b3e0be9f00504aef4542da52c3a280c282d8df71
-  data.tar.gz: 580d4c3840526dba3e62826433f8dddd09cdc42f569a2db05beb69b3428bfbd0fea92dc2faa0367175870327605f27f960e04be3d65d5c2dbd3c44c61719252d
+  metadata.gz: 410eb7d1f17efb474c9bbe5d39df02df4f52ef21994aebd8f4678ad86ef867da8a532cc1c96dc7803a531ea89282fcc5c59055323de74a6ea5f2719f6fe32154
+  data.tar.gz: 4e3f22d81e1467012a7f60d124816b5a0a7cd2e76fe5fe48aa477ccace7bf2c6d5a2079f5a6ddcde2b8fd061c0cf7bf6225cb9cf0b1c49d8a86e2d4a51d9dbdb

data/.gitignore

@@ -13,3 +13,4 @@ todo
 
 # rspec failure tracking
 .rspec_status
+*.gem

data/lib/scruber/app_searcher.rb

@@ -24,6 +24,13 @@ module Scruber
       end
     end
 
+    def find_scraper(name, app_path)
+      [
+        File.expand_path('../../scrapers/'+name+'.rb', app_path),
+        File.expand_path('../../scrapers/'+name, app_path),
+      ].find{|f| File.exists?(f) }
+    end
+
     def find_executable
       EXECUTABLES.find { |exe| File.file?(exe) }
     end

data/lib/scruber/cli/templates/scrapers/sample.tt

@@ -1,4 +1,4 @@
-Scruber.run do
+Scruber.run :sample do
   queue.add "http://example.com"
   
   parser :seed do |page|

data/lib/scruber/cli.rb

@@ -17,14 +17,15 @@ module Scruber
       def start(name)
         if defined?(APP_PATH)
           # raise ::Thor::Error, "ERROR: Scruber project not found." unless File.exist?(File.expand_path('config/application', Dir.pwd))
-          raise ::Thor::Error, "ERROR: Scraper not found." unless File.exist?(File.expand_path('../../scrapers/'+name+'.rb', APP_PATH))
+          scraper_path = Scruber::AppSearcher.find_scraper(name, APP_PATH)
+          raise ::Thor::Error, "ERROR: Scraper not found." if scraper_path.nil?
           say "booting..."
           require APP_PATH
           Dir[File.expand_path('../initializers/*.rb', APP_PATH)].sort.each do |i|
             require i
           end
           say "starting #{name}"
-          require File.expand_path('../../scrapers/'+name+'.rb', APP_PATH)
+          require scraper_path
         else
           Scruber::AppSearcher.exec_app(name)
         end

data/lib/scruber/core/crawler.rb

@@ -1,14 +1,15 @@
 module Scruber
   module Core
     class Crawler
-      attr_reader :queue, :fetcher
+      attr_reader :queue, :fetcher, :scraper_name
 
-      def initialize(options={})
+      def initialize(scraper_name, options={})
+        @scraper_name = scraper_name
         Scruber.configuration.merge_options(options)
         @callbacks_options = {}
         @callbacks = {}
         @on_complete_callbacks = {}
-        @queue = Scruber::Queue.new
+        @queue = Scruber::Queue.new(scraper_name: scraper_name)
         @fetcher = Scruber::Fetcher.new
         load_extenstions
       end
@@ -39,8 +40,8 @@ module Scruber
 
       def method_missing(method_sym, *arguments, &block)
         Scruber::Core::Crawler._registered_method_missings.find do |(pattern, func)|
-          if method_sym.to_s =~ pattern
-            instance_exec method_sym, arguments, &(func)
+          if (scan_results = method_sym.to_s.scan(pattern)).present?
+            instance_exec method_sym, scan_results, arguments, &(func)
             true
           else
             false

data/lib/scruber/core/extensions/csv_output.rb

@@ -17,8 +17,8 @@ module Scruber
           end
 
           def self.included(base)
-            Scruber::Core::Crawler.register_method_missing /\Acsv_(\w+)_file\Z/ do |meth, args|
-              file_id = meth.to_s.scan(/\Acsv_(\w+)_file\Z/).first.first.to_sym
+            Scruber::Core::Crawler.register_method_missing /\Acsv_(\w+)_file\Z/ do |meth, scan_results, args|
+              file_id = scan_results.first.first.to_sym
               path, options = args
               options = {} if options.nil?
               csv_file path, options.merge({file_id: file_id})

data/lib/scruber/queue_adapters/abstract_adapter.rb

@@ -28,6 +28,8 @@ module Scruber
         def initialize(queue, url, options={})
           @queue = queue
           @url = url
+
+          options = options.with_indifferent_access
           @method = options.fetch(:method) { :get }
           @user_agent = options.fetch(:user_agent) { nil }
           @post_body = options.fetch(:post_body) { nil }
@@ -90,6 +92,10 @@ module Scruber
           instance_variable_get("@#{k.to_s}")
         end
 
+        def delete
+          raise NotImplementedError
+        end
+
       end
 
       def initialize(options={})

data/lib/scruber/queue_adapters/memory.rb

@@ -7,12 +7,16 @@ module Scruber
         def save
           if self.fetched_at > 0
             @queue.add_downloaded self
-          elsif self.retry_count >= self.max_retry_times.to_i
+          elsif self.max_retry_times && self.retry_count >= self.max_retry_times.to_i
             @queue.add_error_page self
           else
             @queue.push self
           end
         end
+
+        def delete
+          @queue.delete self
+        end
       end
 
       def initialize(options={})
@@ -63,6 +67,12 @@ module Scruber
         @queue.count > 0 || @downloaded_pages.count > 0
       end
 
+      def delete(page)
+        @queue -= [page]
+        @downloaded_pages -= [page]
+        @error_pages -= [page]
+      end
+
     end
   end
 end

data/lib/scruber/version.rb

@@ -1,3 +1,3 @@
 module Scruber
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/scruber.rb

@@ -52,10 +52,10 @@ module Scruber
   class << self
     attr_writer :configuration
 
-    def run(options={}, &block)
+    def run(scraper_name, options={}, &block)
       raise "You need a block to build!" unless block_given?
 
-      Core::Crawler.new(options).run(&block)
+      Core::Crawler.new(scraper_name, options).run(&block)
     end
 
     def configuration

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scruber
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Ivan Goncharov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-06 00:00:00.000000000 Z
+date: 2018-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus