scruber 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8cc15677a33ddfea6a5a02bc0051d16cffea104f
-  data.tar.gz: 858c94d1eab0bd11688f719f9a012ef93bf198e8
+  metadata.gz: 937243b2503853e755c95800e0d34ae5ecb939a7
+  data.tar.gz: 1e2472ff1d1487da8df94292daa57a27d5b52c5a
 SHA512:
-  metadata.gz: 410eb7d1f17efb474c9bbe5d39df02df4f52ef21994aebd8f4678ad86ef867da8a532cc1c96dc7803a531ea89282fcc5c59055323de74a6ea5f2719f6fe32154
-  data.tar.gz: 4e3f22d81e1467012a7f60d124816b5a0a7cd2e76fe5fe48aa477ccace7bf2c6d5a2079f5a6ddcde2b8fd061c0cf7bf6225cb9cf0b1c49d8a86e2d4a51d9dbdb
+  metadata.gz: 2a30b80a4301ca18b87e1c91217ebdfd881e13c511e643ce198e7e9f28984d271b0133b11a065b6d971690fca49b948b289147884eed7c01ee5bfc985d35cc69
+  data.tar.gz: 77201fa5bbde27dfac6f7205711b05a3af9f85ab0ad705c6d08af5ea6b55f2c18c37a4e80878e3e7de7280ad945d20aca79aedc179087036f7bf176ecf05f597

data/exe/scruber

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
 require "scruber/cli"
+require "scruber/app_searcher"
 
-Scruber::CLI::Root.start(ARGV)
+Scruber::CLI::Root.start(ARGV) unless Scruber::AppSearcher.exec_app

data/lib/scruber.rb

@@ -23,6 +23,8 @@ require "scruber/core/page_format/html"
 require "scruber/core/extensions/base"
 require "scruber/core/extensions/loop"
 require "scruber/core/extensions/csv_output"
+require "scruber/core/extensions/queue_aliases"
+require "scruber/core/extensions/parser_aliases"
 
 # require "scruber/core/configuration"
 # require "scruber/core/configuration"
@@ -52,10 +54,10 @@ module Scruber
   class << self
     attr_writer :configuration
 
-    def run(scraper_name, options={}, &block)
+    def run(*args, &block)
       raise "You need a block to build!" unless block_given?
 
-      Core::Crawler.new(scraper_name, options).run(&block)
+      Core::Crawler.new(*args).run(&block)
     end
 
     def configuration

data/lib/scruber/app_searcher.rb

@@ -6,7 +6,7 @@ module Scruber
     RUBY = Gem.ruby
     EXECUTABLES = ["bin/scruber"]
 
-    def exec_app(name)
+    def exec_app
       original_cwd = Dir.pwd
 
       loop do
@@ -22,6 +22,7 @@ module Scruber
         # Otherwise keep moving upwards in search of an executable.
         Dir.chdir("..")
       end
+      true
     end
 
     def find_scraper(name, app_path)

data/lib/scruber/cli.rb

@@ -1,22 +1,23 @@
 require "thor"
 require "scruber"
 require "scruber/cli/project_generator"
+require "scruber/cli/generators"
 require "scruber/app_searcher"
 
 module Scruber
   module CLI
 
     class Root < Thor
-     def self.exit_on_failure?
+      def self.exit_on_failure?
         true
       end
 
-      register(ProjectGenerator, "new", "new PATH", "Create new project")
+      register ProjectGenerator, "new", "new PATH", "Create new project"
+      register Generators, 'generate', 'generate [GENERATOR]', 'Generate something'
 
       desc 'start', 'Run scraper'
       def start(name)
         if defined?(APP_PATH)
-          # raise ::Thor::Error, "ERROR: Scruber project not found." unless File.exist?(File.expand_path('config/application', Dir.pwd))
           scraper_path = Scruber::AppSearcher.find_scraper(name, APP_PATH)
           raise ::Thor::Error, "ERROR: Scraper not found." if scraper_path.nil?
           say "booting..."
@@ -24,10 +25,12 @@ module Scruber
           Dir[File.expand_path('../initializers/*.rb', APP_PATH)].sort.each do |i|
             require i
           end
-          say "starting #{name}"
+          ENV['SCRUBER_SCRAPER_NAME'] = File.basename(scraper_path).gsub(/\.rb\Z/, '').underscore
+          say "starting #{ENV['SCRUBER_SCRAPER_NAME']}"
+          
           require scraper_path
         else
-          Scruber::AppSearcher.exec_app(name)
+          raise ::Thor::Error, "ERROR: Scruber project not found."
         end
       end
 

data/lib/scruber/cli/generators.rb

@@ -0,0 +1,33 @@
+require "thor"
+require 'fileutils'
+
+module Scruber
+  module CLI
+    class Generators < Thor
+
+      class ScraperGenerator < Thor::Group
+        include Thor::Actions
+
+        argument :name
+
+        def self.source_root
+          File.dirname(__FILE__) + '/templates'
+        end
+
+        def create_files
+          if defined?(APP_PATH)
+            scraper_path = Scruber::AppSearcher.find_scraper(name, APP_PATH)
+            if scraper_path.present?
+              raise ::Thor::Error, "ERROR: Scraper already exists" 
+            end
+            template 'scrapers/sample.tt', File.expand_path('../../scrapers/'+name+'.rb', APP_PATH)
+          else
+            raise ::Thor::Error, "ERROR: Scruber project not found."
+          end
+        end
+      end
+
+      register ScraperGenerator, 'scraper', 'scraper [NAME]', 'Generate scraper'
+    end
+  end
+end

data/lib/scruber/cli/project_generator.rb

@@ -25,9 +25,9 @@ module Scruber
         template 'Gemfile.tt', path+'/Gemfile'
         template 'gitignore.tt', path+'/.gitignore'
         template 'bin/scruber.tt', path+'/bin/scruber'
+        chmod path+'/bin/scruber', '+x'
         template 'application.tt', path+'/config/application.rb'
         template 'boot.tt', path+'/config/boot.rb'
-        template 'boot.tt', path+'/config/boot.rb'
         template 'initializers/proxies.tt', path+'/config/initializers/proxies.rb'
         template 'initializers/user_agents.tt', path+'/config/initializers/user_agents.rb'
         template 'scrapers/sample.tt', path+'/scrapers/sample.rb'

data/lib/scruber/cli/templates/scrapers/sample.tt

@@ -1,7 +1,9 @@
-Scruber.run :sample do
-  queue.add "http://example.com"
+Scruber.run do
+  get "http://..."
   
-  parser :seed do |page|
-    puts page.response_body
+  parse :html do |page, doc|
+    # page - queue page object
+    # doc - processed object, in this case Nokogiri::HTML(page.response_body) object
+    puts doc.at('title').text
   end
 end

data/lib/scruber/core/crawler.rb

@@ -3,8 +3,17 @@ module Scruber
     class Crawler
       attr_reader :queue, :fetcher, :scraper_name
 
-      def initialize(scraper_name, options={})
-        @scraper_name = scraper_name
+      def initialize(*args)
+        if args.first.is_a?(Hash)
+          scraper_name = nil
+          options = args.first
+        else
+          scraper_name, options = args
+          options ||= {}
+        end
+        @scraper_name = scraper_name.present? ? scraper_name : ENV['SCRUBER_SCRAPER_NAME']
+        raise Scruber::ArgumentError.new("Scraper name is empty. Pass it to `Scruber.run :name do` or through ENV['SCRUBER_SCRAPER_NAME']") if @scraper_name.blank?
+        @scraper_name = @scraper_name.to_sym
         Scruber.configuration.merge_options(options)
         @callbacks_options = {}
         @callbacks = {}
@@ -26,6 +35,7 @@ module Scruber
             if @callbacks[page.page_type.to_sym]
               processed_page = process_page(page, page.page_type.to_sym)
               instance_exec page, processed_page, &(@callbacks[page.page_type.to_sym])
+              page.processed! unless page.sent_to_redownload?
             end
           end
         end
@@ -39,14 +49,12 @@ module Scruber
       end
 
       def method_missing(method_sym, *arguments, &block)
-        Scruber::Core::Crawler._registered_method_missings.find do |(pattern, func)|
+        Scruber::Core::Crawler._registered_method_missings.each do |(pattern, func)|
           if (scan_results = method_sym.to_s.scan(pattern)).present?
-            instance_exec method_sym, scan_results, arguments, &(func)
-            true
-          else
-            false
+            return instance_exec(method_sym, scan_results, arguments+[block], &(func))
           end
-        end || super
+        end
+        super
       end
 
       def respond_to?(method_sym, include_private = false)
@@ -81,7 +89,7 @@ module Scruber
         end
 
         def process_page(page, page_type)
-          page_format = @callbacks_options[page_type].fetch(:page_format){ nil }
+          page_format = @callbacks_options[page_type].fetch(:format){ nil }
           Scruber::Core::PageFormat.process(page, page_format)
         end
 

data/lib/scruber/core/extensions/parser_aliases.rb

@@ -0,0 +1,24 @@
+module Scruber
+  module Core
+    module Extensions
+      class ParserAliases < Base
+        module CoreMethods
+          def parse(*args, &block)
+            page_format = args.shift
+            parser('seed', {format: page_format}, &block)
+          end
+
+          def self.included(base)
+            Scruber::Core::Crawler.register_method_missing /\Aparse_(\w+)\Z/ do |meth, scan_results, args|
+              page_type = scan_results.first.first
+              page_format = args.first.is_a?(Symbol) ? args.shift : nil
+              block = args.shift
+              parser(page_type, {format: page_format}, &block)
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/scruber/core/extensions/queue_aliases.rb

@@ -0,0 +1,30 @@
+module Scruber
+  module Core
+    module Extensions
+      class QueueAliases < Base
+        module CoreMethods
+          %w(get post head).each do |meth|
+            define_method meth.to_sym do |url, options={}|
+              queue.add url, options.merge({method: meth.to_sym})
+            end
+          end
+
+          def self.included(base)
+            Scruber::Core::Crawler.register_method_missing /\A(get|post|head)_(\w+)\Z/ do |m, scan_results, args|
+              meth, page_type = scan_results.first
+              url, options = args
+              options = {} if options.nil?
+              Scruber::Core::Crawler.class_eval do
+                define_method "#{meth}_#{page_type}".to_sym do |url, options={}|
+                  queue.add url, options.merge({method: meth.to_sym, page_type: page_type})
+                end
+              end
+              queue.add url, options.merge({method: meth.to_sym, page_type: page_type})
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/scruber/core/page_format/xml.rb

@@ -3,7 +3,7 @@ module Scruber
     module PageFormat
       class XML < Base
         def self.process(page)
-          Nokogiri.parse(page.response_body) rescue nil
+          Nokogiri::XML(page.response_body) rescue nil
         end
       end
     end

data/lib/scruber/queue_adapters/abstract_adapter.rb

@@ -53,6 +53,7 @@ module Scruber
 
           @_fetcher_agent = false
           @_proxy = false
+          @_redownload = false
         end
 
         def fetcher_agent
@@ -96,6 +97,25 @@ module Scruber
           raise NotImplementedError
         end
 
+        def processed!
+          @processed_at = Time.now.to_i
+          @_redownload = false
+          save
+        end
+
+        def redownload!
+          @_redownload = true
+
+          @processed_at = nil
+          @retry_count += 1
+          @fetched_at = 0
+          @response_body = nil
+          save
+        end
+
+        def sent_to_redownload?
+          @_redownload
+        end
       end
 
       def initialize(options={})
@@ -113,6 +133,10 @@ module Scruber
       def fetch_downloaded(count=nil)
         raise NotImplementedError
       end
+
+      def initialized?
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/scruber/queue_adapters/memory.rb

@@ -5,7 +5,9 @@ module Scruber
 
       class Page < Scruber::QueueAdapters::AbstractAdapter::Page
         def save
-          if self.fetched_at > 0
+          if self.processed_at.to_i > 0
+            nil
+          elsif self.fetched_at > 0
             @queue.add_downloaded self
           elsif self.max_retry_times && self.retry_count >= self.max_retry_times.to_i
             @queue.add_error_page self
@@ -73,6 +75,10 @@ module Scruber
         @error_pages -= [page]
       end
 
+      def initialized?
+        @queue.present? || @downloaded_pages.present? || @error_pages.present?
+      end
+
     end
   end
 end

data/lib/scruber/version.rb

@@ -1,3 +1,3 @@
 module Scruber
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scruber
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Ivan Goncharov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-15 00:00:00.000000000 Z
+date: 2018-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -171,6 +171,7 @@ files:
 - lib/scruber.rb
 - lib/scruber/app_searcher.rb
 - lib/scruber/cli.rb
+- lib/scruber/cli/generators.rb
 - lib/scruber/cli/project_generator.rb
 - lib/scruber/cli/templates/Gemfile.tt
 - lib/scruber/cli/templates/application.tt
@@ -185,6 +186,8 @@ files:
 - lib/scruber/core/extensions/base.rb
 - lib/scruber/core/extensions/csv_output.rb
 - lib/scruber/core/extensions/loop.rb
+- lib/scruber/core/extensions/parser_aliases.rb
+- lib/scruber/core/extensions/queue_aliases.rb
 - lib/scruber/core/page_format.rb
 - lib/scruber/core/page_format/base.rb
 - lib/scruber/core/page_format/html.rb