scruber 0.1.6 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b845332207b108efa91983b4721cf7631120ea36
-  data.tar.gz: 4c9d931ccdbf777c9d469d7cf2697f00acfe94dc
+  metadata.gz: b74171eb49e730929f3b303cc1b65f7d55eb54f5
+  data.tar.gz: 67debe50e980a21e66fc9966e509b9c4fb65fc2b
 SHA512:
-  metadata.gz: b7c9bc638e7f168401bfd15de02746b691c5477563404db66810f894e7a1925821064b096d47c065209223856782e787bdc3f6272a179068aa0fdfcb6c14994d
-  data.tar.gz: f4c73d7e94e795b32c647285f320cf9e25b9f6efa0c528aaa3138ab99c263627f9ad47088851126d2df07421e633037fa3e8f434d7303a91ad350804d9a64903
+  metadata.gz: 2f997b9b072270c8a28dc00f6dcde03a9041adb8f1fc6245d862f4423e3d155bf9e6fe478c11fe0d6081a878407fdc35792fe3a17dbf93b3f566daeb3fd22a60
+  data.tar.gz: a5d4f2bafe347e9a775bf01cbfd2170678596fd133cb788a444099a03a95bcc7f72116bd725f32506766f9dbb0756c06f1e3dc909e526d684cfa14e3535f6ae2

data/lib/scruber.rb

@@ -23,6 +23,7 @@ require "scruber/core/page_format"
 require "scruber/core/page_format/base"
 require "scruber/core/page_format/xml"
 require "scruber/core/page_format/html"
+require "scruber/core/page_format/json"
 
 require "scruber/core/extensions/base"
 require "scruber/core/extensions/loop"
@@ -58,7 +59,7 @@ module Scruber
 
     def run(*args, &block)
       raise "You need a block to build!" unless block_given?
-      
+
       Core::Crawler.new(*args).run(&block)
     end
 

data/lib/scruber/core/crawler.rb

@@ -1,10 +1,10 @@
 module Scruber
   module Core
-    # 
+    #
     # Crawler class
-    # 
+    #
     # Main class-runner for scrapers.
-    # 
+    #
     # @example Simple scraper
     #   Scruber::Core::Crawler.new(:simple) do
     #     get 'http://example.com'
@@ -12,29 +12,29 @@ module Scruber
     #       puts html.at('title').text
     #     end
     #   end
-    # 
+    #
     # @author Ivan Goncharov
-    #     
+    #
     class Crawler
       attr_reader :queue, :fetcher, :scraper_name
 
-      # 
+      #
       # Initialize crawler with scraper name and/or with options
-      #     
+      #
       #   Crawler.new(:sample, fetcher_adapter: :custom)
       #   Crawler.new(:sample)
       #   Crawler.new(fetcher_adapter: :custom)
-      # 
+      #
       # @param args [Array] if first arg is a Symbol, it will be used as scraper_name, hash will me used as configuration options (see {Scruber::Core::Configuration})
-      # 
+      #
       # @return [Scruber::Core::Crawler] [description]
       def initialize(*args)
         if args.first.is_a?(Hash)
           scraper_name = nil
-          options = args.first
+          @options = args.first
         else
-          scraper_name, options = args
-          options ||= {}
+          scraper_name, @options = args
+          @options ||= {}
         end
         @scraper_name = scraper_name.present? ? scraper_name : ENV['SCRUBER_SCRAPER_NAME']
         raise Scruber::ArgumentError.new("Scraper name is empty. Pass it to `Scruber.run :name do` or through ENV['SCRUBER_SCRAPER_NAME']") if @scraper_name.blank?
@@ -44,7 +44,7 @@ module Scruber
         @on_page_error_callback = nil
         @on_complete_callbacks = []
 
-        Scruber.configuration.merge_options(options)
+        Scruber.configuration.merge_options(@options)
         ActiveSupport::Dependencies.autoload_paths = Scruber.configuration.autoload_paths
 
         @queue = Scruber::Queue.new(scraper_name: @scraper_name)
@@ -53,9 +53,9 @@ module Scruber
         load_extenstions
       end
 
-      # 
+      #
       # Crawling engine
-      # 
+      #
       # @param block [Proc] crawler body
       def run(&block)
         instance_eval &block
@@ -75,33 +75,33 @@ module Scruber
             end
           end
         end
-        @on_complete_callbacks.sort_by{|c| -c[0] }.each do |(_,callback)|
+        @on_complete_callbacks.sort_by{|c| -c[0] }.map do |(_,callback)|
           instance_exec &(callback)
-        end
+        end.first
       end
 
-      # 
+      #
       # Register parser
-      # 
+      #
       # @param page_type [Symbol] type of page
       # @param options [Hash] options for parser
-      # @option options [Symbol] :format format of page. Scruber automatically process 
+      # @option options [Symbol] :format format of page. Scruber automatically process
       #                                  page body depends on this format. For example :json or :html
       # @param block [Proc] body of parser
-      # 
+      #
       # @return [void]
       def parser(page_type, options={}, &block)
         register_callback(page_type, options, &block)
       end
 
-      # 
+      #
       # Method missing callback. Scruber allows to register
       # regexp and proc body to process calls
-      # 
+      #
       # @param method_sym [Symbol] missing method name
       # @param arguments [Array] arguments
       # @param block [Proc] block (if passed)
-      # 
+      #
       # @return [type] [description]
       def method_missing(method_sym, *arguments, &block)
         Scruber::Core::Crawler._registered_method_missings.each do |(pattern, func)|
@@ -124,27 +124,27 @@ module Scruber
 
       class << self
 
-        # 
+        #
         # Register method missing callback
-        # 
+        #
         # @param pattern [Regexp] Regexp to match missing name
         # @param block [Proc] Body to process missing method
-        # 
+        #
         # @return [void]
         def register_method_missing(pattern, &block)
           _registered_method_missings[pattern] = block
         end
 
-        # 
+        #
         # Registered method missing callbacks dictionary
-        # 
+        #
         # @return [Hash] callbacks
         def _registered_method_missings
           @registered_method_missings ||= {}
         end
       end
 
-      # 
+      #
       # Register callback which will be executed when
       # downloading and parsing will be completed.
       # For example when you need to write results to file,
@@ -153,16 +153,16 @@ module Scruber
       #     on_complete -1 do
       #       Scruber::Core::Extensions::CsvOutput.close_all
       #     end
-      # 
+      #
       # @param priority [Integer] priority of this callback
       # @param block [Proc] body of callback
-      # 
+      #
       # @return [void]
       def on_complete(priority=1, &block)
         @on_complete_callbacks.push [priority,block]
       end
 
-      # 
+      #
       # Register callback which will be executed for
       # error pages, like 404 or 500
       # Attention! You should call one of these methods for page
@@ -178,9 +178,9 @@ module Scruber
       #         page.delete
       #       end
       #     end
-      # 
+      #
       # @param block [Proc] body of callback
-      # 
+      #
       # @return [void]
       def on_page_error(&block)
         @on_page_error_callback = block
@@ -188,46 +188,46 @@ module Scruber
 
       private
 
-        # 
+        #
         # Register parser
-        # 
+        #
         # @param page_type [Symbol] type of page
         # @param options [Hash] options for parser
-        # @option options [Symbol] :format format of page. Scruber automatically process 
+        # @option options [Symbol] :format format of page. Scruber automatically process
         #                                  page body depends on this format. For example :json or :html
         # @param block [Proc] body of parser
-        # 
+        #
         # @return [void]
         def register_callback(page_type, options, &block)
           @callbacks_options[page_type.to_sym] = options || {}
           @callbacks[page_type.to_sym] = block
         end
 
-        # 
+        #
         # Process page body depends on format of this page
         # For example, if page_format = :html, then
         # it will return Nokogiri::HTML(page.response_body)
-        # 
+        #
         # @param page [Page] page from queue
         # @param page_type [Symbol] name of parser
-        # 
+        #
         # @return [Object] depends on page_type it will return different objects
         def process_page(page, page_type)
           page_format = @callbacks_options[page_type].fetch(:format){ nil }
           Scruber::Core::PageFormat.process(page, page_format)
         end
 
-        # 
+        #
         # Loads all extensions
-        # 
+        #
         # @return [void]
         def load_extenstions
           Scruber::Core::Extensions::Base.descendants.each(&:register)
         end
 
-        # 
+        #
         # Initialize progressbar, that shows progress in console
-        # 
+        #
         # @return [void]
         def initialize_progressbar
           unless Scruber.configuration.silent
@@ -243,9 +243,9 @@ module Scruber
           end
         end
 
-        # 
+        #
         # Out progress to console
-        # 
+        #
         # @return [void]
         def show_progress
           if @progressbar

data/lib/scruber/core/page_format/json.rb

@@ -0,0 +1,13 @@
+module Scruber
+  module Core
+    module PageFormat
+      class Json < Base
+        def self.process(page)
+          JSON.parse(page.response_body) rescue nil
+        end
+      end
+    end
+  end
+end
+
+Scruber::Core::PageFormat.add(:json, Scruber::Core::PageFormat::Json)

data/lib/scruber/fetcher_adapters/abstract_adapter.rb

@@ -1,3 +1,5 @@
+require 'charlock_holmes'
+
 module Scruber
   module FetcherAdapters
     class AbstractAdapter
@@ -45,9 +47,24 @@ module Scruber
             page.fetched_at = Time.now.to_i
           end
         end
+        if page.response_headers
+          page.response_headers = page.response_headers.inject({}) {|acc, (k,v)| acc[k.gsub('.', '_')] = v.is_a?(Array) ? v.map{|v1| convert_to_utf8(v1) } : convert_to_utf8(v); acc }
+        end
+        page.response_body = convert_to_utf8(page.response_body)
         page
       end
 
+      def convert_to_utf8(text)
+        unless text.to_s.empty?
+          detection = CharlockHolmes::EncodingDetector.detect(text)
+          if detection && detection[:encoding].present?
+            text = CharlockHolmes::Converter.convert(text, detection[:encoding], 'UTF-8') rescue text
+          end
+        end
+
+        text
+      end
+
       def headers_for(page)
         if page.fetcher_agent
           headers = page.fetcher_agent.headers

data/lib/scruber/fetcher_adapters/typhoeus_fetcher.rb

@@ -60,7 +60,7 @@ module Scruber
       def on_complete_callback(page, response)
         page.response_code = response.code
         page.response_body = response.body
-        page.response_headers = response.response_headers
+        page.response_headers = response.headers
         page.response_total_time = response.total_time
         
         if response.timed_out?

data/lib/scruber/queue_adapters/abstract_adapter.rb

@@ -134,6 +134,16 @@ module Scruber
           raise NotImplementedError
         end
 
+
+        # 
+        # Join url of current page with another path or url
+        # @param link_url [String] link
+        # 
+        # @return [String] joined url
+        def url_join(link_url)
+          URI.join(url, link_url).to_s
+        end
+
         def [](k)
           instance_variable_get("@#{k.to_s}")
         end

data/lib/scruber/version.rb

@@ -1,3 +1,3 @@
 module Scruber
-  VERSION = "0.1.6"
+  VERSION = "0.1.9"
 end

data/scruber.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport", '~> 5.1', '>= 5.1.5'
   spec.add_dependency "powerbar", '~> 2.0', '>= 2.0.1'
   spec.add_dependency "paint", '~> 2.0', '>= 2.0.1'
+  spec.add_dependency "charlock_holmes", '~> 0.7', '>= 0.7.6'
   spec.add_runtime_dependency "thor", "0.20.0"
   spec.add_development_dependency "bundler", "~> 1.15"
   spec.add_development_dependency "rake", "~> 10.0"

data/spec/core/extensions/loop_spec.rb

@@ -11,16 +11,19 @@ RSpec.describe Scruber::Core::Extensions::Loop do
 
     it "should add dictionary and read info" do
       Scruber::Core::Extensions::Loop.register
-      $zip_codes = []
-      Scruber.run :sample do
+      zip_codes = Scruber.run :sample do
         add_dictionary :zip_codes_usa, File.expand_path(File.dirname(__FILE__))+'/dict.csv', :csv
+        @zip_codes = []
+
         seed do
           loop :zip_codes_usa, state: 'NY' do |row|
-            $zip_codes.push row['zip']
+            @zip_codes.push row['zip']
           end
         end
+
+        on_complete { @zip_codes }
       end
-      expect($zip_codes).to eq(['10001', '10002'])
+      expect(zip_codes).to eq(['10001', '10002'])
     end
   end
 end

data/spec/core/extensions/parser_aliases_spec.rb

@@ -19,15 +19,17 @@ RSpec.describe Scruber::Core::Extensions::ParserAliases do
 
         stub_request(:get, "http://example.com").to_return(body: 'Example Domain')
 
-        Scruber.run :sample do
+        page = Scruber.run :sample do
           get "http://example.com"
-          
+
           parse do |page|
-            $page = page
+            @queue_page = page
           end
+
+          on_complete { @queue_page }
         end
-        expect($page.url).to eq("http://example.com")
-        expect($page.page_type.to_s).to eq("seed")
+        expect(page.url).to eq("http://example.com")
+        expect(page.page_type.to_s).to eq("seed")
       end
 
       it "should register parser with custom page_type" do
@@ -35,16 +37,18 @@ RSpec.describe Scruber::Core::Extensions::ParserAliases do
 
         stub_request(:post, "http://example.com").to_return(body: 'Example Domain')
 
-        Scruber.run :sample do
+        page = Scruber.run :sample do
           post_product "http://example.com"
-          
+
           parse_product do |page|
-            $page = page
+            @queue_page = page
           end
+
+          on_complete { @queue_page }
         end
-        expect($page.url).to eq("http://example.com")
-        expect($page.method.to_s).to eq("post")
-        expect($page.page_type.to_s).to eq("product")
+        expect(page.url).to eq("http://example.com")
+        expect(page.method.to_s).to eq("post")
+        expect(page.page_type.to_s).to eq("product")
       end
     end
 
@@ -54,17 +58,19 @@ RSpec.describe Scruber::Core::Extensions::ParserAliases do
 
         stub_request(:get, "http://example.com").to_return(body: '<div><span>Example Domain</span></div>')
 
-        Scruber.run :sample do
+        page, doc = Scruber.run :sample do
           get "http://example.com"
-          
+
           parse :html do |page,doc|
-            $page = page
-            $doc = doc
+            @queue_page = page
+            @doc = doc
           end
+
+          on_complete { [@queue_page, @doc] }
         end
-        expect($doc.at('span').text).to eq("Example Domain")
-        expect($page.page_type.to_s).to eq("seed")
-        expect($page.method.to_s).to eq("get")
+        expect(doc.at('span').text).to eq("Example Domain")
+        expect(page.page_type.to_s).to eq("seed")
+        expect(page.method.to_s).to eq("get")
       end
 
       it "should register parser with custom page_type" do
@@ -72,17 +78,19 @@ RSpec.describe Scruber::Core::Extensions::ParserAliases do
 
         stub_request(:post, "http://example.com").to_return(body: '<div><span>Example Post</span></div>')
 
-        Scruber.run :sample do
+        page, doc = Scruber.run :sample do
           post_product "http://example.com"
-          
+
           parse_product :html do |page,doc|
-            $page = page
-            $doc = doc
+            @queue_page = page
+            @doc = doc
           end
+
+          on_complete { [@queue_page, @doc] }
         end
-        expect($doc.at('span').text).to eq("Example Post")
-        expect($page.method.to_s).to eq("post")
-        expect($page.page_type.to_s).to eq("product")
+        expect(doc.at('span').text).to eq("Example Post")
+        expect(page.method.to_s).to eq("post")
+        expect(page.page_type.to_s).to eq("product")
       end
     end
   end

data/spec/core/extensions/queue_aliases_spec.rb

@@ -19,25 +19,29 @@ RSpec.describe Scruber::Core::Extensions::QueueAliases do
       it "should add page to queue" do
         described_class.register
 
-        Scruber.run :sample do
+        page = Scruber.run :sample do
           get "http://example.com"
-          $page = queue.fetch_pending
+
+          @queue_page = queue.fetch_pending
+          on_complete { @queue_page }
         end
-        expect($page.url).to eq("http://example.com")
-        expect($page.method.to_s).to eq("get")
-        expect($page.page_type.to_s).to eq("seed")
+        expect(page.url).to eq("http://example.com")
+        expect(page.method.to_s).to eq("get")
+        expect(page.page_type.to_s).to eq("seed")
       end
 
       it "should add page to queue" do
         described_class.register
 
-        Scruber.run :sample do
+        page = Scruber.run :sample do
           post_product "http://example.com"
-          $page = queue.fetch_pending
+
+          @queue_page = queue.fetch_pending
+          on_complete { @queue_page }
         end
-        expect($page.url).to eq("http://example.com")
-        expect($page.method.to_s).to eq("post")
-        expect($page.page_type).to eq("product")
+        expect(page.url).to eq("http://example.com")
+        expect(page.method.to_s).to eq("post")
+        expect(page.page_type).to eq("product")
       end
     end
 
@@ -45,27 +49,31 @@ RSpec.describe Scruber::Core::Extensions::QueueAliases do
       it "should add page to queue" do
         described_class.register
 
-        Scruber.run :sample do
+        page = Scruber.run :sample do
           get "http://example.com", user_agent: 'Agent 1'
-          $page = queue.fetch_pending
+
+          @queue_page = queue.fetch_pending
+          on_complete { @queue_page }
         end
-        expect($page.url).to eq("http://example.com")
-        expect($page.method.to_s).to eq("get")
-        expect($page.page_type.to_s).to eq("seed")
-        expect($page.user_agent).to eq('Agent 1')
+        expect(page.url).to eq("http://example.com")
+        expect(page.method.to_s).to eq("get")
+        expect(page.page_type.to_s).to eq("seed")
+        expect(page.user_agent).to eq('Agent 1')
       end
 
       it "should add page to queue" do
         described_class.register
 
-        Scruber.run :sample do
+        page = Scruber.run :sample do
           post_product "http://example.com", user_agent: 'Agent 1'
-          $page = queue.fetch_pending
+
+          @queue_page = queue.fetch_pending
+          on_complete { @queue_page }
         end
-        expect($page.url).to eq("http://example.com")
-        expect($page.method.to_s).to eq("post")
-        expect($page.page_type).to eq("product")
-        expect($page.user_agent).to eq('Agent 1')
+        expect(page.url).to eq("http://example.com")
+        expect(page.method.to_s).to eq("post")
+        expect(page.page_type).to eq("product")
+        expect(page.user_agent).to eq('Agent 1')
       end
     end
   end

data/spec/core/extensions/seed_spec.rb

@@ -14,31 +14,32 @@ RSpec.describe Scruber::Core::Extensions::Seed do
     stub_request(:get, "http://example.com").to_return(body: '<div><a>Main</a></div>')
     stub_request(:get, "http://example.com/contacts").to_return(body: '<div><a>Contacts</a></div>')
   end
-  
+
   it "should execute seed block" do
-    $queue_size = 0
-    Scruber.run :sample do
+    queue_size = Scruber.run :sample do
       seed do
         get 'http://example.com'
       end
-      $queue_size = queue.size
+      @queue_size = queue.size
+      on_complete { @queue_size }
     end
-    expect($queue_size).to eq(1)
+    expect(queue_size).to eq(1)
   end
 
   it "should not execute seed block" do
-    $queue_size = 0
-    Scruber.run :sample do
+    queue_size, page = Scruber.run :sample do
       seed do
         get 'http://example.com'
       end
       seed do
         get 'http://example.com/contacts'
       end
-      $queue_size = queue.size
-      $page = queue.fetch_pending
+      @queue_size = queue.size
+      @queue_page = queue.fetch_pending
+
+      on_complete { [@queue_size, @queue_page] }
     end
-    expect($queue_size).to eq(1)
-    expect($page.url).to eq("http://example.com")
+    expect(queue_size).to eq(1)
+    expect(page.url).to eq("http://example.com")
   end
 end

data/spec/helpers/fetcher_agent_adapters/abstract_adapter_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Scruber::Helpers::FetcherAgentAdapters::AbstractAdapter do
                           cookie_jar: cookie_jar_string,
                           disable_proxy: true
     end
-    
+
     it "set values" do
       expect(agent.id).to eq(1)
       expect(agent.user_agent).to eq('Scruber')
@@ -33,7 +33,8 @@ RSpec.describe Scruber::Helpers::FetcherAgentAdapters::AbstractAdapter do
     end
 
     it "parse cookies from page" do
-      page = Scruber::QueueAdapters::AbstractAdapter::Page.new(nil, url: 'http://example.com', response_headers: {"Connection" => "keep-alive","Set-Cookie" => "__cfduid=dc8db498b1e419b7943052a69c8e9d1d01504311966; expires=Sun, 02-Sep-18 00:26:06 GMT; path=/; domain=example.com; HttpOnly"})
+      page = Scruber::QueueAdapters::AbstractAdapter::Page.new(nil, url: 'http://example.com', response_headers: {"Connection" => "keep-alive","Set-Cookie" => "__cfduid=dc8db498b1e419b7943052a69c8e9d1d01504311966; expires=Sun, 02-Sep-#{Date.today.year+1} 00:26:06 GMT; path=/; domain=example.com; HttpOnly"})
+      puts page.response_cookies.inspect
       agent.parse_cookies_from_page!(page)
       expect(agent.cookie_for('http://example.com')).to eq('__cfduid=dc8db498b1e419b7943052a69c8e9d1d01504311966; feed_flow=top')
     end

data/spec/scruber_spec.rb

@@ -20,7 +20,7 @@ RSpec.describe Scruber do
         config.fetcher_adapter = :typhoeus_fetcher
       end
     end
-    
+
     it "returns :typhoeus_fetcher as fetcher" do
       expect(Scruber.configuration.fetcher_adapter).to eq(:typhoeus_fetcher)
     end
@@ -34,70 +34,70 @@ RSpec.describe Scruber do
 
       it "should set scraper name from ENV" do
         ENV['SCRUBER_SCRAPER_NAME'] = 'sample'
-        Scruber.run do
-          $scraper_name = scraper_name
+        name = Scruber.run do
+          on_complete { scraper_name }
         end
-        expect($scraper_name).to eq(:sample)
+        expect(name).to eq(:sample)
       end
     end
 
     context "with args" do
       it "should set scraper name from first arg" do
-        Scruber.run :sample1 do
-          $scraper_name = scraper_name
+        name = Scruber.run :sample1 do
+          on_complete { scraper_name }
         end
-        expect($scraper_name).to eq(:sample1)
+        expect(name).to eq(:sample1)
       end
 
       it "should set scraper name from first arg, and options from second" do
-        Scruber.run :sample2, queue_adapter: :test do
-          $scraper_name = scraper_name
-          $opt = Scruber.configuration.queue_adapter
+        name, opt = Scruber.run :sample2, queue_adapter: :test do
+          on_complete { [scraper_name, Scruber.configuration.queue_adapter] }
         end
-        expect($scraper_name).to eq(:sample2)
-        expect($opt).to eq(:test)
+        expect(name).to eq(:sample2)
+        expect(opt).to eq(:test)
       end
 
       it "options from first arg and scraper_name from ENV" do
         ENV['SCRUBER_SCRAPER_NAME'] = 'sample'
-        Scruber.run queue_adapter: :test2 do
-          $scraper_name = scraper_name
-          $opt = Scruber.configuration.queue_adapter
+        name, opt = Scruber.run queue_adapter: :test2 do
+          on_complete { [scraper_name, Scruber.configuration.queue_adapter] }
         end
-        expect($scraper_name).to eq(:sample)
-        expect($opt).to eq(:test2)
+        expect(name).to eq(:sample)
+        expect(opt).to eq(:test2)
       end
 
       it "should raise error if passed only options without ENV" do
         ENV['SCRUBER_SCRAPER_NAME'] = nil
-        expect { Scruber.run(queue_adapter: :test2) { $title = scraper_name } }.to raise_error(Scruber::ArgumentError)
+        expect { Scruber.run(queue_adapter: :test2) { scraper_name } }.to raise_error(Scruber::ArgumentError)
       end
     end
 
     it "simple example" do
       stub_request(:get, "http://example.com").to_return(body: 'Example Domain')
 
-      Scruber.run :sample do
+      body = Scruber.run :sample do
         queue.add "http://example.com"
-        
+
         parser :seed do |page|
-          $title = page.response_body
+          @page_response_body = page.response_body
         end
+        on_complete { @page_response_body }
       end
-      expect($title).to eq('Example Domain')
+      expect(body).to eq('Example Domain')
     end
 
     it "should return Nokogiri object" do
       stub_request(:get, "http://example.com/contacts.html").to_return(body: '<div><a>Contacts</a></div>')
 
-      Scruber.run :sample do
+      title = Scruber.run :sample do
         queue.add "http://example.com/contacts.html"
-        
+
         parser :seed, format: :html do |page, html|
-          $title = html.at('a').text
+          @title = html.at('a').text
         end
+        on_complete { @title }
       end
-      expect($title).to eq('Contacts')
+      expect(title).to eq('Contacts')
     end
 
     context "complex example" do
@@ -107,10 +107,9 @@ RSpec.describe Scruber do
         stub_request(:get, "http://example.com/product2").to_return(body: '<div><h1>Product 2</h1></div>')
         stub_request(:get, "http://example.com/product3").to_return(body: '<div><h1>Product 3</h1></div>')
 
-        $products = []
-        Scruber.run :sample do
+        products = Scruber.run :sample do
           get "http://example.com/catalog"
-          
+
           parse :html do |page, doc|
             doc.search('a').each do |a|
               get_product URI.join(page.url, a.attr('href')).to_s
@@ -118,29 +117,34 @@ RSpec.describe Scruber do
           end
 
           parse_product :html do |page,doc|
-            $products.push doc.at('h1').text
+            @products ||= []
+            @products.push doc.at('h1').text
           end
+
+          on_complete { @products }
         end
-        expect($products.sort).to eq((1..3).map{|i| "Product #{i}"}.sort)
+        expect(products.sort).to eq((1..3).map{|i| "Product #{i}"}.sort)
       end
 
       it "should redownload page and increase retry" do
         stub_request(:get, "http://example.com/").to_return(body: '<div>blocked</div>').times(2).then.to_return(body: '<div><h1>Product</h1></div>')
 
-        Scruber.run :sample do
+        title, retry_count = Scruber.run :sample do
           get "http://example.com/"
-          
+
           parse :html do |page, doc|
             if page.response_body =~ /blocked/
               page.redownload!
             else
-              $title = doc.at('h1').text
-              $retry_count = page.retry_count
+              @title = doc.at('h1').text
+              @retry_count = page.retry_count
             end
           end
+
+          on_complete { [@title, @retry_count] }
         end
-        expect($title).to eq('Product')
-        expect($retry_count).to eq(2)
+        expect(title).to eq('Product')
+        expect(retry_count).to eq(2)
       end
     end
 
@@ -148,39 +152,41 @@ RSpec.describe Scruber do
       it "should process 500 error page" do
         stub_request(:get, "http://example.com").to_return(body: '<div><h1>500</h1></div>', status: 500)
 
-        $error_title = nil
-        Scruber.run :sample do
+        error_title = Scruber.run :sample do
           get "http://example.com", max_retry_times: 1
 
           parse :html do |page,doc|
-            $error_title = doc.at('h1').text
+            @error_title = doc.at('h1').text
           end
 
           on_page_error do |page|
-            $error_title = page.response_body
+            @error_title = page.response_body
             page.processed!
           end
+
+          on_complete { @error_title }
         end
-        expect($error_title).to eq('<div><h1>500</h1></div>')
+        expect(error_title).to eq('<div><h1>500</h1></div>')
       end
 
       it "should process 404 error page" do
         stub_request(:get, "http://example.com").to_return(body: '<div><h1>404</h1></div>', status: 404)
 
-        $error_title = nil
-        Scruber.run :sample do
+        error_title = Scruber.run :sample do
           get "http://example.com", max_retry_times: 1
 
           parse :html do |page,doc|
-            $error_title = doc.at('h1').text
+            @error_title = doc.at('h1').text
           end
 
           on_page_error do |page|
-            $error_title = page.response_body
+            @error_title = page.response_body
             page.processed!
           end
+
+          on_complete { @error_title }
         end
-        expect($error_title).to eq('<div><h1>404</h1></div>')
+        expect(error_title).to eq('<div><h1>404</h1></div>')
       end
     end
   end

data/spec/support/queue/queue_adapter.rb

@@ -167,5 +167,10 @@ RSpec.shared_examples "queue_adapter" do
       expect(page1.id).not_to be_blank
       expect(page1.id).not_to eq(page2.id)
     end
+
+    it "should join url" do
+      page1 = page_class.new queue, url: "http://example.com/product1"
+      expect(page1.url_join('/abc')).to eq("http://example.com/abc")
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scruber
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.9
 platform: ruby
 authors:
 - Ivan Goncharov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-23 00:00:00.000000000 Z
+date: 2018-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -138,6 +138,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.6
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -252,6 +272,7 @@ files:
 - lib/scruber/core/page_format.rb
 - lib/scruber/core/page_format/base.rb
 - lib/scruber/core/page_format/html.rb
+- lib/scruber/core/page_format/json.rb
 - lib/scruber/core/page_format/xml.rb
 - lib/scruber/fetcher.rb
 - lib/scruber/fetcher_adapters/abstract_adapter.rb