scruber 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 937243b2503853e755c95800e0d34ae5ecb939a7
-  data.tar.gz: 1e2472ff1d1487da8df94292daa57a27d5b52c5a
+  metadata.gz: 8deee66960a3768ace0af72a5cb1eced62c90329
+  data.tar.gz: a0d3f330d8b838aee078f2d752226a1e5432b311
 SHA512:
-  metadata.gz: 2a30b80a4301ca18b87e1c91217ebdfd881e13c511e643ce198e7e9f28984d271b0133b11a065b6d971690fca49b948b289147884eed7c01ee5bfc985d35cc69
-  data.tar.gz: 77201fa5bbde27dfac6f7205711b05a3af9f85ab0ad705c6d08af5ea6b55f2c18c37a4e80878e3e7de7280ad945d20aca79aedc179087036f7bf176ecf05f597
+  metadata.gz: 30df32ccd86afde913d47483e9f327b94869c52a21f7c1a43f442ef8a1f138a1500d0746d4641a4542937aa2dbba7e53e4697c95b234cfdc3d07eeb8ab3d13ed
+  data.tar.gz: 4e57023647a62f7f312a8a77b89097920e7ab6750c1fd6562d1b6b6b4b3ff239b484f08aed6920a259365100ba4efe9d3f168b1a873183a377c877261d49ba15

data/README.md

@@ -1,38 +1,42 @@
 # Scruber
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/scruber`. To experiment with that code, run `bin/console` for an interactive prompt.
+Scruber is an open source scraping framework for Ruby.
 
-TODO: Delete this and the text above, and describe your gem
+## Getting started
 
-## Installation
+1. Install Scruber at the command prompt if you haven't yet:
 
-Add this line to your application's Gemfile:
+    $ gem install scruber
 
-```ruby
-gem 'scruber'
-```
+2. Create a new workspace
 
-And then execute:
+    $ scruber new myworkspace
 
-    $ bundle
+3. Create a new scraper
 
-Or install it yourself as:
+    $ scruber new scraper example
 
-    $ gem install scruber
 
-## Usage
+```ruby
+Scruber.run do
+  csv_file 'output.csv', col_sep: ','
+
+  get 'http://example.com'
 
-TODO: Write usage instructions here
+  parse :html do |page, html|
+    csv_out html.at('title').text
+  end
+end
+```
 
-## Development
+4. Run your scraper
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+    $ scruber start example
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/scruber.
+Bug reports and pull requests are welcome on GitHub at https://github.com/scruber/scruber.
 
 ## License
 

data/lib/scruber/core/crawler.rb

@@ -17,7 +17,7 @@ module Scruber
         Scruber.configuration.merge_options(options)
         @callbacks_options = {}
         @callbacks = {}
-        @on_complete_callbacks = {}
+        @on_complete_callbacks = []
         @queue = Scruber::Queue.new(scraper_name: scraper_name)
         @fetcher = Scruber::Fetcher.new
         load_extenstions
@@ -39,7 +39,7 @@ module Scruber
             end
           end
         end
-        @on_complete_callbacks.each do |_,callback|
+        @on_complete_callbacks.sort_by{|c| -c[0] }.each do |(_,callback)|
           instance_exec &(callback)
         end
       end
@@ -84,8 +84,8 @@ module Scruber
           @callbacks[page_type.to_sym] = block
         end
 
-        def on_complete_callback(name, &block)
-          @on_complete_callbacks[name] = block
+        def on_complete(priority=1, &block)
+          @on_complete_callbacks.push [priority,block]
         end
 
         def process_page(page, page_type)

data/lib/scruber/core/extensions/csv_output.rb

@@ -7,7 +7,7 @@ module Scruber
             file_id = options.fetch(:file_id) { :default }.to_sym
             options.delete(:file_id)
             Scruber::Core::Extensions::CsvOutput.register_csv file_id, path, options
-            on_complete_callback :close_csv_files do
+            on_complete -1 do
               Scruber::Core::Extensions::CsvOutput.close_all
             end
           end

data/lib/scruber/helpers/dictionary_reader/csv.rb

@@ -7,7 +7,7 @@ module Scruber
         end
 
         def read(options={})
-          col_sep = options.delete(:col_sep) || ';'
+          col_sep = options.delete(:col_sep) || ','
 
           CSV.foreach(@file_path, col_sep: col_sep, headers: true, encoding: 'utf-8') do |csv_row|
             if options.blank?

data/lib/scruber/version.rb

@@ -1,3 +1,3 @@
 module Scruber
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/lib/scruber.rb

@@ -26,6 +26,10 @@ require "scruber/core/extensions/csv_output"
 require "scruber/core/extensions/queue_aliases"
 require "scruber/core/extensions/parser_aliases"
 
+require "scruber/helpers/dictionary_reader"
+require "scruber/helpers/dictionary_reader/xml"
+require "scruber/helpers/dictionary_reader/csv"
+
 # require "scruber/core/configuration"
 # require "scruber/core/configuration"
 
@@ -44,11 +48,6 @@ module Scruber
       autoload :AbstractAdapter,  "scruber/helpers/fetcher_agent_adapters/abstract_adapter"
       autoload :Memory,           "scruber/helpers/fetcher_agent_adapters/memory"
     end
-    autoload :DictionaryReader,   "scruber/helpers/dictionary_reader"
-    module DictionaryReader
-      autoload :Xml,              "scruber/helpers/dictionary_reader/xml"
-      autoload :Csv,              "scruber/helpers/dictionary_reader/csv"
-    end
   end
 
   class << self

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scruber
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Ivan Goncharov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-17 00:00:00.000000000 Z
+date: 2018-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus