extraloop 0.0.5 → 0.0.6

data/examples/google_news_scraper.rb

@@ -4,7 +4,7 @@ results = []
 
 ExtraLoop::IterativeScraper.new("https://www.google.com/search?tbm=nws&q=Egypt", :log => {
   :log_level => :debug,
-  :appenders => [Logging.appenders.stderr ]
+  :appenders => [ Logging.appenders.stderr ]
 
 }).set_iteration(:start, (1..101).step(10)).
    loop_on("h3") { |nodes| nodes.map(&:parent) }.

data/examples/wikipedia_categories.rb

@@ -23,7 +23,9 @@ options = {
     :log_level => :info
   }
 }
-request_arguments = { :params => params }
+request_arguments = { :params => params, :headers => {
+  "User-Agent" => "ExtraLoop - ruby data extraction toolkit: http://github.com/afiore/extraloop"
+}}
 
 
 #
@@ -39,9 +41,9 @@ ExtraLoop::IterativeScraper.new(api_url, options, request_arguments).
     extract(:ns).
     extract(:type).
     extract(:timestamp).
-  on(:data, proc { |results|
+  on(:data) do |results|
     results.each { |record| all_results << record }
-  }).
+  end.
   continue_with(:cmcontinue, ['query-continue', 'categorymembers', 'cmcontinue']).
   run()
 

data/examples/wikipedia_categories_recoursive.rb

@@ -29,7 +29,9 @@ class WikipediaCategoryScraper < ExtraLoop::IterativeScraper
       :format => :json,
       :log => false
     }
-    request_arguments = { :params => params }
+    request_arguments = { :params => params, :headers => {
+      "User-Agent" => "ExtraLoop - ruby data extraction toolkit: http://github.com/afiore/extraloop"
+    }}
 
     super(@@api_url, options, request_arguments)
 
@@ -38,17 +40,16 @@ class WikipediaCategoryScraper < ExtraLoop::IterativeScraper
       extract(:ns).
       extract(:type).
       extract(:timestamp).
-    on(:data, proc { |results|
+    on(:data) do |results|
       puts "#{"\t" * (@options[:depth] - 2).abs }  #{@scraper.request_arguments[:params][:cmtitle]}"
       categories = results.select{ |record| record.ns === 14  }.each { |category| results.delete(category) }
 
-
       categories.each do |record|
         # Instanciate a sub scraper if the current depth is greater than zero and the category member is a sub category.
         WikipediaCategoryScraper.new(record.title, @options[:depth] - 1, @scraper.request_arguments[:params][:cmtitle] ).run unless @options[:depth] <= 0
       end
 
-    }).
+    end.
     continue_with(:cmcontinue, ['query-continue', 'categorymembers', 'cmcontinue'])
   end
 end

data/lib/extraloop/iterative_scraper.rb

@@ -89,8 +89,8 @@ module ExtraLoop
     # Returns itself.
     #
 
-    def set_iteration(param, *args)
-      #TODO: allow passing ranges as well as arrays
+    def set_iteration(param, *args, &block)
+      args << block if block
       if args.first.respond_to?(:map)
         @iteration_set = Array(args.first).map &:to_s
       else
@@ -110,7 +110,8 @@ module ExtraLoop
     #
     # Returns itself.
 
-    def continue_with(param, *extractor_args)
+    def continue_with(param, *extractor_args, &block)
+      extractor_args << block if block
       raise Exceptions::NonGetAsyncRequestNotYetImplemented.new "the #continue_with method currently requires the 'async' option to be set to false" if @options[:async]
 
       @continue_clause_args = extractor_args

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: extraloop
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-28 00:00:00.000000000Z
+date: 2012-01-30 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &19726880 !ruby/object:Gem::Requirement
+  requirement: &15579720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *19726880
+  version_requirements: *15579720
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &19726420 !ruby/object:Gem::Requirement
+  requirement: &15579260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *19726420
+  version_requirements: *15579260
 - !ruby/object:Gem::Dependency
   name: typhoeus
-  requirement: &19725960 !ruby/object:Gem::Requirement
+  requirement: &15578800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *19725960
+  version_requirements: *15578800
 - !ruby/object:Gem::Dependency
   name: logging
-  requirement: &19725500 !ruby/object:Gem::Requirement
+  requirement: &15578340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.6.1
   type: :runtime
   prerelease: false
-  version_requirements: *19725500
+  version_requirements: *15578340
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &19725040 !ruby/object:Gem::Requirement
+  requirement: &15577880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *19725040
+  version_requirements: *15577880
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &19724580 !ruby/object:Gem::Requirement
+  requirement: &15577420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 1.0.4
   type: :development
   prerelease: false
-  version_requirements: *19724580
+  version_requirements: *15577420
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &19724060 !ruby/object:Gem::Requirement
+  requirement: &15576960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,7 +87,7 @@ dependencies:
         version: 0.9.7.4
   type: :development
   prerelease: false
-  version_requirements: *19724060
+  version_requirements: *15576960
 description: A Ruby library for extracting data from websites and web based APIs.
   Supports most common document formats (i.e. HTML, XML, and JSON), and comes with
   a handy mechanism  for iterating over paginated datasets.