polipus 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0e3c15f4ab5c3b4eca8f4703b80d7d8fa74210c3
-  data.tar.gz: ae0dd32d81bbcbef350949e0c51fecaaf59ac8ff
+  metadata.gz: ac053145ab327da9f8807deb635759ebb81019f1
+  data.tar.gz: 4b9cbdf6a33783f9443ebe07c3c424e887c348ee
 SHA512:
-  metadata.gz: c3a57d1cbfbab4c77cf3b746c7b73eb84cf675e7998eb5979af6d17d997ee26781803e519eae50de6762590fc7287d09ccf4ac0d640dff40602b8a507a356de2
-  data.tar.gz: 743b128ee0dd8fefbbdd1bd7443afbc4a2c62dd2f3d3751d3d693719e9a8e2e145668ccd58945b77f5f780f50757a11248df2b1963c8e7cc35706e937627e5e0
+  metadata.gz: faedd0cc2b0c085fcda3337ab52b06abb7fe9b07f51df3ecff4bedfdbd97c40df939a12e5cd52253e6006aa22862f6bb372957c9eeb2a01496089652163eeb02
+  data.tar.gz: c62335cefa15204d304407a2248b62faa2e29aa060bf4513e42a984cb04b32ce5f11df69025c9a9760c6c611ff976bdf78cf9e82f4b8a47d1c9927ab4afa90fc

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.5.1 (2015-07-17)
+
+[Compare changes in gem](https://github.com/taganaka/polipus/compare/0.5.0...0.5.1)
+
+* Propagate user_data values from Redis message to fetched pages
+
 ## 0.5.0 (2015-07-08)
 
 [Compare changes in gem](https://github.com/taganaka/polipus/compare/0.4.0...0.5.0)

data/lib/polipus.rb

@@ -200,7 +200,7 @@ module Polipus
 
             execute_plugin 'on_before_download'
 
-            pages = http.fetch_pages(url, page.referer, page.depth)
+            pages = http.fetch_pages(url, page.referer, page.depth, page.user_data)
             if pages.count > 1
               rurls = pages.map { |e| e.url.to_s }.join(' --> ')
               @logger.info { "Got redirects! #{rurls}" }

data/lib/polipus/http.rb

@@ -35,37 +35,39 @@ module Polipus
     # Fetch a single Page from the response of an HTTP request to *url*.
     # Just gets the final destination page.
     #
-    def fetch_page(url, referer = nil, depth = nil)
-      fetch_pages(url, referer, depth).last
+    def fetch_page(url, referer = nil, depth = nil, user_data = nil)
+      fetch_pages(url, referer, depth, user_data).last
     end
 
     #
     # Create new Pages from the response of an HTTP request to *url*,
     # including redirects
     #
-    def fetch_pages(url, referer = nil, depth = nil)
+    def fetch_pages(url, referer = nil, depth = nil, user_data = nil)
       url = URI(url)
       pages = []
       get(url, referer) do |response, code, location, redirect_to, response_time|
         handle_compression response
-        pages << Page.new(location, body: response.body,
-                                    code: code,
-                                    headers: response.to_hash,
-                                    referer: referer,
-                                    depth: depth,
-                                    redirect_to: redirect_to,
-                                    response_time: response_time,
-                                    fetched_at: Time.now.to_i)
+        page = Page.new(location, body: response.body,
+                                  code: code,
+                                  headers: response.to_hash,
+                                  referer: referer,
+                                  depth: depth,
+                                  redirect_to: redirect_to,
+                                  response_time: response_time,
+                                  fetched_at: Time.now.to_i)
+        page.user_data = user_data unless user_data.nil?
+        pages << page
       end
-
       pages
     rescue *RESCUABLE_ERRORS => e
       if verbose?
         puts e.inspect
         puts e.backtrace
       end
-
-      [Page.new(url, error: e, referer: referer, depth: depth)]
+      page = Page.new(url, error: e, referer: referer, depth: depth)
+      page.user_data = user_data unless user_data.nil?
+      [page]
     end
 
     #

data/lib/polipus/version.rb

@@ -1,5 +1,5 @@
 # encoding: UTF-8
 module Polipus
-  VERSION = '0.5.0'
+  VERSION = '0.5.1'
   HOMEPAGE = 'https://github.com/taganaka/polipus'
 end

data/spec/polipus/http_spec.rb

@@ -8,11 +8,15 @@ describe Polipus::HTTP do
   it 'should download a page' do
     VCR.use_cassette('http_test') do
       http = Polipus::HTTP.new
-      page = http.fetch_page('http://sfbay.craigslist.org/apa/')
+      user_data = {
+        'mydata' => 'myvalue'
+      }
+      page = http.fetch_page('http://sfbay.craigslist.org/apa/', '', 0, user_data)
       expect(page).to be_an_instance_of(Polipus::Page)
       expect(page.doc.search('title').text.strip).to eq 'SF bay area apts/housing for rent classifieds  - craigslist'
       expect(page.fetched_at).not_to be_nil
       expect(page.fetched?).to be_truthy
+      expect(page.user_data).to eq user_data
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polipus
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Francesco Laurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-09 00:00:00.000000000 Z
+date: 2015-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -385,3 +385,4 @@ test_files:
 - spec/polipus/url_tracker_spec.rb
 - spec/polipus_spec.rb
 - spec/spec_helper.rb
+has_rdoc: