upton 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0bc8fddf34dc974bde7491e7dd311eb09b5d393e
-  data.tar.gz: b8a8010408cd715b010406163cd14e45045af2d6
+  metadata.gz: d9de3851524b4f26a558ad8400edf24d6bc39b49
+  data.tar.gz: 3183dc8c0f75a659722354e3b58f72ce1b55661d
 SHA512:
-  metadata.gz: 0c5cdda936dcaf7a045afbc6cb317fc463191823a13d585732717f6ddfb3d4970a94c51df0324a343022c938702ca8b0fdbbf9e8b54fb0cc5fafec1dd8af8276
-  data.tar.gz: e5f2bd0c9f9ba843607b0ac7816c84df21cc6acbb0de13ec5918e3edb866fa41d7e6e9b39d4d0af7ea74c0ebf4628240ee783612edc3370842f860039ccc6465
+  metadata.gz: 8372d26f5f55208c657331b8778d22da42a61d437625b73bda5e85d757c6eb255ef128f47ced6bd2cc7a1cb65b3616cd7d4b93d3c443499786b0e1bef8c8cca6
+  data.tar.gz: b3e6c479b6b03e82dab3164cab8d27b88b7fa935e7e9d58317df6a11c7af3d419c65b7205133a3240f78e8cadc31d096fabd6ee57610ba5d552fbcea21d52e4a

data/lib/upton.rb

@@ -111,8 +111,9 @@ module Upton
 
       # Folder name for stashes, if you want them to be stored somewhere else,
       # e.g. under /tmp.
-      @stash_folder ||= "stashes"
-      FileUtils.mkdir_p(@stash_folder) unless Dir.exists?(@stash_folder)
+      if @stash_folder
+        FileUtils.mkdir_p(@stash_folder) unless Dir.exists?(@stash_folder)
+      end
     end
 
     ##
@@ -207,7 +208,17 @@ module Upton
     ##
     def get_page(url, stash=false, options={})
       return EMPTY_STRING if url.empty?
-      resp_and_cache = Downloader.new(url, {:cache => stash, :verbose => @verbose}.merge(options)).get
+      global_options = {
+        :cache => stash,
+        :verbose => @verbose
+      }
+      if @stash_folder
+        global_options.merge!({
+            :cache_location => @stash_folder,
+            :readable_filenames => true
+        })
+      end
+      resp_and_cache = Downloader.new(url, global_options.merge(options)).get
       if resp_and_cache[:from_resource]
         puts "sleeping #{@sleep_time_between_requests} secs" if @verbose
         sleep @sleep_time_between_requests

data/spec/upton_spec.rb

@@ -170,11 +170,29 @@ describe Upton do
   it "should sleep after uncached requests" do
     stub_request(:get, "www.example.com")
     u = Upton::Scraper.new("http://www.example.com", '.whatever')
+    u.sleep_time_between_requests = 1 #don't sleep too long, that's annoying.
     u.should_receive(:sleep)
     stub = stub_request(:get, "http://www.example.com")
     u.scrape
   end
 
+  it "should save to the designated stash folder" do
+    custom_cache_folder = "#{Dir.tmpdir}/upton/test"
+    FileUtils.rm_rf(custom_cache_folder)
+    stub_request(:get, "www.example.com").
+      to_return(:body => '', :status => 200)
+
+    u = Upton::Scraper.new("http://www.example.com", '.whatever')
+    u.stash_folder = custom_cache_folder
+    u.debug = true
+    u.scrape do
+      1+1
+    end
+    puts [custom_cache_folder, custom_cache_folder + "/*", Dir.glob(custom_cache_folder)].inspect
+    files = Dir.glob(custom_cache_folder)
+    expect(files).not_to be_empty
+  end
+
   it "should be silent if verbose if false" do
     pending
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upton
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - Jeremy B. Merrill
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-15 00:00:00.000000000 Z
+date: 2013-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack