s3_download_by_date 0.2.5 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68310667f45c318e02c4c93ab60c894782d81db9
-  data.tar.gz: ed42d5bbb1b26a3f0bdc439111c7882c74025334
+  metadata.gz: d0cb9fe9de33e920a98d32abff4b1619b20b2634
+  data.tar.gz: e5c27d16c8924369f1d6085ba7306e79858736fa
 SHA512:
-  metadata.gz: 0f7c905d178f11b8eb1dce2d711dcc2a9a4e6c1a1321cb8d83cbf74ec36f87a370758c078b11fe5aad07e76292ce4673c1f859c02ba18110aa4b42dd3d400db0
-  data.tar.gz: dc201134e2a781ae61144e75eb693ffd89f5f6141cb62762966b04c4164cafe84663a430bd33c9d28361f8e80c37b4f3a04ffba62bafebbd158a6ebd4c244cfb
+  metadata.gz: b5e1dee76df19e6da56da6aa26294fc0849d455f31a61baf183ab8358b135008d9204c083d6ae6efd62b81d9976ffb4ce2fa61a881067999a475c1ff8d1e68d7
+  data.tar.gz: f5c46c57cf2d81c2bfaa3c988442994ea4045fac7a85f671476fbd426b55d1c6dad022fe00c913cee0b030e93ad04c79df453e36c23e046418fed2d1fff80f18

data/lib/s3download/cli.rb

@@ -10,7 +10,7 @@ require 'json'
 
 class S3download::Cli < Thor
   default_task :fetch
-  attr_accessor :s3, :debug, :files_found, :range, :target, :bucket, :from, :to
+  attr_accessor :s3, :debug, :files_found, :range, :target, :bucket, :from, :to, :prefix
   
   desc 'fetch', 'Download S3 files by range (date)'
   class_option  :bucket, :alias => 'b',                                                                      :banner => 'S3 Bucket Name'
@@ -23,12 +23,21 @@ class S3download::Cli < Thor
   class_option  :verbose,                                   :default => false, :type => :boolean            
   def fetch
     raise Thor::RequiredArgumentMissingError, 'You must supply an S3 bucket name' if options[:bucket].nil?
-    raise Thor::RequiredArgumentMissingError, 'You must supply a prefix (folder within the s3 bucket to filter by' if options[:prefix].nil?
     raise Thor::RequiredArgumentMissingError, 'You must supply a location where to save the downloaded files to' if options[:save_to].nil?
+    
+    if options[:prefix].nil?
+      say "You did not pass a --prefix option, this will scan the entire bucket and can be very slow if there too many files in that bucket\n", color = :yellow
+      say "If you're trying to script this and bypass this question, use --prefix '' (empty string)\n", color = :yellow
+      response = ask "Continue without a prefix? (y/n)", color = :white
+      exit 0 if response == 'n'
+      self.prefix  = ''
+    else
+      self.prefix = options[:prefix]
+    end
+
     init  
 
-    ProgressBar.new("Filter Files", bucket.count) do |pbar|
-      
+    ProgressBar.new("Looking for files: ", bucket.count) do |pbar|      
       bucket.each do |object|
         if options[:verbose]
           say "timezone: #{options[:timezone]}" 
@@ -71,7 +80,7 @@ class S3download::Cli < Thor
   def init
     aws_init
     self.debug = true if options[:verbose]
-    self.bucket = self.s3.buckets[options[:bucket]].objects.with_prefix(options[:prefix])
+    self.bucket = self.s3.buckets[options[:bucket]].objects.with_prefix(self.prefix)
     self.from = Chronic.parse("#{options[:from]}").in_time_zone(options[:timezone])   
     self.to = Chronic.parse("#{options[:to]}").in_time_zone(options[:timezone])
     self.range = self.from..self.to
@@ -81,26 +90,33 @@ class S3download::Cli < Thor
     FileUtils.mkdir_p "#{self.target}"
 
     File.open("#{self.target}/download_info.txt", "w") {|f| 
-      f.write("#{Time.now} - Downloaded bucket #{options[:bucket]}/#{options[:prefix]} from: #{from} - to #{to}")
+      f.write("#{Time.now} - Downloaded bucket #{options[:bucket]}/#{self.prefix} from: #{from} - to #{to}")
     }
 
     say "S3 Search & Download", color = :white
     say "--------------------\n", color = :white
-    say "Bucket: #{self.bucket}", color = :cyan
+    say "Bucket: #{options[:bucket]}", color = :cyan
+    say "Prefix: #{self.prefix}", color = :cyan
     say "TimeZone: #{options[:timezone]}", color = :cyan
     say "From: #{from}", color = :cyan
     say "To: #{to}", color = :cyan
-    say("Download target: #{options[:save_to]}/#{options[:prefix]}", color = :cyan) if options[:debug]
+    say("Download target: #{options[:save_to]}/#{self.prefix}", color = :cyan) if options[:debug]
     say("Range: #{self.range}", color = :green) if options[:debug]
   end
 
   def aws_init
-    AWS.config({
+    config = {
       :access_key_id => ENV['AWS_ACCESS_KEY'],
       :secret_access_key => ENV['AWS_SECRET_KEY'],
       :region => ENV['REGION'] || 'us-east-1'
-    })
+    }
+
+    if config[:access_key_id].nil? || config[:secret_access_key].nil?
+      say "You must set your AWS Secret Key and AWS Access Key Id in your environment variables", color = :red
+      say "export REGION='eu-west-1' (default to us-east-1)\nexport AWS_ACCESS_KEY=\"YOUR AWS KEY ID\"\nexport AWS_SECRET_KEY=\"YOUR AWS SECRET KEY\"", color = :green
+      exit 1
+    end
 
-    self.s3 = AWS::S3.new
+    self.s3 = AWS::S3.new(config)
   end
 end

data/lib/s3download.rb

@@ -1,5 +1,5 @@
 module S3download
-  VERSION   = "0.2.5"
+  VERSION   = "0.2.6"
   ABOUT     = "S3download by date v#{VERSION} (c) #{Time.now.strftime("2014-%Y")} @innovia\nFind files & folders between dates on S3 given bucket and download them"
   
   autoload :Cli, 		  's3download/cli'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_download_by_date
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.6
 platform: ruby
 authors:
 - Ami Mahloof