s3_download_by_date 0.1.2 → 0.1.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZTA1ZWNkY2JlNzk2Y2Y2MzBmYjIzMDFhMmFkODQ4MjM3Zjc1NDRkOA==
+    N2MzMDM4N2Y5NGJhZjEwY2U4MWRiZmRmNWI1NDljNjkyNzdiN2Y2Yg==
   data.tar.gz: !binary |-
-    MjAyOGVkNDhiMmE2MzYzN2MzOTMyMzc3MDJmNjM0ODgxNDBjMzJkZg==
+    MjgwOThiMGQ1MmEzNjJkZGE5ZDRlMDY0YzNjNzY0ZTQ4MWEyZDg1NA==
 SHA512:
   metadata.gz: !binary |-
-    YWZkZTJhMDRiMDlmNDVjM2EwOTBmN2VjMTAzYjI0NjQxYWRkMWQ1ZGZiYjU4
-    MDhhZGY1NTBiNjFlNzQ5MjA3OTZiNDEyMTYzZTM5YmIyMDA2NTFiODA0ODJi
-    OWM0OTNmYWJiYWQ1NWQ4NTk4NDFmMjdiN2IzZWQ1NThmYmI5N2E=
+    NzFjODU0ZGUzYWFmZTNkZTNjZGQxMzJkOWZmMmY5N2EyZTQ2NDcxNDY3ZGI4
+    NzkzZTE1N2QzYWUyNmFjMTkzMjRhYTFkZjhkYmI1NmM5NDBkNjM2N2FhYThl
+    YzFmMTgyYWQxMjlmMjdiMTZkYTE3MTI4ZDg1MThkMTU3M2YxMjc=
   data.tar.gz: !binary |-
-    MTkzMGMxYTIxZGJhOTRjMzFlZDhkOTI2YjE3OTdiNjRjMzUyNjAxNDEwYTVl
-    YjBkYTgxOTFjNzgwODg1ODg2ZjQ3MWYxZTU4YzFmZGNmNWM0MjBlMWM4ZGE1
-    MTk0MThjYzI3MzY2ODkzNTcyNDNkZjBmZDc0MTlhZjE5MzFmMTE=
+    ZGQ4NTE2Nzg0NjUwNzE0ZWYzMTljY2FjNmNmYWU4ZDllMDcwOTE3Mzc2OGQ2
+    MjJiY2E5MDQ1NTc1OGUzZjA1N2M5NDg1YmEyYTU2NzMyNWVjNzRmMzUyZjJj
+    NGYwYjNjMTQ1YTI0YmNhZDNjNDQ1MzZkMDZhYzk5NDU2MjEwN2Y=

data/bin/s3download

@@ -29,7 +29,7 @@ class S3Downloader < Thor
 Download S3 files by range (date)
 
 Usage:
-  s3download range --bucket=S3 Bucket Name --prefix=folder inside the specified bucket --save-to=Location of downloaded files
+  s3download range  --timezone='Eastern Time (US & Canada)' --bucket=S3 Bucket Name --prefix=folder inside the specified bucket --save-to=Location of downloaded files
 
 Options:
   --bucket=S3 Bucket Name                                                               # S3 Bucket Name
@@ -38,10 +38,10 @@ Options:
                                                                                         # Default: 2014-08-04 00:00:00 +0300
   [--to=To in a natural language date/time like yesterday, last week, etc...]
                                                                                         # Default: 2014-08-04 23:59:59 +0300
+  --timezone filter files by date [defaults to UTC] ex: "Eastern Time (US & Canada)
   --save-to=Location of downloaded files                                                # The target directory where download files be stored
-  --debug, [--no-debug]                                                               
 
-Utility:
+Utility (Get a list of timezones strings):
   s3download list_timezones 
   {
   "International Date Line West": "Pacific/Midway",
@@ -80,16 +80,24 @@ Utility:
     say("Range: #{range}", color=:green) if options[:debug]
 
     ProgressBar.new("Filter Files", bucket.count) do |pbar|
+      FileUtils.mkdir_p "#{target}"
+      File.open("#{target}/download_info.txt", "w") {|f| f.write("#{Time.now} - Downloaded bucket #{options[:bucket]}/#{options[:prefix]} from: #{from} - to #{to}")}
+      
       bucket.each do |object|
+        # puts "timezone: #{timezone}" if options[:debug]
+        # puts "object last modified: #{object.last_modified}" if options[:debug]
+        # puts "object last modified in timezone: #{object.last_modified.in_time_zone(timezone)}" if options[:debug]
+        # puts "object falls in range: #{range}? => #{range.cover?(object.last_modified.in_time_zone(timezone))}"
         if range.cover?(object.last_modified.in_time_zone(timezone))
           say("Downloading #{object.key} #{object.last_modified.in_time_zone(timezone)}\n", color=:white) if options[:debug]
-          FileUtils.mkdir_p "#{target}/#{range}/#{object.key.match(/(.+)\//)[1]}"
+          FileUtils.mkdir_p "#{target}/#{object.key.match(/(.+)\//)[1]}"
 
           begin
-            File.open("#{target}/#{range.gsub('/', ':')}/#{object.key}", "w") do |f|
+            File.open("#{target}/#{object.key}", "w") do |f|
               f.write(object.read)
             end
           rescue Exception => e
+             puts "Unable to save file: #{e}"
           end
         end
         pbar.inc

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_download_by_date
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Ami Mahloof
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-06 00:00:00.000000000 Z
+date: 2014-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor