s3_download_by_date 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 96b132630b449b04ef3bed3c75d739df3839f90c
-  data.tar.gz: f987dbc5b81233507e2b302689053988e7708d4a
+  metadata.gz: bb808c568d4ae76a89af1cb9cee48822cae37732
+  data.tar.gz: cecc15bb7626a11de2f3f1a4ff2af0845d164d55
 SHA512:
-  metadata.gz: c3432fbd986b0ca273ac452a2763de08fd0d8793f4867f08fed8407e434dd7896dcafac8b2ec90545fe362609b32a8bc035e61154e26c24d6ff0301d29a95574
-  data.tar.gz: 5b65d15d652e92548d11f44ffb2cca9267e95ab0334f2a477b4bb78e6f497db937f341db7b1822c1af99dda3c731cdb3c99668659cd5034f62360460791f3e1d
+  metadata.gz: 300c9352d0d1fe0f6d00c7963be58b3a20ab146eb1031c9ceed1f67e239ff5b000d20a656f358c80bd068154f7c5e741865e03438fba833e24bc4f6354c04513
+  data.tar.gz: 6783cdbae30906fbf1e66ac810c65c898d34d3fa32a667d7e9c3433ab97ef7ab7bdfe291b436c0a91bbd0618060de8638af4ba0e7de15812989f5df90ee70045

data/lib/s3download.rb

@@ -0,0 +1,6 @@
+module S3download
+  VERSION   = "0.2.4"
+  ABOUT     = "S3download by date v#{VERSION} (c) #{Time.now.strftime("2014-%Y")} @innovia\nFind files & folders between dates on S3 given bucket and download them"
+  
+  autoload :Cli, 		  's3download/cli'
+end

data/lib/s3download/cli.rb

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+require 'aws-sdk'
+require 'thor'
+require 'chronic'
+require 'progressbar'
+require 'active_support/time'
+require 'fileutils'
+require 'json'
+
+class S3download::Cli < Thor
+  default_task :fetch
+  attr_accessor :s3, :debug, :files_found, :range, :target, :bucket, :from, :to
+  
+  desc 'fetch', 'Download S3 files by range (date)'
+  class_option  :bucket, :alias => 'b',                                                                      :banner => 'S3 Bucket Name'
+  class_option  :prefix, :alias => 'f',                                                                      :banner => 'Folder inside the specified bucket'
+  class_option  :from,                                      :default =>  Chronic.parse('today at 00:00:00'), :desc => 'From in a natural language date/time like yesterday, \'last week\', etc...' 
+  class_option  :to,                                        :default =>  Chronic.parse('today at 23:59:59'), :desc => 'To in a natural language date/time like yesterday, \'last week\', etc...' 
+  class_option  :save_to,                                                                                    :banner => 'Location of downloaded files'
+  class_option  :timezone,                                  :default => 'UTC',                               :desc => 'timezone to filter files by date [UTC] ex: "Eastern Time (US & Canada)"'
+  class_option  :debug,                                     :default => false, :type => :boolean            
+  class_option  :verbose,                                   :default => false, :type => :boolean            
+  def fetch
+    raise Thor::RequiredArgumentMissingError, 'You must supply an S3 bucket name' if options[:bucket].nil?
+    raise Thor::RequiredArgumentMissingError, 'You must supply a prefix (folder within the s3 bucket to filter by' if options[:prefix].nil?
+    raise Thor::RequiredArgumentMissingError, 'You must supply a location where to save the downloaded files to' if options[:save_to].nil?
+    init  
+
+    ProgressBar.new("Filter Files", bucket.count) do |pbar|
+      
+      bucket.each do |object|
+        if options[:verbose]
+          say "timezone: #{options[:timezone]}" 
+          say "object last modified: #{object.last_modified}"
+          say "object last modified in timezone: #{object.last_modified.in_time_zone(options[:timezone])}"
+          say "object falls in range: #{self.range}? => #{self.range.cover?(object.last_modified.in_time_zone(options[:timezone]))}"
+        end
+
+        if self.range.cover?(object.last_modified.in_time_zone(options[:timezone]))
+          self.files_found += 1
+          say("Downloading #{object.key} #{object.last_modified.in_time_zone(options[:timezone])}\n", color = :white) if options[:debug]
+          FileUtils.mkdir_p "#{target}/#{object.key.match(/(.+)\//)[1]}"
+
+          begin
+            File.open("#{target}/#{object.key}", "w") do |f|
+              f.write(object.read)
+            end
+          rescue Exception => e
+             say "Unable to save file: #{e}", color = :red
+          end
+        end
+        pbar.inc
+      end
+    end
+    say "Total files downloaded from S3: #{self.files_found}", color = :yellow
+  end
+
+  desc 'list_timezones', 'list timezones'
+  def list_timezones
+     say JSON.pretty_generate(ActiveSupport::TimeZone::MAPPING), color = :white
+  end
+
+  map ["-v", "--version"] => :version
+  desc "version", "version"
+  def version
+    say S3download::ABOUT
+  end
+
+  private
+  def init
+    aws_init
+    self.debug = true if options[:verbose]
+    self.bucket = self.s3.buckets[options[:bucket]].objects.with_prefix(options[:prefix])
+    self.from = Chronic.parse("#{options[:from]}").in_time_zone(options[:timezone])   
+    self.to = Chronic.parse("#{options[:to]}").in_time_zone(options[:timezone])
+    self.range = self.from..self.to
+    self.target = File.expand_path(options[:save_to])
+    self.files_found = 0
+
+    FileUtils.mkdir_p "#{self.target}"
+
+    File.open("#{self.target}/download_info.txt", "w") {|f| 
+      f.write("#{Time.now} - Downloaded bucket #{options[:bucket]}/#{options[:prefix]} from: #{from} - to #{to}")
+    }
+
+    say "S3 Search & Download", color = :white
+    say "--------------------\n", color = :white
+    say "Bucket: #{self.bucket}", color = :cyan
+    say "TimeZone: #{options[:timezone]}", color = :cyan
+    say "From: #{from}", color = :cyan
+    say "To: #{to}", color = :cyan
+    say("Download target: #{options[:save_to]}/#{options[:prefix]}", color = :cyan) if options[:debug]
+    say("Range: #{self.range}", color = :green) if options[:debug]
+  end
+
+  def aws_init
+    AWS.config({
+      :access_key_id => ENV['AWS_ACCESS_KEY'],
+      :secret_access_key => ENV['AWS_SECRET_KEY'],
+      :region => ENV['REGION'] || 'us-east-1'
+    })
+
+    self.s3 = AWS::S3.new
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_download_by_date
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Ami Mahloof
@@ -123,6 +123,8 @@ files:
 - LICENSE
 - README.md
 - bin/s3download
+- lib/s3download.rb
+- lib/s3download/cli.rb
 - s3_download_by_date.gemspec
 homepage: https://github.com/innovia/s3_download_by_date
 licenses: