s3_download_by_date 0.1.5 → 0.2

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODhlMmIwYWY4Y2EwYTNmMjI2YjQzYjFmMjkxMTY4MTcyN2Y5NzY3YQ==
-  data.tar.gz: !binary |-
-    YjE4ZDUwYzNkNDQ4Nzk5MTJiM2VkZDVlMzhlM2U5YWFlM2MxM2UyYw==
+SHA1:
+  metadata.gz: 42355945432c4611f873ccd0f06097a5efdbae0c
+  data.tar.gz: 2fc954497f44325e226566ecf787d2b07c797981
 SHA512:
-  metadata.gz: !binary |-
-    ZWMyOGE1YTMwY2FkZjcxMDJjMjAwYzdmMjY4ZmFmNDM1MjE0Yjk1NmQ3NmQ1
-    ODA2OTYyMDgzNDY3OTBiMjhiYTI3NTdkNGMyMjllNDM5NGI0MGUzMWNjMmEx
-    ZTMwMTBiYWRmMmQ5NjA2NjVmOGQ1ZDliZjBkOTE3NzZkNWE3OTQ=
-  data.tar.gz: !binary |-
-    YWM4ZThlZDVhYjQwYzE2Njg5NjUyZjkwMjU0ZDVjYTU5MWU4YWZkZDYwYjli
-    OWEwOGE0ZTc0ZTcwNDRmMjI2NDY4OWJkZjUyYmNjMDFmYjZmMTg0ZjA4YjBk
-    NzBlYTRhYzA1ZmM0ZWNlOTczMThkOWQ4MDFiY2JjMGYzMjU4MjE=
+  metadata.gz: 11bb79df712ad1f15b148f32324a955475ccfc142a020ef2e6cba7b6f86c0aaa8f6e019945ebda69fc59801613925f1fdf7b334d8d9910384a0e8d33d1ba0396
+  data.tar.gz: 2b32215a4c9d8f4627f8ab854ab737593adf2513e9d9d25d41c4e9a51130b18eaaed30e100b8ab75454d1a060d09e2d72db9d20f87db05e65f87b88748ed1d29

data/.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+*.gem

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Glide Talk, LTD.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,63 @@
+# S3 Download by date range [![Gem Version](https://badge.fury.io/rb/s3_download_by_date.svg)](http://badge.fury.io/rb/s3_download_by_date)
+
+S3 Download files by modifed date (Range)
+
+## Installation
+
+    $ gem install s3-download-by-date
+
+
+## Configuration
+add to your ~/.bash_profile (~/.zprofile if using ZSH)
+
+```bash
+export REGION='eu-west-1' (default to us-east-1)
+export AWS_ACCESS_KEY="YOUR AWS KEY ID"
+export AWS_SECRET_KEY="YOUR AWS SECRET KEY"
+
+```
+
+## Usage
+
+````bash
+s3download range --bucket=s3-bucket-name --prefix=folder_or_file_prefix --from='yesterday at noon' --to='today at noon' --save-to=~/Downloads
+````
+
+s3download uses [Chronic](https://github.com/mojombo/chronic) library to set the --from and --to
+
+
+Or download by timezone
+
+````bash
+s3download range --timezone='Eastern Time (US & Canada)'  ' --bucket=s3-bucket-name --prefix=folder_or_file_prefix --from='yesterday at noon' --to='today at noon' --save-to=~/Downloads 
+````
+
+**Getting a list of timezones strings:**
+
+ ````bash
+ s3download list_timezones 
+ ````
+
+ ````javascript
+  {
+  "International Date Line West": "Pacific/Midway",
+  "Midway Island": "Pacific/Midway",
+  "American Samoa": "Pacific/Pago_Pago",
+  "Hawaii": "Pacific/Honolulu",
+  "Alaska": "America/Juneau",
+  "Pacific Time (US & Canada)": "America/Los_Angeles"
+  .
+  .
+  .
+  }
+````
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## LICENSE
+s3_download_by_date is released under [MIT License](http://www.opensource.org/licenses/MIT)

data/bin/s3download

@@ -1,115 +1,20 @@
 #!/usr/bin/env ruby
-# encoding: UTF-8
-require 'aws-sdk'
+$:.unshift File.expand_path(File.join(File.dirname(__FILE__), %w[.. lib]))
 require 'thor'
-require 'chronic'
-require 'progressbar'
-require 'active_support/time'
-require 'fileutils'
-require 'json'
+require 'S3download'
 
-class S3Downloader < Thor
-  default_task :show_help
-  AWS.config({
-    :access_key_id => ENV['AWS_ACCESS_KEY'],
-    :secret_access_key => ENV['AWS_SECRET_KEY'],
-    :region => ENV['REGION'] || 'us-east-1'
-  })
-
-  @@s3 = AWS::S3.new
-
-  desc 'list_timezones', 'list timezones'
-  def list_timezones
-     puts JSON.pretty_generate(ActiveSupport::TimeZone::MAPPING)
-  end
-
-  desc 'show_help', 'show full help'
-  def show_help
-    puts <<-HELP
-Download S3 files by range (date)
-
-Setup:
-  export REGION='eu-west-1' (default to us-east-1)
-  export AWS_ACCESS_KEY="YOUR AWS KEY ID"
-  export AWS_SECRET_KEY="YOUR AWS SECRET KEY"
-
-Usage:
-  s3download range  --timezone='Eastern Time (US & Canada)' --bucket=S3 Bucket Name --prefix=folder inside the specified bucket --save-to=Location of downloaded files
-
-Options:
-  --bucket=S3 Bucket Name                                                               # S3 Bucket Name
-  --prefix=folder inside the specified bucket                                           # Folder inside the specified bucket
-  [--from=From in a natural language date/time like yesterday, last week, etc...]
-                                                                                        # Default: 2014-08-04 00:00:00 +0300
-  [--to=To in a natural language date/time like yesterday, last week, etc...]
-                                                                                        # Default: 2014-08-04 23:59:59 +0300
-  --timezone filter files by date [defaults to UTC] ex: "Eastern Time (US & Canada)
-  --save-to=Location of downloaded files                                                # The target directory where download files be stored
-
-Utility (Get a list of timezones strings):
-  s3download list_timezones 
-  {
-  "International Date Line West": "Pacific/Midway",
-  "Midway Island": "Pacific/Midway",
-  "American Samoa": "Pacific/Pago_Pago",
-  "Hawaii": "Pacific/Honolulu",
-  "Alaska": "America/Juneau",
-  "Pacific Time (US & Canada)": "America/Los_Angeles"
-  .
-  .
-  .
-  }
-    HELP
-  end
-
-  desc 'download', 'Download S3 files by range (date)'
-  method_option :bucket, :required => true,  :alias => 'b', :desc => 'S3 Bucket Name', :banner => 'S3 Bucket Name'
-  method_option :prefix, :required => true,  :alias => 'f', :desc => 'Folder inside the specified bucket', :banner => 'folder inside the specified bucket'
-  method_option :from, :default =>  Chronic.parse('today at 00:00:00'), :banner => 'From in a natural language date/time like yesterday, \'last week\', etc...' 
-  method_option :to, :default =>  Chronic.parse('today at 23:59:59'), :banner => 'To in a natural language date/time like yesterday, \'last week\', etc...' 
-  method_option :save_to, :required => true, :desc => 'the target directory where download files be stored', :banner => 'Location of downloaded files'
-  method_option :debug, :default => false, :type => :boolean,  :desc => 'the target directory where download files be stored'
-  method_option :timezone, :default => 'UTC', :banner => 'timezone to filter files by date [UTC] ex: "Eastern Time (US & Canada)"'
-  def range
-    timezone = options[:timezone]
-    puts "TimeZone: #{timezone}"
-    bucket = @@s3.buckets[options[:bucket]].objects.with_prefix(options[:prefix])
-    from = Chronic.parse("#{options[:from]}").in_time_zone(timezone)
-    to = Chronic.parse("#{options[:to]}").in_time_zone(timezone)
-    puts "From: #{from}"
-    puts "To: #{to}"
-    range = from..to
-    target = File.expand_path(options[:save_to])
-
-    say("download target: #{options[:save_to]}/#{options[:prefix]}", color=:cyan) if options[:debug]
-    say("Range: #{range}", color=:green) if options[:debug]
-
-    ProgressBar.new("Filter Files", bucket.count) do |pbar|
-      FileUtils.mkdir_p "#{target}"
-      File.open("#{target}/download_info.txt", "w") {|f| f.write("#{Time.now} - Downloaded bucket #{options[:bucket]}/#{options[:prefix]} from: #{from} - to #{to}")}
-      
-      bucket.each do |object|
-        # puts "timezone: #{timezone}" if options[:debug]
-        # puts "object last modified: #{object.last_modified}" if options[:debug]
-        # puts "object last modified in timezone: #{object.last_modified.in_time_zone(timezone)}" if options[:debug]
-        # puts "object falls in range: #{range}? => #{range.cover?(object.last_modified.in_time_zone(timezone))}"
-        if range.cover?(object.last_modified.in_time_zone(timezone))
-          say("Downloading #{object.key} #{object.last_modified.in_time_zone(timezone)}\n", color=:white) if options[:debug]
-          FileUtils.mkdir_p "#{target}/#{object.key.match(/(.+)\//)[1]}"
-
-          begin
-            File.open("#{target}/#{object.key}", "w") do |f|
-              f.write(object.read)
-            end
-          rescue Exception => e
-             puts "Unable to save file: #{e}"
-          end
-        end
-        pbar.inc
-      end
-    end
-  end
+def help
+	S3download::Cli.start(%w{help fetch}) 
+end
 
+begin
+  ENV["THOR_DEBUG"] = "1" 
+  S3download::Cli.start
+rescue Thor::RequiredArgumentMissingError => e
+  puts "\e[31mMissing Arguments: #{e}\e[0m\n\n"
+  help
+rescue Thor::InvocationError => e
+	puts "\e[31m#{e.to_s.gsub(/Usage:.+"/, '').chomp} but there's no such option\e[0m\n\n"
+	help
 end
 
-S3Downloader.start

data/s3_download_by_date.gemspec

@@ -0,0 +1,22 @@
+$:.push File.expand_path("../lib", __FILE__)
+require File.expand_path('../lib/s3download', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "s3_download_by_date"
+  s.version     = S3download::VERSION
+  s.authors     = ["Ami Mahloof"]
+  s.email       = "ami.mahloof@gmail.com"
+  s.homepage    = "https://github.com/innovia/s3_download_by_date"
+  s.summary     = "Download from S3 by date range"
+  s.description = "Download files by modified date range"
+  s.required_rubygems_version = ">= 1.3.6"
+  s.add_runtime_dependency 'thor', '~> 0.19', '>= 0.19.1'
+  s.add_runtime_dependency 'chronic', '~> 0.10', '>= 0.10.2'
+  s.add_runtime_dependency 'progressbar', '~> 0.21', '>= 0.21.0'
+  s.add_runtime_dependency 'aws-sdk', '~> 1.40', '>= 1.40.0'
+  s.add_runtime_dependency 'activesupport', '~> 4.1', '>= 4.1.4'
+  s.files = `git ls-files`.split($\).reject{|n| n =~ %r[png|gif\z]}.reject{|n| n =~ %r[^(test|spec|features)/]}
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.extra_rdoc_files = ['README.md', 'LICENSE']
+  s.license = 'MIT'
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_download_by_date
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: '0.2'
 platform: ruby
 authors:
 - Ami Mahloof
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-03 00:00:00.000000000 Z
+date: 2015-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -17,7 +17,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.19'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.19.1
   type: :runtime
@@ -27,7 +27,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.19'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.19.1
 - !ruby/object:Gem::Dependency
@@ -37,7 +37,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.10'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.2
   type: :runtime
@@ -47,7 +47,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.10'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.2
 - !ruby/object:Gem::Dependency
@@ -57,7 +57,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.21'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.21.0
   type: :runtime
@@ -67,7 +67,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.21'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.21.0
 - !ruby/object:Gem::Dependency
@@ -77,7 +77,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.40'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.40.0
   type: :runtime
@@ -87,7 +87,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.40'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.40.0
 - !ruby/object:Gem::Dependency
@@ -97,7 +97,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '4.1'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 4.1.4
   type: :runtime
@@ -107,7 +107,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '4.1'
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 4.1.4
 description: Download files by modified date range
@@ -115,9 +115,15 @@ email: ami.mahloof@gmail.com
 executables:
 - s3download
 extensions: []
-extra_rdoc_files: []
+extra_rdoc_files:
+- README.md
+- LICENSE
 files:
+- .gitignore
+- LICENSE
+- README.md
 - bin/s3download
+- s3_download_by_date.gemspec
 homepage: https://github.com/innovia/s3_download_by_date
 licenses:
 - MIT
@@ -128,17 +134,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: Download from S3 by date range