s3reamer 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44cb148d5be1f3d30b47a839a447490c31f2f054
-  data.tar.gz: 81ac44c4a694bbedf5ba05fa520ec2457e1b8b38
+  metadata.gz: 87b01cd26a019b04330bdce3ab30d8d55ae39c5d
+  data.tar.gz: 66eabdddfd8e19b19435d90b714bc6717b220c87
 SHA512:
-  metadata.gz: 574f10b3d0a0a955574ed45f78d32665e19e8d4f6ecca277409fa2fc4800df1b596fc06de36c58d8ebaa7b9715099b1ff9827dbbdd578fc534494fb665b5a60a
-  data.tar.gz: 1b36d85b66f01033f1b32f87a1da047408e720573947b1c0ed79fee4db7e4474369da8b3c113414b2e9470e715557334ede336398c8ac49b0e0200bd89ddcd7e
+  metadata.gz: f7e40f9d91ef30e5d8a9127bfa602f37a9e34003278cf97ffe425e10d379b1d60e4a0bcadf461aca0f1ca03c6cbbf1a879c9d42275d5dc2d5fe2150b4f37ede8
+  data.tar.gz: 6b647a4ccec6cf6734fbbc669d42f09bada1cbbce6b7c411cf4ccb313c87f69b3458b9236e707640363c0d28eca83c457b9f305ec1639c0587a1223a29d461eb

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Chris Mullins
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,65 @@
+# s3reamer
+
+*(pronounced "ss-three-mer")*
+
+Automatically upload files to S3 as they're created
+
+## Details
+
+s3reamer watches a provided directory and starts uploading newly created files as soon as they're created. It uses [`rb-inotify`](https://github.com/rb-inotify) for filesystem event notifications. As such, it only works on Linux systems.
+
+When it detects a new file, it immediately begins uploading it to S3 using the multipart upload API. It buffers each part (default and minimum 5MB) in memory.
+
+## Installing
+
+s3reamer is available on [Rubygems](https://rubygems.org). You can install it with:
+
+```
+$ gem install s3reamer
+```
+
+You can also add it to your Gemfile:
+
+```
+gem 's3reamer'
+```
+
+In order to access the executables, you might have to do something like:
+
+```
+rbenv rehash
+```
+
+depending on how your Ruby environment is configured.
+
+## Usage
+
+It's probably easiest to use the bundled ruby script:
+
+```
+$ s3reamer
+Usage: s3reamer DIRECTORY BUCKET_NAME [options]
+    -r, --region [AWS_REGION]
+        --file-read-timeout [SECONDS]
+                                     Number of seconds to wait for a file to grow before timing out (defaults to 10)
+        --reader-sleep-interval [SECONDS]
+                                     Number of seconds to sleep after an attempted read (defaults to 1)
+    -n, --parallelism [N]            Maximum number of concurrent files being processed (defaults to 4)
+    -v, --verbose
+    -c, --aws-credientials [PATH]    Path to AWS credentials file. Defaults to ~/.aws/credentials
+    -p [PROFILE],                    AWS credentials profile. Defaults to "default".
+        --aws-credentials-profile
+        --s3-prefix [PREFIX]         Prefix to append to all uploaded files. Defaults to empty string.
+```        
+
+It expects credentials to be in the standard AWS credentials conf format, and reads from `~/.aws/credentials` by default:
+
+```
+$ cat ~/.aws/credentials
+[default]
+aws_access_key_id = <access_key_id>
+aws_secret_access_key = <secret_access_key>
+region = us-east-1
+```
+
+It's important to note that s3reamer will **NOT** attempt to upload files that already exist (and don't change) when it's started.

data/bin/s3reamer

@@ -15,7 +15,8 @@ options = {
   log_level: Logger::INFO,
   credentials_file: ENV['HOME'] + '/.aws/credentials',
   credentials_profile: 'default',
-  prefix: ''
+  prefix: '',
+  log_file: STDOUT
 }
 opts = OptionParser.new do |opts|
   opts.banner = banner
@@ -58,6 +59,20 @@ opts = OptionParser.new do |opts|
     v = "#{v}/" unless v.end_with?("/")
     options[:prefix] = v
   end
+
+  opts.on("-l", "--log-file [FILE]", String,
+      "Where logs should be written to. Defaults to STDOUT.") do |v|
+    f = File.new(v)
+    if !f.file?
+      $stderr.puts "Specified log file is not a file: #{v}"
+      exit 1
+    elsif !File.exists?((log_dir = f.basename))
+      $stderr.puts "Log file's directory does not exist: #{log_dir}"
+      exit 1
+    end
+
+    options[:log_file] = File.open(v, File::WRONLY | File::APPEND)
+  end
 end
 
 if ARGV.length < 2

data/lib/s3reamer/directory_streamer.rb

@@ -14,14 +14,16 @@ module S3reamer
       pool_size: 4,
       log_level: Logger::INFO,
       reader_sleep_interval: 1,
-      reader_timeout: 10
+      reader_timeout: 10,
+      encryption_key: nil,
+      log_file: STDOUT
     }
 
     attr_reader :options
 
     def initialize(options = {})
       @options = DEFAULT_OPTIONS.merge(options)
-      @log = Logger.new(STDOUT)
+      @log = Logger.new(options[:log_file])
       @log.level = options[:log_level]
     end
 

data/lib/s3reamer/version.rb

@@ -1,3 +1,3 @@
 module S3reamer
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3reamer
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Christopher Mullins
@@ -103,6 +103,8 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - Gemfile
+- LICENSE
+- README.md
 - bin/s3reamer
 - lib/s3reamer.rb
 - lib/s3reamer/directory_streamer.rb