s3reamer 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f2d922c65a2453f84342e5092f94aa9dab51959c
+  data.tar.gz: 8f56f94a47e7330a89e5624d801c542c23818ef9
+SHA512:
+  metadata.gz: 65d8f1fc518f44517bd1febe677d12f81ea0449456a04cf229c1e661c25eb618a28612aaae25e3332bad8563f19b4f442b6896854b4abc08e39e87d929df668e
+  data.tar.gz: fad7fd698d7f91f1d996527fc8e1cdc6645db27826642f1b79f04dcb98e323f1ef613b1cdce504c492ebb17be0cac529facf1da18846616c1b44c2fe713b5411

data/.gitignore

@@ -0,0 +1,38 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# IntelliJ
+*.iml

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/bin/s3reamer

@@ -0,0 +1,84 @@
+#!/usr/bin/ruby
+
+require 'optparse'
+require 'aws-sdk'
+
+require_relative 'directory_streamer'
+
+banner = "Usage: s3reamer DIRECTORY BUCKET_NAME [options]"
+if ARGV.length < 2
+  $stderr.puts banner
+  exit 1
+end
+
+dir = ARGV.shift
+if !File.exists?(dir)
+  $stderr.puts "Directory does not exist: #{dir}"
+  exit 1
+elsif !File.directory?(dir)
+  $stderr.puts "Not a directory: #{dir}"
+  exit 1
+end
+
+bucket_name = ARGV.shift
+
+options = {
+  region: 'us-east-1',
+  bucket_name: bucket_name,
+  directory: dir,
+  reader_timeout: 30,
+  reader_sleep_interval: 1,
+  pool_size: 4,
+  log_level: Logger::INFO,
+  credentials_file: ENV['HOME'] + '/.aws/credentials',
+  credentials_profile: 'default',
+  prefix: ''
+}
+OptionParser.new do |opts|
+  opts.banner = banner
+
+  opts.on("-r", "--region [AWS_REGION]") do |v|
+    options[:region] = v
+  end
+
+  opts.on("--file-read-timeout [SECONDS]", Integer,
+      "Number of seconds to wait for a file to grow before timing out (defaults to 10)") do |v|
+    options[:reader_timeout] = v
+  end
+
+  opts.on("--reader-sleep-interval [SECONDS]", Integer,
+      "Number of seconds to sleep after an attempted read (defaults to 1)") do |v|
+    options[:reader_sleep_interval] = v
+  end
+
+  opts.on("-n", "--parallelism [N]", Integer,
+      "Maximum number of concurrent files being processed (defaults to 4)") do |v|
+    options[:pool_size] = v
+  end
+
+  opts.on("-v", "--verbose") do
+    options[:log_level] = Logger::DEBUG
+  end
+
+  opts.on("-c", "--aws-credientials [PATH]", String,
+      "Path to AWS credentials file. Defaults to ~/.aws/credentials") do |v|
+    options[:credentials_file] = v
+  end
+
+  opts.on("-p", "--aws-credentials-profile [PROFILE]", String,
+      "AWS credentials profile. Defaults to \"default\".") do |v|
+    options[:credentials_profile] = v
+  end
+
+  opts.on("--s3-prefix [PREFIX]", String,
+      "Prefix to append to all uploaded files. Defaults to empty string.") do |v|
+    v = "#{v}/" unless v.end_with?("/")
+    options[:prefix] = v
+  end
+end.parse!
+
+credentials = Aws::SharedCredentials.new(path: options[:credentials_file], profile_name: options[:credentials_profile])
+s3 = Aws::S3::Resource.new(region: options[:region], credentials: credentials)
+bucket = s3.bucket(options[:bucket_name])
+watcher = S3reamer::DirectoryStreamer.new(options)
+watcher.stream_directory(directory: options[:directory], bucket: bucket)

data/lib/s3reamer/directory_streamer.rb

@@ -0,0 +1,125 @@
+require 'set'
+require 'logger'
+require 'rb-inotify'
+require 'aws-sdk'
+require 'thread/pool'
+require 'concurrent'
+require 'thread_safe'
+
+require 's3reamer/s3_write_stream'
+
+module S3reamer
+  class DirectoryStreamer
+    DEFAULT_OPTIONS = {
+      pool_size: 4,
+      log_level: Logger::INFO,
+      reader_sleep_interval: 1,
+      reader_timeout: 10
+    }
+
+    attr_reader :options
+
+    def initialize(options = {})
+      @options = DEFAULT_OPTIONS.merge(options)
+      @log = Logger.new(STDOUT)
+      @log.level = options[:log_level]
+    end
+
+    def stream_directory(directory:, bucket:)
+      file_statuses = ThreadSafe::Hash.new
+      dir_watch = INotify::Notifier.new
+      pool = Thread.pool(options[:pool_size])
+
+      dir_watch.watch(directory, :open, :close, :recursive) do |e|
+        filename = e.absolute_name
+
+        log.debug "Events #{e.flags.inspect} received for: #{filename}"
+
+        # Don't process directories
+        next unless File.exists?(filename) and !File.directory?(filename)
+
+        # If this is an "open" event, we should only process it if we haven't
+        # already started on this file.
+        next if e.flags.include?(:open) and file_statuses.include?(filename)
+
+        # If this is a "close" event, we should update the status to inform the
+        # worker thread
+        if e.flags.include?(:close) and file_statuses.include?(filename)
+          file_statuses[filename] = :close
+          next
+        end
+
+        log.info "File opened: #{filename}"
+        file_statuses[filename] = :open
+
+        pool.process {
+          log.debug "Starting process for: #{filename}"
+
+          begin
+            prefix = Pathname.new(filename)
+            prefix = prefix.relative_path_from(Pathname.new(directory))
+            prefix = "#{options[:prefix]}#{prefix}"
+
+            obj = bucket.object(prefix)
+            io = S3reamer::S3WriteStream.new(obj)
+          rescue Exception => e
+            log.error "Error initializing S3 streamer: #{e}\n#{e.backtrace.join("\n")}"
+            raise e
+          end
+
+          log.debug "Initialized S3 streamer"
+
+          open(filename) do |file|
+            stopped = false
+            size = 0
+            last_successful_read = Time.now
+
+            # Start with bytes_read != 0 to force at least one read of the file.
+            # This addresses the race condition caused by files being opened and
+            # closed quickly.
+            bytes_read = -1
+
+            # Go until the file has closed, or until we've not seen any new
+            # bytes written to the file past some threshold (specified by
+            # options[:reader_timeout]).
+            while (file_statuses[filename] == :open || bytes_read != 0) &&
+              (last_successful_read + options[:reader_timeout]) > Time.now
+
+              b = file.read
+              bytes_read = b.length
+              io.write(b)
+
+              # If we read any bytes, reset the time at which we last saw new
+              # bytes in the file. This prevents the read timeout condition from
+              # triggering.
+              if bytes_read > 0
+                log.debug "Read #{bytes_read} bytes: #{filename}"
+                last_successful_read = Time.now
+              end
+
+              sleep options[:reader_sleep_interval] unless file_statuses[filename] != :open
+            end
+
+            log.info "File closed. Completing S3 upload: #{filename}"
+          end
+
+          begin
+            io.close
+          rescue Exception => e
+            log.error "Error completing S3 upload: #{e}:\n#{e.backtrace.join("\n")}"
+          end
+
+          file_statuses.delete(filename)
+        }
+      end
+
+      dir_watch.run
+      pool.shutdown
+    end
+
+    private
+      def log
+        @log
+      end
+  end
+end

data/lib/s3reamer/s3_write_stream.rb

@@ -0,0 +1,33 @@
+module S3reamer
+  class S3WriteStream
+    DEFAULT_OPTIONS = {
+      chunk_size: 5 * 1024 * 1024
+    }
+
+    def initialize(object, options = {})
+      @buffer = String.new
+      @options = DEFAULT_OPTIONS.merge(options)
+      @multipart_upload = object.initiate_multipart_upload
+      @closed = false
+    end
+
+    def write(data)
+      raise RuntimeError.new("Illegal state: cannot write after close.") if @closed
+
+      @buffer << data
+      flush if @buffer.length >= @options[:chunk_size]
+    end
+
+    def flush
+      part = @multipart_upload.part(@multipart_upload.parts.count + 1)
+      part.upload(body: @buffer)
+      @buffer.clear
+    end
+
+    def close
+      flush unless @buffer.empty?
+      @multipart_upload.complete(compute_parts: true)
+      @closed = true
+    end
+  end
+end

data/lib/s3reamer/version.rb

@@ -0,0 +1,3 @@
+module S3reamer
+  VERSION = '0.0.1'
+end

data/lib/s3reamer.rb

@@ -0,0 +1 @@
+require 's3reamer/directory_streamer'

data/s3reamer.gemspec

@@ -0,0 +1,33 @@
+$:.push File.expand_path('../lib', __FILE__)
+
+require "s3reamer/version"
+
+Gem::Specification.new do |gem|
+  gem.name    = 's3reamer'
+  gem.version = S3reamer::VERSION
+
+  gem.summary = "Automatically upload files to S3 as they're created"
+
+  gem.authors  = ['Christopher Mullins']
+  gem.email    = 'chris@sidoh.org'
+  gem.homepage = 'http://github.com/sidoh/s3reamer'
+
+  gem.add_dependency 'rake'
+  gem.add_dependency 'aws-sdk', '~> 2'
+  gem.add_dependency 'rb-inotify', '~> 0.9'
+  gem.add_dependency 'thread', '~> 0.2'
+  gem.add_dependency 'thread_safe', '~> 0.3'
+  gem.add_dependency 'concurrent-ruby', '~> 1'
+
+  ignores  = File.readlines(".gitignore").grep(/\S+/).map(&:chomp)
+  dotfiles = %w[.gitignore]
+
+  all_files_without_ignores = Dir["**/*"].reject { |f|
+    File.directory?(f) || ignores.any? { |i| File.fnmatch(i, f) }
+  }
+
+  gem.files = (all_files_without_ignores + dotfiles).sort
+  gem.executables = ["s3reamer"]
+
+  gem.require_path = "lib"
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: s3reamer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Christopher Mullins
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rb-inotify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: thread_safe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+description: 
+email: chris@sidoh.org
+executables:
+- s3reamer
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- bin/s3reamer
+- lib/s3reamer.rb
+- lib/s3reamer/directory_streamer.rb
+- lib/s3reamer/s3_write_stream.rb
+- lib/s3reamer/version.rb
+- s3reamer.gemspec
+homepage: http://github.com/sidoh/s3reamer
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Automatically upload files to S3 as they're created
+test_files: []