better_s3 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e9070018615cde079bd13bdedecdf4f7e4eb0345
+  data.tar.gz: 5f56a68976186bda80105a0eccee454c3df78a33
+SHA512:
+  metadata.gz: 67018e1f8ce3ab64b15ffcc27c7cee11ec4f67a4a8a5a10b195d08996c7144e3de89391f614e370b9e22ede2f862b6299d195ae4b41dd0d60411564c04757a92
+  data.tar.gz: 72ee474a7b6a221c27f7087db93293ba280c69d0abee324b2d6294d83b5eb0979edcc4a5f311dbc05241c347122cb39bfcf644e363053d0129cd935d815f47a5

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rubocop.yml

@@ -0,0 +1,87 @@
+inherit_from: .rubocop_todo.yml
+
+Style/MutableConstant:
+  Enabled: false
+
+Lint/UselessAssignment:
+  Enabled: true
+
+Lint/UnusedMethodArgument:
+  Enabled: true
+
+Lint/UnusedBlockArgument:
+  Enabled: true
+
+Metrics/AbcSize:
+  Exclude:
+    - spec/**/*
+
+Style/DotPosition:
+  Enabled: true
+  EnforcedStyle: trailing
+
+Style/PercentLiteralDelimiters:
+  PreferredDelimiters:
+    '%':  ()    # interpolated String
+    '%i': '[]'  # Array of symbols
+    '%q': ()    # Single quoted string
+    '%Q': ()    # Double quoted string
+    '%r': '{}'  # Regex
+    '%s': ()    # Symbol
+    '%w': '[]'  # Array of strings
+    '%W': '[]'  # Array of strings, interpolated
+    '%x': ()    # shell command
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/CollectionMethods:
+  Enabled: true
+  # Mapping from undesired method to desired_method
+  # e.g. use `detect` over `find`:
+  PreferredMethods:
+    find: 'detect'
+    find_all: 'select'
+
+Style/AlignHash:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
+
+Style/AndOr:
+  # Whether `and` and `or` are banned only in conditionals (conditionals)
+  # or completely (always).
+  EnforcedStyle: conditionals # and/or are sometimes used for flow control.
+
+Style/BlockDelimiters:
+  EnforcedStyle: semantic
+
+Style/HashSyntax:
+  EnforcedStyle: ruby19_no_mixed_keys
+
+Style/NumericLiterals:
+  MinDigits: 5
+
+Style/SpaceBeforeFirstArg:
+  Enabled: false
+
+# Configuration parameters: EnforcedStyleForMultiline, SupportedStyles.
+Style/TrailingCommaInLiteral:
+  EnforcedStyleForMultiline: comma
+
+Style/SingleLineBlockParams:
+  Enabled: false
+
+Style/EachWithObject:
+  Enabled: false
+
+Style/Lambda:
+  Enabled: false
+
+AllCops:
+  Exclude:
+    - spec/support/mocks.rb
+    - better_s3.gemspec
+    - db/**/*.rb
+    - rakefile.rb
+    - spec/spec_helper.rb
+    - vendor/**/*  # NOT RESPONSIBLE FOR THIS CODE

data/.rubocop_todo.yml

@@ -0,0 +1,20 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2016-10-11 20:58:39 +0000 using RuboCop version 0.42.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 13
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 107
+
+# Offense count: 1
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/better_s3.rb'

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Referly
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# better_s3
+An idiomatic pattern for working with s3 in Ruby
+
+[![CircleCI](https://circleci.com/gh/Referly/better_s3.svg?style=svg)](https://circleci.com/gh/Referly/better_s3)
+
+## Get the contents of a remote file
+
+```ruby
+require "better_s3"
+s3 = BetterS3.new
+s3.get "some-filename.txt"
+ => "filescontentshere"
+```
+
+## Delete local copy of remote file
+
+When downloading files from s3 there are benefits associated with storing a copy locally (most notably it enables
+retry during download). As such BetterS3 stores a copy of files you get locally. If you want to delete one ...
+
+```ruby
+require "better_s3"
+s3 = BetterS3.new
+s3.delete_local_file_copy "some-filename.txt"
+```
+
+## Put an arbitrary String to a remote file
+
+```ruby
+require "better_s3"
+s3 = BetterS3.new
+content = { foo: "bar" }.to_json
+s3.put content, "filename.ext"
+```
+
+## Configuration
+
+To configure BetterS3 use the configuration block pattern
+
+```ruby
+require "better_s3"
+BetterS3.configure do |config|
+    # The name of the S3 bucket you plan to interact with
+    config.bucket = "your-bucket-name"
+    
+    # The local path you want to use for temporary file
+    # storage
+    config.tmp_dir_path = "../tmp"
+    
+    # If you want to hardcode which region of SQS should be used then you can set this option. It is recommended
+    # to use the environment variable ENV["AWS_REGION"] instead
+    config.region = "us-west-2"
+    
+    # for aws_access_key_id and aws_secret_access_key you can set them in this fashion, but it is strongly
+    # recommended that you just use the environment variables instead: ENV["AWS_ACCESS_KEY_ID"], 
+    # ENV["AWS_SECRET_ACCESS_KEY"]
+end
+```

data/better_s3.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.push File.expand_path("../lib", __FILE__)
+require "better_s3/version"
+
+Gem::Specification.new do |s|
+  s.name        = "better_s3"
+  s.version     = BetterS3::VERSION
+  s.date        = BetterS3::VERSION_DATE
+  s.license     = "MIT"
+  s.summary     = "Idiomatic S3"
+  s.description = "Interact with S3 buckets in an idiomatic fashion."
+  s.authors     = ["Courtland Caldwell"]
+  s.email       = "engineering@mattermark.com"
+  s.files         = `git ls-files`.split("\n") - %w[Gemfile Gemfile.lock]
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.homepage =
+      "https://github.com/Referly/better_s3"
+  s.add_runtime_dependency "aws-sdk", "~> 2" # Apache https://github.com/aws/aws-sdk-ruby/blob/master/LICENSE.txt
+  s.add_development_dependency "rspec", "~> 3.2" # MIT https://github.com/rspec/rspec/blob/master/License.txt
+  s.add_development_dependency "rb-readline", "~> 0.5", ">= 0.5.3" # BSD (content is BSD) https://github.com/ConnorAtherton/rb-readline/blob/master/LICENSE
+  s.add_development_dependency "byebug", "~> 3.5" # BSD (content is BSD) https://github.com/deivid-rodriguez/byebug/blob/master/LICENSE
+  s.add_development_dependency "simplecov", "~> 0.10" # MIT https://github.com/colszowka/simplecov/blob/master/MIT-LICENSE
+  s.add_development_dependency "rubocop", "~> 0.31" # Create Commons Attribution-NonCommerical https://github.com/bbatsov/rubocop/blob/master/LICENSE.txt
+  s.add_development_dependency "rspec_junit_formatter", "~> 0.2" # MIT https://github.com/sj26/rspec_junit_formatter/blob/master/LICENSE
+end

data/circle.yml

@@ -0,0 +1,15 @@
+machine:
+
+  timezone:
+    America/Los_Angeles # Set the timezone
+
+  # Version of ruby to use
+  ruby:
+    version:
+      2.1.5
+
+test:
+  override:
+    - bundle exec rubocop
+    - mkdir -p $CIRCLE_TEST_REPORTS/rspec
+    - bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/rspec.xml spec --format progress

data/lib/better_s3/configuration.rb

@@ -0,0 +1,29 @@
+class BetterS3
+  # Provides configuration management for the BetterS3 gem
+  class Configuration
+    attr_accessor :bucket,
+                  :tmp_dir_path,
+                  :aws_access_key_id,
+                  :aws_secret_access_key
+
+    def initialize
+      @aws_access_key_id      = ENV["AWS_ACCESS_KEY_ID"]
+      @aws_secret_access_key  = ENV["AWS_SECRET_ACCESS_KEY"]
+      @region                 = ENV["AWS_REGION"] || "us-east-1"
+    end
+
+    def configure_aws
+      return configure_region_only unless aws_secret_access_key && aws_access_key_id
+      Aws.config.update(
+        region:      region,
+        credentials: Aws::Credentials.new(aws_access_key_id, aws_secret_access_key)
+      )
+    end
+
+    private
+
+    def configure_region_only
+      Aws.config.update(region: region)
+    end
+  end
+end

data/lib/better_s3/version.rb

@@ -0,0 +1,4 @@
+class BetterS3
+  VERSION = "0.1.0"
+  VERSION_DATE = "2016-10-11"
+end

data/lib/better_s3.rb

@@ -0,0 +1,90 @@
+require "aws-sdk"
+require_relative "better_s3/configuration"
+require_relative "better_s3/version"
+class BetterS3
+  class << self
+    # Allows the user to set configuration options
+    #  by yielding the configuration block
+    #
+    # @param opts [Hash] an optional hash of options, supported options are `reset: true`
+    # @param block [Block] an optional configuration block
+    # @return [Configuration] the current configuration object
+    def configure(opts = {}, &_block)
+      @configuration = nil if opts.key?(:reset) && opts[:reset]
+      yield(configuration) if block_given?
+
+      configuration.configure_aws
+      configuration
+    end
+
+    # Returns the singleton class's configuration object
+    #
+    # @return [Configuration] the current configuration object
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configured?
+      !@configuration.nil?
+    end
+
+    def logger
+      LincolnLogger.logger
+    end
+  end
+
+  attr_accessor :s3, :_payload_body, :file_copied
+  def s3
+    @s3 ||= Aws::S3::Client.new
+  end
+
+  # Returns the body of the payload, retrieved from S3
+  #
+  # @param remote_file_name [String] the s3 bucket key
+  # @return [String] the payload body from S3
+  def get(remote_file_name)
+    copy_file_from_s3 remote_file_name unless file_copied
+    @_payload_body ||= File.read(full_file_path(remote_file_name))
+  end
+
+  # Puts a Ruby hash to s3 as a file
+  #
+  # @param str [String] the string to be pushed to s3
+  # @param remote_file_name [String] the s3 bucket object key where the file should be put
+  def put(str, remote_file_name)
+    push_object_to_s3 str, remote_file_name
+  end
+
+  # Attempts to delete the local copy of the file, if it exists
+  #
+  # @param remote_file_name [String] the name of the file in the s3 bucket (the s3 object key)
+  def delete_local_file_copy(remote_file_name)
+    File.delete(full_file_path(remote_file_name)) if File.exist?(full_file_path(remote_file_name))
+  end
+
+  private
+
+  # Puts a Ruby hash to s3 as a file
+  #
+  # @param hsh [String] the content to be put into s3 (it really should be JSON)
+  # @param remote_file_name [String] the s3 bucket object key where the file should be put
+  def push_object_to_s3(hsh, remote_file_name)
+    s3.put_object(bucket: BetterS3.configuration.bucket.to_s,
+                  key:    remote_file_name.to_s,
+                  body:   hsh)
+  end
+
+  def copy_file_from_s3(remote_file_name)
+    s3.get_object({ bucket: BetterS3.configuration.bucket.to_s, key: remote_file_name.to_s },
+                  target: full_file_path(remote_file_name))
+    @file_copied = true
+  end
+
+  # Returns the path on the server where the s3 file is downloaded locally
+  #
+  # @param remote_file_name [String] the name of the file in the s3 bucket (the s3 object key)
+  # @return [String] the complete path to location of the temporary local copy of the file
+  def full_file_path(remote_file_name)
+    "#{BetterS3.configuration.tmp_dir_path}/#{remote_file_name}"
+  end
+end

data/spec/better_s3_spec.rb

@@ -0,0 +1,69 @@
+require "spec_helper"
+
+describe BetterS3 do
+  subject { described_class.new }
+  let(:remote_file_name) { "somefile" }
+  let(:s3_client) { mock_s3_client }
+
+  before do
+    allow(Aws::S3::Client).to receive(:new).and_return s3_client
+  end
+
+  describe "#get" do
+    before do
+      allow(s3_client).to receive(:get_object)
+      allow(File).to receive(:read).and_return remote_file_contents
+    end
+
+    let(:remote_file_contents) {
+      { "foo" => "bar" }
+    }
+
+    it "downloads a copy of the remote file to the local file system" do
+      expect(s3_client).
+        to receive(:get_object).
+        with({
+               bucket: BetterS3.configuration.bucket.to_s,
+               key:    remote_file_name,
+             },
+             target: "#{BetterS3.configuration.tmp_dir_path}/#{remote_file_name}")
+
+      subject.get remote_file_name
+    end
+
+    it "returns the contents of the file" do
+      expect(subject.get(remote_file_name)).to eq remote_file_contents
+    end
+  end
+
+  describe "#put" do
+    let(:obj) { { "hi" => "toyou" } }
+    it "uploads the object to s3" do
+      expect(s3_client).to receive(:put_object).with(bucket: BetterS3.configuration.bucket.to_s,
+                                                     key:    remote_file_name,
+                                                     body:   obj.to_json)
+
+      subject.put obj.to_json, remote_file_name
+    end
+  end
+
+  describe "#delete_local_file_copy" do
+    it "checks to see if the file exists locally" do
+      expect(File).
+        to receive(:exist?).
+        with("#{BetterS3.configuration.tmp_dir_path}/#{remote_file_name}").
+        and_return false
+
+      subject.delete_local_file_copy remote_file_name
+    end
+
+    context "when the file exists locally" do
+      it "deletes the file from the local file system" do
+        allow(File).to receive(:exist?).and_return true
+        expect(File).to receive(:delete).with("#{BetterS3.configuration.tmp_dir_path}/#{remote_file_name}")
+
+        subject.delete_local_file_copy remote_file_name
+      end
+    end
+  end
+end

data/spec/simplecov_custom_profile.rb

@@ -0,0 +1,4 @@
+require "simplecov"
+SimpleCov.profiles.define "better_s3" do
+  add_filter "/spec"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require_relative "simplecov_custom_profile"
+SimpleCov.start "better_s3"
+require "better_s3"
+require "byebug"
+require_relative "support/mocks"

data/spec/support/mocks.rb

@@ -0,0 +1,3 @@
+def mock_s3_client
+  double "s3Client"
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: better_s3
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Courtland Caldwell
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-10-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: rb-readline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.31'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.31'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+description: Interact with S3 buckets in an idiomatic fashion.
+email: engineering@mattermark.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rubocop.yml
+- .rubocop_todo.yml
+- LICENSE
+- README.md
+- better_s3.gemspec
+- circle.yml
+- lib/better_s3.rb
+- lib/better_s3/configuration.rb
+- lib/better_s3/version.rb
+- spec/better_s3_spec.rb
+- spec/simplecov_custom_profile.rb
+- spec/spec_helper.rb
+- spec/support/mocks.rb
+homepage: https://github.com/Referly/better_s3
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: Idiomatic S3
+test_files:
+- spec/better_s3_spec.rb
+- spec/simplecov_custom_profile.rb
+- spec/spec_helper.rb
+- spec/support/mocks.rb
+has_rdoc: