s3-publisher 0.9.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f80c63b946040a1ad65feb3d5ccf603f2dc8125
-  data.tar.gz: 3d10cace13da6a1e150cc869394f738bc0687d27
+  metadata.gz: 4c5f5fd908bd87713d65bad6e7429c1090284f35
+  data.tar.gz: d58aabf2f780673028dd3b5e342dbba86aa00b21
 SHA512:
-  metadata.gz: 28cd6ac4555baecede5cc524e37d8a187e3931db820f43345548c39334748684ad11db3f266f32505b220ee3bdc461f5ef51a08cc19aef001279a94c934efe87
-  data.tar.gz: d991ec0ae159bad2c0687baa6df0edb840923491366f21f399915a088f34cc4e1a65ea90591d03fbfb4c133309330857dc60679412875c535fab2e6910020ed4
+  metadata.gz: 5281ecababe99562e4795a2f13072ec5791ba8c0a74f6087a5a298a844c8251208ddd3d3ce2d63779fbc92fa80059339b4e8283b43c4775ec8538e897ea0575f
+  data.tar.gz: 55284b8a3218c84982ad925297cd5fefd9d7d4a7bb59018c883622bf7a73611033cfcabf3286eb31faf34398492d5ffa86a684e45f7d381fbb6f7fe1f4880fa3

data/Gemfile.lock

@@ -1,22 +1,48 @@
 PATH
   remote: .
   specs:
-    s3-publisher (0.4.3)
-      aws_credentials
-      right_aws (>= 3.0.0)
+    s3-publisher (2.0.0)
+      aws-sdk (>= 2.0)
+      mime-types
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws_credentials (0.7.0)
-    right_aws (3.1.0)
-      right_http_connection (>= 1.2.5)
-    right_http_connection (1.4.0)
-    thoughtbot-shoulda (2.11.1)
+    aws-sdk (2.2.29)
+      aws-sdk-resources (= 2.2.29)
+    aws-sdk-core (2.2.29)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.2.29)
+      aws-sdk-core (= 2.2.29)
+    diff-lcs (1.2.5)
+    jmespath (1.1.3)
+    metaclass (0.0.4)
+    mime-types (3.0)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0221)
+    mocha (1.1.0)
+      metaclass (~> 0.0.1)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.4)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  mocha (>= 1.1.0)
+  rspec
   s3-publisher!
-  thoughtbot-shoulda
+
+BUNDLED WITH
+   1.11.2

data/README.md

@@ -2,6 +2,8 @@
 
 Quickly pub your data files to S3.
 
+**This version depends on aws-sdk 2.x.  For aws-sdk 1.x projects, use version <= 1.0 of this gem.**
+
 Reasons you might want to use this instead of aws-sdk directly:
 
   * parallel uploads using ruby threads.  Concurrency defaults to 3 but can be increased.
@@ -38,7 +40,7 @@ end
 ### Slightly more advanced example:
 
 ```
-S3Publisher.publish('my-bucket', :base_path => 'world_cup', :region => 'us-west-1') do |p|
+S3Publisher.publish('my-bucket', base_path: 'world_cup', region: 'us-west-1') do |p|
     p.push('events.xml', data: '<xml>...', ttl: 15)
 end
 ```
@@ -55,7 +57,11 @@ See class docs for more options.
 
 Since S3Publisher uses [aws-sdk](https://github.com/aws/aws-sdk-ruby) any of the usual credential stores will work, including:
 
- * `AWS.config(access_key_id: '...', secret_access_key: '...', region: 'us-west-2')`
- * `config/aws.yml` in a RAILS project
- * ENV vars
-
+ * `ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], ENV['AWS_REGION']`
+ * IAM role
+ * ```
+  Aws.config.update({
+    region: 'us-west-2',
+    credentials: Aws::Credentials.new('akid', 'secret')
+  })
+  ```

data/lib/s3-publisher.rb

@@ -27,24 +27,19 @@ class S3Publisher
   # @option opts [String] :base_path Path prepended to supplied file_name on upload
   # @option opts [Integer] :workers Number of threads to use when pushing to S3. Defaults to 3.
   # @option opts [Object] :logger A logger object to recieve 'uploaded' messages.  Defaults to STDOUT.
-  # @option opts [String] :access_key_id AWS access key to use, if different than global AWS config
-  # @option opts [String] :secret_access_key AWS secret access key to use, if different than global AWS config
-  # @option opts [String] :region AWS region to use, if different than global AWS config
+  #
+  # Additional keys will be passed through to the Aws::S3::Client init, including:
+  # @option opts [String] :region AWS region to use, if different than global Aws config
+  # @option opts [Object] :credentials - :access_key_id, :secret_access_key, and :session_token options, if different than global Aws config
+  # See http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method for full details.
 
   def initialize bucket_name, opts={}
     @publish_queue = Queue.new
-    @workers_to_use = opts[:workers] || 3
-    @logger = opts[:logger] || $stdout
-
-    s3_opts = {}
-    s3_opts[:access_key_id]     = opts[:access_key_id]     if opts.key?(:access_key_id)
-    s3_opts[:secret_access_key] = opts[:secret_access_key] if opts.key?(:secret_access_key)
-    s3_opts[:region]            = opts[:region]            if opts.key?(:region)
-
-    @s3 = AWS::S3.new(s3_opts)
+    @workers_to_use = opts.delete(:workers) || 3
+    @logger         = opts.delete(:logger)  || $stdout
+    @bucket_name, @base_path = bucket_name, opts.delete(:base_path)
 
-    @bucket_name, @base_path = bucket_name, opts[:base_path]
-    raise ArgumentError, "#{bucket_name} doesn't seem to be a valid bucket on your account" if @s3.buckets[bucket_name].nil?
+    @s3 = Aws::S3::Client.new(opts)
   end
 
   # Queues a file to be published.
@@ -124,8 +119,6 @@ class S3Publisher
     
       try_count = 0
       begin
-        obj = @s3.buckets[bucket_name].objects[item[:key_name]]
-
         gzip = item[:gzip] != false && !item[:write_opts][:content_type].match('image/')
 
         if gzip
@@ -134,7 +127,14 @@ class S3Publisher
           item[:contents] = gzip(gzip_body)
         end
 
-        obj.write(item[:contents], item[:write_opts])
+        write_opts = {
+          bucket: bucket_name,
+          key: item[:key_name],
+          body: item[:contents]
+        }
+        write_opts.merge!(item[:write_opts])
+
+        @s3.put_object(write_opts)
 
       rescue Exception => e # backstop against transient S3 errors
         raise e if try_count >= 1

data/s3-publisher.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = "s3-publisher"
-  s.version     = "0.9.3"
+  s.version     = "2.0.0"
   s.authors     = ["Ben Koski"]
   s.email       = "bkoski@nytimes.com"
   s.summary     = "Publish data to S3 for the world to see"
@@ -16,7 +16,9 @@ Gem::Specification.new do |s|
 
   s.require_paths = ["lib"]
 
-  s.add_development_dependency(%q<rspec>, [">= 0"])
-  s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.0"])
+  s.add_runtime_dependency(%q<aws-sdk>, [">= 2.0"])
   s.add_runtime_dependency(%q<mime-types>, [">= 0"])
+
+  s.add_development_dependency(%q<rspec>, [">= 0"])
+  s.add_development_dependency(%q<mocha>, [">= 1.1.0"])
 end

data/spec/s3_publisher_spec.rb

@@ -4,16 +4,16 @@ describe S3Publisher do
   describe "#push" do
            
     describe "file_name" do
-      it "prepends base_path if provided" do
+      it "prepends base_path if provided", focus: true do
         set_put_expectation(key_name: 'world_cup_2010/events.xml')
-        p = S3Publisher.new('test-bucket', :logger => Logger.new(nil), :base_path => 'world_cup_2010')
+        p = S3Publisher.new('test-bucket', logger: Logger.new(nil), base_path: 'world_cup_2010')
         p.push('events.xml', data: '1234')
         p.run
       end
       
       it "passes name through unaltered if base_path not specified" do
         set_put_expectation(key_name: 'events.xml')
-        p = S3Publisher.new('test-bucket', :logger => Logger.new(nil))
+        p = S3Publisher.new('test-bucket', logger: Logger.new(nil))
         p.push('events.xml', data: '1234')
         p.run
       end      
@@ -92,10 +92,6 @@ describe S3Publisher do
     #  * :data
     #  * :content_type, :cache_control, :content_encoding
     def set_put_expectation opts
-      s3_stub = mock()
-      bucket_stub = mock()
-      object_stub = mock()
-
       key_name = opts[:key_name] || 'myfile.txt'
 
       expected_entries = {}
@@ -111,12 +107,9 @@ describe S3Publisher do
         expected_contents = anything
       end
 
-      object_stub.expects(:write).with(expected_contents, has_entries(expected_entries))
+      expected_entries.merge!(bucket: 'test-bucket', key: key_name, body: expected_contents)
 
-      s3_stub.stubs(:buckets).returns({'test-bucket' => bucket_stub })
-      bucket_stub.stubs(:objects).returns({ key_name => object_stub })
-      
-      AWS::S3.stubs(:new).returns(s3_stub)
+      Aws::S3::Client.any_instance.expects(:put_object).with(has_entries(expected_entries))
     end
     
     def gzip data
@@ -130,7 +123,7 @@ describe S3Publisher do
     end
     
     def push_test_data file_name, opts
-      p = S3Publisher.new('test-bucket', :logger => Logger.new(nil))
+      p = S3Publisher.new('test-bucket', logger: Logger.new(nil))
       p.push(file_name, opts)
       p.run
     end

data/spec/spec_helper.rb

@@ -7,6 +7,8 @@
 
 require File.expand_path('../lib/s3-publisher.rb', File.dirname(__FILE__))
 
+ENV['AWS_REGION'] = 'us-east-1'
+
 RSpec.configure do |config|
   # Limit the spec run to only specs with the focus metadata. If no specs have
   # the filtering metadata and `run_all_when_everything_filtered = true` then

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3-publisher
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 2.0.0
 platform: ruby
 authors:
 - Ben Koski
@@ -11,47 +11,61 @@ cert_chain: []
 date: 2016-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '2.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: mime-types
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: mime-types
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 description: Publish data to S3 for the world to see
 email: bkoski@nytimes.com
 executables: []