asset_cloud 2.3.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f22a2b32e4f1e13f4743603220642baf85b8d1b3ddb30191c5851e84a0e12239
-  data.tar.gz: 3e5aece0ed39d47752f86be2f5da8a78a388db4b5f686df6eb100059f2435d6e
+  metadata.gz: 929356d981cb9cbc1f805c713071b909ff594ff05d8a54231d1f0bc85672ffd0
+  data.tar.gz: '09b7b0e5ce21d53385a5824619fed80dc76c11c348ea9a9cda531dd33e9a12d4'
 SHA512:
-  metadata.gz: 2f2d1ccf6ad860ab28e84aa67046bf45b0b8bd9e5a47b4c1973e8bbb23c8b686d040a85420d260e47aaa68ee22d0ced688c52383f0f02e46e97edaf3d773a3f5
-  data.tar.gz: 0e53e2aab25cc6dbeb0b7de72d94f5a8f3c671850451948577f9f6d876229fdd0cefc3bbace06561145dde8b838f7fc8688aa484e2fdd8c89a6b19403a8fc610
+  metadata.gz: 998ab0f3e4795f228adfa07b01ab1369a140c9ea5be4a729fd089953670fca677e31a5cee59d7adbdde0a507df33410fa82517367d0411e8f8b94e86baee4272
+  data.tar.gz: ba2de4239a9778afc6416cedbad4830a5d1f56f59824772a0fdbbfaaeaea1d20a90d6dd80db615a1dde8a12fc1eb6e053d54ec05f60dce4d9b44b9514300b62e

data/Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'aws-sdk', '~> 1.34.0', :require => nil
+gem 'aws-sdk-s3', '>= 1.60.2', require: false
 gem 'google-cloud-storage'
 
 gemspec

data/asset_cloud.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{asset_cloud}
-  s.version = "2.3.1"
+  s.version = "2.4.0"
 
   s.authors = %w(Shopify)
   s.summary = %q{An abstraction layer around arbitrary and diverse asset stores.}

data/lib/asset_cloud/buckets/s3_bucket.rb

@@ -1,46 +1,72 @@
-require 'aws'
+require 'aws-sdk-s3'
 
 module AssetCloud
   class S3Bucket < Bucket
     def ls(key = nil)
       key = absolute_key(key)
 
-      objects = cloud.s3_bucket(key).objects
-      objects = objects.with_prefix(key) if key
+      options = {}
+      options[:prefix] = key if key
 
+      objects = cloud.s3_bucket(key).objects(options)
       objects.map { |o| cloud[relative_key(o.key)] }
     end
 
     def read(key, options = {})
-      cloud.s3_bucket(key).objects[absolute_key(key)].read(options)
-    rescue ::AWS::Errors::Base
+      options = options.dup
+
+      options[:range] = http_byte_range(options[:range]) if options[:range]
+
+      bucket = cloud.s3_bucket(key)
+      if encryption_key = options.delete(:encryption_key)
+        bucket = encrypted_bucket(bucket, encryption_key)
+      end
+
+      object = bucket.object(absolute_key(key)).get(options)
+
+      object.body.respond_to?(:read) ? object.body.read : object.body
+    rescue ::Aws::Errors::ServiceError
       raise AssetCloud::AssetNotFoundError, key
     end
 
     def write(key, data, options = {})
-      object = cloud.s3_bucket(key).objects[absolute_key(key)]
+      options = options.dup
+
+      bucket = cloud.s3_bucket(key)
+      if encryption_key = options.delete(:encryption_key)
+        bucket = encrypted_bucket(bucket, encryption_key)
+      end
 
-      object.write(data, options)
+      object = bucket.object(absolute_key(key))
+      object.put(options.merge(body: data))
     end
 
     def delete(key)
-      object = cloud.s3_bucket(key).objects[absolute_key(key)]
-
+      object = cloud.s3_bucket(key).object(absolute_key(key))
       object.delete
-
       true
     end
 
     def stat(key)
-      object = cloud.s3_bucket(key).objects[absolute_key(key)]
-      metadata = object.head
+      bucket = cloud.s3_bucket(key)
+      metadata = bucket.client.head_object(bucket: bucket.name, key: absolute_key(key))
 
       AssetCloud::Metadata.new(true, metadata[:content_length], nil, metadata[:last_modified])
-    rescue AWS::S3::Errors::NoSuchKey
+    rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NotFound
       AssetCloud::Metadata.new(false)
     end
 
-    protected
+    private
+
+    def encrypted_bucket(source_bucket, key)
+      Aws::S3::Resource.new(
+        client: Aws::S3::Encryption::Client.new(
+          client: source_bucket.client,
+          encryption_key: key,
+        )
+      ).bucket(source_bucket.name)
+    end
+
     def path_prefix
       @path_prefix ||= @cloud.url
     end
@@ -58,5 +84,11 @@ module AssetCloud
     def relative_key(key)
       key =~ /^#{path_prefix}\/(.+)/ ? $1 : key
     end
+
+    def http_byte_range(range)
+      # follows https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
+      return "bytes=#{[range.begin, range.max].join('-')}" if range.is_a?(Range)
+      range
+    end
   end
 end

data/spec/mock_s3_interface.rb

@@ -1,73 +1,71 @@
+require 'ostruct'
+
 class MockS3Interface
+  VALID_ACLS = %w(
+    private public-read public-read-write authenticated-read aws-exec-read bucket-owner-read bucket-owner-full-control
+  )
+
   attr_reader :bucket_storage
 
-  def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={})
-    @bucket_storage = {}
+  def initialize(aws_access_key_id = nil, aws_secret_access_key = nil, params = {})
+    @buckets = {}
   end
 
-  def buckets
-    @bucket_collection ||= BucketCollection.new(self)
+  def buckets(**options)
+    buckets = @buckets.values
+    buckets = buckets.select { |v| v.start_with?(options[:prefix]) } if options[:prefix]
+    buckets
   end
 
-  def client
-    @client ||= Client.new
+  def bucket(name)
+    @buckets[name] ||= Bucket.new(self, name)
   end
 
-  class Client
+  def client
+    self
   end
 
-  class BucketCollection
-    def initialize(interface)
-      @interface = interface
-    end
+  def head_object(options = {})
+    options = bucket(options[:bucket])
+      .object(options[:key])
+      .get
 
-    def [](name)
-      @interface.bucket_storage[name] ||= Bucket.new(name)
-    end
+    {
+      content_length: options.body.size,
+      last_modified: Time.parse("Mon Aug 27 17:37:51 UTC 2007")
+    }
   end
 
   class Bucket
-    attr_reader :name
-    def initialize(name)
+    attr_reader :name, :client
+    def initialize(client, name)
+      @client = client
       @name = name
       @storage = {}
-      @storage_options = {}
     end
 
-    def with_prefix(prefix)
-      keys = @storage
-      keys = keys.select {|k,v| k.starts_with?(prefix)}
-      keys.map {|k,v| S3Object.new(self, k, v)}
+    def objects(**options)
+      objects = @storage.values
+      objects = objects.select { |v| v.start_with?(options[:prefix]) } if options[:prefix]
+      objects
     end
 
-    def objects
-      Collection.new(self, @storage.keys)
+    def object(key)
+      @storage[key] ||= NullS3Object.new(self, key)
     end
 
-    def get(key)
-      if @storage.key?(key)
-        @storage[key]
-      else
-        raise AWS::S3::Errors::NoSuchKey.new(nil, nil)
-      end
-    end
+    def put_object(options = {})
+      options = options.dup
 
-    def get_options(key)
-      if @storage_options.key?(key)
-        @storage_options[key]
-      else
-        raise AWS::S3::Errors::NoSuchKey.new(nil, nil)
+      if options[:acl] && !VALID_ACLS.include?(options[:acl])
+        raise "Invalid ACL `#{options[:acl].inspect}`, must be one of: #{VALID_ACLS.inspect}"
       end
-    end
+      
+      options[:body] = options[:body].force_encoding(Encoding::BINARY)
 
-    def put(key, data, options={})
-      @storage[key] = data.dup.force_encoding(Encoding::BINARY)
-      @storage_options[key] = options.dup
-      true
-    end
+      key = options.delete(:key)
 
-    def delete(key)
-      @storage.delete(key)
+      @storage[key] = S3Object.new(self, key, options)
       true
     end
 
@@ -80,88 +78,49 @@ class MockS3Interface
     end
   end
 
-  class Collection
-    include Enumerable
-
-    def initialize(bucket, objects)
+  class NullS3Object
+    attr_reader :key
+    def initialize(bucket, key)
       @bucket = bucket
-      @objects = objects
+      @key = key
     end
 
-    def with_prefix(prefix)
-      self.class.new(@bucket, @objects.select {|k| k.start_with?(prefix)})
+    def get(*)
+      raise Aws::S3::Errors::NoSuchKey.new(nil, nil)
     end
 
-    def [](name)
-      S3Object.new(@bucket, name)
+    def delete(*)
     end
 
-    def each
-      @objects.each { |e| yield S3Object.new(@bucket, e) }
+    def put(options = {})
+      @bucket.put_object(options.merge(key: @key))
     end
   end
 
   class S3Object
-    attr_reader :key
+    attr_reader :key, :options
 
-    def initialize(bucket, key, data=nil)
+    def initialize(bucket, key, options = {})
       @bucket = bucket
       @key = key
+      @options = options
     end
 
     def delete
       @bucket.delete(@key)
+      true
     end
 
-    def read(headers={})
-      @bucket.get(@key)
-    end
-
-    def options()
-      @bucket.get_options(@key)
-    end
-
-    def write(data, options={})
-      @bucket.put(@key, data, options)
-    end
-
-    def multipart_upload(options = {})
-      MockMultipartUpload.new(@bucket, @key)
+    def get(*)
+      OpenStruct.new(options)
     end
 
-    def url_for(permission, options={})
-      if options[:secure]
-        URI.parse("https://www.youtube.com/watch?v=oHg5SJYRHA0")
-      else
-        URI.parse("http://www.youtube.com/watch?v=oHg5SJYRHA0")
+    def put(options = {})
+      if options[:acl] && !VALID_ACLS.include?(options[:acl])
+        raise "Invalid ACL `#{options[:acl].inspect}`, must be one of: #{VALID_ACLS.inspect}"
       end
-    end
-
-    def head
-      {
-        content_length: read.size,
-        last_modified: Time.parse("Mon Aug 27 17:37:51 UTC 2007")
-      }
-    end
-  end
-
-  class MockMultipartUpload
-    def initialize(bucket, key)
-      @bucket =bucket
-      @key = key
-      @data = ""
-    end
-
-    def add_part(data)
-      @data << data
-    end
-
-   def abort
-      @bucket.delete(@key)
-   end
 
-   def complete(arg)
-      @bucket.put(@key, @data, {})
+      @bucket.put_object(options.merge(key: @key))
     end
   end
 end

data/spec/remote_s3_bucket_spec.rb

@@ -5,22 +5,24 @@ class RemoteS3Cloud < AssetCloud::Base
   bucket :tmp, AssetCloud::S3Bucket
 
   def s3_bucket(key)
-    s3_connection.buckets[ENV['S3_BUCKET_NAME']]
+    s3_connection.bucket(ENV['S3_BUCKET_NAME'])
   end
 end
 
-describe 'Remote test for AssetCloud::S3Bucket', if:  ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY'] && ENV['S3_BUCKET_NAME'] do
-  require 'aws-sdk'
-
+describe 'Remote test for AssetCloud::S3Bucket', if: ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY'] && ENV['S3_BUCKET_NAME'] do
   directory = File.dirname(__FILE__) + '/files'
 
   before(:all) do
-    AWS.config({
-      access_key_id: ENV['AWS_ACCESS_KEY_ID'],
-      secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
-    })
+    Aws.config = {
+      region: ENV.fetch('AWS_REGION', 'us-east-1'),
+      credentials: Aws::Credentials.new(
+        ENV['AWS_ACCESS_KEY_ID'],
+        ENV['AWS_SECRET_ACCESS_KEY'],
+      ),
+    }
+
     @cloud = RemoteS3Cloud.new(directory , 'testing/assets/files' )
-    @cloud.s3_connection = AWS::S3.new()
+    @cloud.s3_connection = Aws::S3::Resource.new
     @bucket = @cloud.buckets[:tmp]
   end
 
@@ -35,9 +37,18 @@ describe 'Remote test for AssetCloud::S3Bucket', if:  ENV['AWS_ACCESS_KEY_ID'] &
 
     ls = @bucket.ls('tmp')
 
-    ls.first.class.should == AssetCloud::Asset
-    keys = ls.map(&:key)
-    ['tmp/test1.txt', 'tmp/test2.txt'].all? {|key| keys.include? key }
+    expect(ls).to all(be_an(AssetCloud::Asset))
+    expect(ls.map(&:key) - ['tmp/test1.txt', 'tmp/test2.txt']).to be_empty
+  end
+
+  it "#ls returns all assets" do
+    @cloud['tmp/test1.txt'] = 'test1'
+    @cloud['tmp/test2.txt'] = 'test2'
+
+    ls = @bucket.ls
+
+    expect(ls).to all(be_an(AssetCloud::Asset))
+    expect(ls.map(&:key) - ['tmp/test1.txt', 'tmp/test2.txt']).to be_empty
   end
 
   it "#delete should ignore errors when deleting" do
@@ -59,6 +70,12 @@ describe 'Remote test for AssetCloud::S3Bucket', if:  ENV['AWS_ACCESS_KEY_ID'] &
     metadata.updated_at.should >= start_time
   end
 
+  it "#stat a missing asset" do
+    metadata = @bucket.stat('i_do_not_exist_and_never_will.test')
+    expect(metadata).to be_an(AssetCloud::Metadata)
+    expect(metadata.exist).to be false
+  end
+
   it "#read " do
     value = 'hello world'
     key = 'tmp/new_file.txt'
@@ -67,6 +84,10 @@ describe 'Remote test for AssetCloud::S3Bucket', if:  ENV['AWS_ACCESS_KEY_ID'] &
     data.should == value
   end
 
+  it "#read a missing asset" do
+    expect { @bucket.read("i_do_not_exist_and_never_will.test") }.to raise_error(AssetCloud::AssetNotFoundError)
+  end
+
   it "#reads first bytes when passed options" do
     value = 'hello world'
     key = 'tmp/new_file.txt'

data/spec/s3_bucket_spec.rb

@@ -6,7 +6,7 @@ class S3Cloud < AssetCloud::Base
   attr_accessor :s3_connection, :s3_bucket_name
 
   def s3_bucket(key)
-    s3_connection.buckets[s3_bucket_name]
+    s3_connection.bucket(s3_bucket_name)
   end
 end
 
@@ -27,21 +27,25 @@ describe AssetCloud::S3Bucket do
   end
 
   it "#ls should return assets with proper keys" do
-    collection = MockS3Interface::Collection.new(nil, ["#{@cloud.url}/tmp/blah.gif", "#{@cloud.url}/tmp/add_to_cart.gif"])
+    collection = ["#{@cloud.url}/tmp/blah.gif", "#{@cloud.url}/tmp/add_to_cart.gif"].map do |key|
+      MockS3Interface::S3Object.new(nil, key)
+    end
     expect_any_instance_of(MockS3Interface::Bucket).to receive(:objects).and_return(collection)
+
     ls = @bucket.ls('tmp')
-    ls.first.class.should == AssetCloud::Asset
-    ls.map(&:key).should == ['tmp/blah.gif', 'tmp/add_to_cart.gif']
+
+    expect(ls).to all(be_an(AssetCloud::Asset))
+    expect(ls.map(&:key) - ['tmp/blah.gif', 'tmp/add_to_cart.gif']).to be_empty
   end
 
   it "#delete should not ignore errors when deleting" do
-    expect_any_instance_of(MockS3Interface::Bucket).to receive(:delete).and_raise(StandardError)
+    expect_any_instance_of(MockS3Interface::NullS3Object).to receive(:delete).and_raise(StandardError)
 
     expect { @bucket.delete('assets/fail.gif') }.to raise_error(StandardError)
   end
 
   it "#delete should always return true" do
-    expect_any_instance_of(MockS3Interface::Bucket).to receive(:delete).and_return(nil)
+    expect_any_instance_of(MockS3Interface::NullS3Object).to receive(:delete).and_return(nil)
 
     @bucket.delete('assets/fail.gif').should == true
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: asset_cloud
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.4.0
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-24 00:00:00.000000000 Z
+date: 2020-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport