bfs-s3 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64e82eeda72f79a41868973bca6120a9533742f28e3387fc3ea43a62ff7e2ac5
-  data.tar.gz: bbba7f07472b885148ec0ad66c3278412ecdb16ebdff4b1f06b2ca8604f43f6a
+  metadata.gz: 006a47eb5bb98ed25ce659ef731f72ea815f629382906125dff18d90866b06c9
+  data.tar.gz: '09e8fb5e9c3a7496f2d6004300f20e03e62af725a9e238abd4d67fa0d66b8ed2'
 SHA512:
-  metadata.gz: 594b23762958eb90c9d200d081939be4eba894593d56191772cc55dad3c6f596766481b7834a696fd65220e52a31bfe99b8169d1a5ec1e268b8bd7b188654b27
-  data.tar.gz: bed98d39b1423065bf49077947ac85dcedf36ae31e61d7cb6d921194d20597c4bb9850a90f20bb36331bf4a8d0866e1f9f93114a23c8bd691162d1ebe8e185c5
+  metadata.gz: 0aa16aadd6093b213199286f61a4ac965b54386e96e441b74c567d277fb5c5fc552977fb44f580ce254ce565dc79c12db23941dbd3e490e52e24288911218e17
+  data.tar.gz: a68af4df46647ccb88bdf6d2869b068aa26b83f75fb267a093d443146a30fc092bc73879f45b958d4e7112ceb9d769608e18fa452802e43b5674b12ebb9060d2

data/lib/bfs/bucket/s3.rb

@@ -6,16 +6,20 @@ module BFS
   module Bucket
     # S3 buckets are operating on s3
     class S3 < Abstract
+      NF_ERRORS = [Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound].freeze
+
       attr_reader :name, :sse, :acl, :storage_class
 
       # Initializes a new S3 bucket
       # @param [String] name the bucket name
       # @param [Hash] opts options
       # @option opts [String] :region default region
+      # @option opts [String] :prefix custom namespace within the bucket
       # @option opts [String] :sse default server-side-encryption setting
       # @option opts [Aws::Credentials] :credentials credentials object
       # @option opts [String] :access_key_id custom AWS access key ID
       # @option opts [String] :secret_access_key custom AWS secret access key
+      # @option opts [String] :profile_name custom AWS profile name (for shared credentials)
       # @option opts [Symbol] :acl canned ACL
       # @option opts [String] :storage_class storage class
       # @option opts [Aws::S3::Client] :client custom client, uses default_client by default
@@ -23,26 +27,32 @@ module BFS
         opts = opts.dup
         opts.keys.each do |key|
           val = opts.delete(key)
-          opts[key.to_s] = val unless val.nil?
+          opts[key.to_sym] = val unless val.nil?
         end
 
         @name = name
-        @sse = opts['sse'] || opts['server_side_encryption']
-        @credentials = opts['credentials']
-        @credentials ||= Aws::Credentials.new(opts['access_key_id'].to_s, opts['secret_access_key'].to_s) if opts['access_key_id']
-        @acl = opts['acl'].to_sym if opts['acl']
-        @storage_class = opts['storage_class']
-        @client = opts['client'] || Aws::S3::Client.new(region: opts['region'])
+        @sse = opts[:sse] || opts[:server_side_encryption]
+        @prefix = opts[:prefix]
+        @acl = opts[:acl].to_sym if opts[:acl]
+        @storage_class = opts[:storage_class]
+        @client = opts[:client] || init_client(opts)
       end
 
       # Lists the contents of a bucket using a glob pattern
       def ls(pattern='**/*', opts={})
+        prefix = pattern[%r{^[^\*\?\{\}\[\]]+/}]
+        prefix = File.join(*[@prefix, prefix].compact) if @prefix
+
+        opts = opts.merge(bucket: name, prefix: @prefix)
+        opts[:prefix] = prefix if prefix
+
         next_token = nil
         Enumerator.new do |y|
           loop do
-            resp = @client.list_objects_v2 opts.merge(bucket: name, continuation_token: next_token)
+            resp = @client.list_objects_v2 opts.merge(continuation_token: next_token)
             resp.contents.each do |obj|
-              y << obj.key if File.fnmatch?(pattern, obj.key, File::FNM_PATHNAME)
+              name = trim_prefix(obj.key)
+              y << name if File.fnmatch?(pattern, name, File::FNM_PATHNAME)
             end
             next_token = resp.next_continuation_token.to_s
             break if next_token.empty?
@@ -53,21 +63,18 @@ module BFS
       # Info returns the object info
       def info(path, opts={})
         path = norm_path(path)
-        opts = opts.merge(
-          bucket: name,
-          key: opts[:prefix] ? File.join(opts[:prefix], path) : path,
-        )
+        opts = opts.merge(bucket: name, key: full_path(path))
         info = @client.head_object(opts)
         raise BFS::FileNotFound, path unless info
 
         BFS::FileInfo.new(path, info.content_length, info.last_modified, info.content_type, info.metadata)
-      rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket
+      rescue *NF_ERRORS
         raise BFS::FileNotFound, path
       end
 
       # Creates a new file and opens it for writing
       def create(path, opts={}, &block)
-        path = norm_path(path)
+        path = full_path(path)
         opts = opts.merge(
           bucket: name,
           key: path,
@@ -92,7 +99,7 @@ module BFS
 
       # Opens an existing file for reading
       def open(path, opts={}, &block)
-        path = norm_path(path)
+        path = full_path(path)
         temp = Tempfile.new(File.basename(path), binmode: true)
         temp.close
 
@@ -104,33 +111,43 @@ module BFS
         @client.get_object(opts)
 
         File.open(temp.path, binmode: true, &block)
-      rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket
-        raise BFS::FileNotFound, path
+      rescue *NF_ERRORS
+        raise BFS::FileNotFound, trim_prefix(path)
       end
 
       # Deletes a file.
       def rm(path, opts={})
-        path = norm_path(path)
+        path = full_path(path)
         opts = opts.merge(
           bucket: name,
           key: path,
         )
         @client.delete_object(opts)
-      rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket # rubocop:disable Lint/HandleExceptions
+      rescue *NF_ERRORS # rubocop:disable Lint/HandleExceptions
       end
 
       # Copies a file.
       def cp(src, dst, opts={})
-        src = norm_path(src)
-        dst = norm_path(dst)
+        src = full_path(src)
+        dst = full_path(dst)
         opts = opts.merge(
           bucket: name,
           copy_source: "/#{name}/#{src}",
           key: dst,
         )
         @client.copy_object(opts)
-      rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket
-        raise BFS::FileNotFound, src
+      rescue *NF_ERRORS
+        raise BFS::FileNotFound, trim_prefix(src)
+      end
+
+      private
+
+      def init_client(opts)
+        credentials = opts[:credentials]
+        credentials ||= Aws::Credentials.new(opts[:access_key_id].to_s, opts[:secret_access_key].to_s) if opts[:access_key_id]
+        credentials ||= Aws::SharedCredentials.new(profile_name: opts[:profile_name])
+
+        Aws::S3::Client.new region: opts[:region], credentials: credentials
       end
     end
   end

data/spec/bfs/bucket/s3_spec.rb

@@ -1,58 +1,42 @@
 require 'spec_helper'
 
-RSpec.describe BFS::Bucket::S3 do
-  let(:files) { {} }
-  let :mock_client do
-    client = double Aws::S3::Client
-
-    allow(client).to receive(:put_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
-      files[opts[:key]] = opts[:body].read
-      nil
-    end
-
-    allow(client).to receive(:get_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
-      data = files[opts[:key]]
-      raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless data
-
-      File.open(opts[:response_target], 'wb') {|f| f.write(data) }
-      nil
-    end
-
-    allow(client).to receive(:delete_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
-      raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless files.key?(opts[:key])
-
-      files.delete(opts[:key])
-      nil
-    end
-
-    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', continuation_token: nil) do |*|
-      contents = files.keys.map {|key| Aws::S3::Types::Object.new(key: key) }
-      Aws::S3::Types::ListObjectsV2Output.new contents: contents, next_continuation_token: ''
-    end
-    allow(client).to receive(:head_object).with(bucket: 'mock-bucket', key: 'a/b/c.txt') do |*|
-      Aws::S3::Types::HeadObjectOutput.new content_length: 10, last_modified: Time.now, content_type: 'text/plain', metadata: { 'key' => 'val' }
+sandbox  = { region: 'us-east-1', bucket: 'bsm-bfs-unittest' }.freeze
+run_spec = \
+  begin
+    c = Aws::SharedCredentials.new
+    if c.loadable?
+      s = Aws::S3::Client.new(region: sandbox[:region], credentials: c)
+      s.head_bucket(bucket: sandbox[:bucket])
+      true
+    else
+      false
     end
-    allow(client).to receive(:head_object).with(bucket: 'mock-bucket', key: 'missing.txt') do |*|
-      raise Aws::S3::Errors::NoSuchKey.new(nil, nil)
-    end
-    allow(client).to receive(:copy_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
-      src = opts[:copy_source].sub('/mock-bucket/', '')
-      raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless files.key?(src)
+  rescue Aws::Errors::ServiceError
+    false
+  end
 
-      files[opts[:key]] = files[src]
-      nil
-    end
+RSpec.describe BFS::Bucket::S3, if: run_spec do
+  let(:prefix) { "x/#{SecureRandom.uuid}/" }
 
-    client
+  subject do
+    described_class.new sandbox[:bucket], region: sandbox[:region], prefix: prefix
+  end
+  after :all do
+    bucket = described_class.new sandbox[:bucket], region: sandbox[:region], prefix: 'x/'
+    bucket.ls.each {|name| bucket.rm(name) }
   end
 
-  subject { described_class.new('mock-bucket', client: mock_client) }
   it_behaves_like 'a bucket'
 
   it 'should resolve from URL' do
-    bucket = BFS.resolve('s3://mock-bucket?acl=private&region=eu-west-2')
+    bucket = BFS.resolve("s3://#{sandbox[:bucket]}?acl=private&region=#{sandbox[:region]}")
     expect(bucket).to be_instance_of(described_class)
-    expect(bucket.name).to eq('mock-bucket')
+    expect(bucket.name).to eq(sandbox[:bucket])
     expect(bucket.acl).to eq(:private)
   end
+
+  it 'should enumerate over a large number of files' do
+    bucket = described_class.new sandbox[:bucket], region: sandbox[:region], prefix: 'm/'
+    expect(bucket.ls('**/*').count).to eq(2121)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bfs-s3
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Dimitrij Denissenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-26 00:00:00.000000000 Z
+date: 2018-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.3.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.3.4
 description: https://github.com/bsm/bfs.rb
 email: dimitrij@blacksquaremedia.com
 executables: []
@@ -68,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: S3 bucket adapter for bfs