bfs-s3 0.7.5 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d22f34e604788253bb1ad66fe31eeea816daddd8ae793de9e9a8919e5a8ff7c
-  data.tar.gz: f840bb61c273719aa5305a1c27580b42584db47aec88efdc719beac492cc55e5
+  metadata.gz: fde16d5ecd59f0ba7892b75083f66886361e5b7dc5efea15f5c28c2cdd7f0b44
+  data.tar.gz: e5fc5fb205274be7c10071dbc2a8c9c6bbe7c12cc30c90cb0768b1758692183c
 SHA512:
-  metadata.gz: d96d23659c8c35ca2f140e3d39500c60afd92787cc885ec71730eac6b3f9d6f7d8597feec95c55ad8ca7e032d581d160daa78b2bf81dce9e78a42b8accf3b646
-  data.tar.gz: f3da02ff82b2b995090c0e8fd79730174dc29e78a88e8ff930ddc5d520599a62636d386a3746298e3c78672b33a840bd3909a8f6b417384f85196315407bac3e
+  metadata.gz: 720120b887769f077a4a5f2ac1874cb71f6c2bc9f11825be5aa41096787d226e50ba18008409a0fad45dbe608f61ac7a1ae0314fb811261e7bd2d09bdbfb2a5b
+  data.tar.gz: 3c45d88593ebf385de8cc81c292cd86219ea13d8a6d962cf40b96ff06d34bd5188ab97af85b642beb5be39fffb9503cdf41f2fffe03ae23a9574145047c550bb

data/lib/bfs/bucket/s3.rb

@@ -33,22 +33,19 @@ module BFS
 
       # Lists the contents of a bucket using a glob pattern
       def ls(pattern = '**/*', **opts)
-        prefix = pattern[%r{^[^*?\{\}\[\]]+/}]
-        prefix = File.join(*[@prefix, prefix].compact) if @prefix
-
-        opts = opts.merge(bucket: name, prefix: @prefix)
-        opts[:prefix] = prefix if prefix
+        Enumerator.new do |acc|
+          walk(pattern, **opts) do |path, _|
+            acc << path
+          end
+        end
+      end
 
-        next_token = nil
-        Enumerator.new do |y|
-          loop do
-            resp = @client.list_objects_v2 opts.merge(continuation_token: next_token)
-            resp.contents.each do |obj|
-              name = trim_prefix(obj.key)
-              y << name if File.fnmatch?(pattern, name, File::FNM_PATHNAME)
-            end
-            next_token = resp.next_continuation_token.to_s
-            break if next_token.empty?
+      # Iterates over the contents of a bucket using a glob pattern
+      def glob(pattern = '**/*', **opts)
+        Enumerator.new do |acc|
+          walk(pattern, **opts) do |path, obj|
+            info = BFS::FileInfo.new(path: path, size: obj.size, mtime: obj.last_modified)
+            acc << info
           end
         end
       end
@@ -60,7 +57,7 @@ module BFS
         info = @client.head_object(**opts)
         raise BFS::FileNotFound, path unless info
 
-        BFS::FileInfo.new(path: path, size: info.content_length, mtime: info.last_modified, content_type: info.content_type, metadata: norm_meta(info.metadata))
+        BFS::FileInfo.new path: path, size: info.content_length, mtime: info.last_modified, content_type: info.content_type, metadata: norm_meta(info.metadata)
       rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound
         raise BFS::FileNotFound, path
       end
@@ -82,7 +79,7 @@ module BFS
         opts[:server_side_encryption] ||= @sse if @sse
         opts[:storage_class] ||= @storage_class if @storage_class
 
-        BFS::TempWriter.new(path, encoding: encoding, perm: perm) do |t|
+        BFS::Writer.new(path, encoding: encoding, perm: perm) do |t|
           File.open(t, encoding: encoding) do |file|
             @client.put_object(opts.merge(body: file))
           end
@@ -144,9 +141,33 @@ module BFS
         config[:credentials] = opts[:credentials] if opts[:credentials]
         config[:credentials] ||= Aws::Credentials.new(opts[:access_key_id].to_s, opts[:secret_access_key].to_s) if opts[:access_key_id]
         config[:credentials] ||= Aws::SharedCredentials.new(profile_name: opts[:profile_name]) if opts[:profile_name]
+        config[:credentials] = Aws::AssumeRoleCredentials.new(
+          client: config[:credentials] ? Aws::STS::Client.new(credentials: config[:credentials]) : nil,
+          role_arn: opts[:assume_role],
+          role_session_name: SecureRandom.urlsafe_base64(12),
+        ) if opts[:assume_role]
 
         Aws::S3::Client.new(config)
       end
+
+      def walk(pattern, **opts)
+        prefix = pattern[%r{^[^*?\{\}\[\]]+/}]
+        prefix = File.join(*[@prefix, prefix].compact) if @prefix
+
+        opts = opts.merge(bucket: name, prefix: @prefix)
+        opts[:prefix] = prefix if prefix
+
+        next_token = nil
+        loop do
+          resp = @client.list_objects_v2 opts.merge(continuation_token: next_token)
+          resp.contents.each do |obj|
+            path = trim_prefix(obj.key)
+            yield(path, obj) if File.fnmatch?(pattern, path, File::FNM_PATHNAME)
+          end
+          next_token = resp.next_continuation_token.to_s
+          break if next_token.empty?
+        end
+      end
     end
   end
 end
@@ -154,7 +175,7 @@ end
 BFS.register('s3') do |url, opts, block|
   prefix = BFS.norm_path(opts[:prefix] || url.path)
   opts[:prefix] = prefix.empty? ? nil : prefix
-  opts = opts.slice(:prefix, :region, :sse, :access_key_id, :secret_access_key, :acl, :storage_class, :encoding)
+  opts = opts.slice(:prefix, :region, :sse, :access_key_id, :secret_access_key, :acl, :storage_class, :encoding, :assume_role)
 
   BFS::Bucket::S3.open url.host, **opts, &block
 end

data/spec/bfs/bucket/s3_spec.rb

@@ -1,38 +1,39 @@
 require 'spec_helper'
 
-sandbox = { bucket: 'bsm-bfs-unittest' }.freeze
+bucket_name = 'bsm-bfs-unittest'
 
 RSpec.describe BFS::Bucket::S3, s3: true do
-  let(:prefix) { "x/#{SecureRandom.uuid}/" }
-
   subject do
-    described_class.new sandbox[:bucket], prefix: prefix
+    described_class.new bucket_name, prefix: prefix
   end
-  after :all do
-    bucket = described_class.new sandbox[:bucket], prefix: 'x/'
+
+  let(:prefix) { "x/#{SecureRandom.uuid}/" }
+
+  after do
+    bucket = described_class.new bucket_name, prefix: prefix
     bucket.ls.each {|name| bucket.rm(name) }
   end
 
   it_behaves_like 'a bucket'
 
-  it 'should resolve from URL' do
-    bucket = BFS.resolve("s3://#{sandbox[:bucket]}/?acl=private&encoding=binary")
+  it 'resolves from URL' do
+    bucket = BFS.resolve("s3://#{bucket_name}/?acl=private&encoding=binary")
     expect(bucket).to be_instance_of(described_class)
-    expect(bucket.name).to eq(sandbox[:bucket])
+    expect(bucket.name).to eq(bucket_name)
     expect(bucket.acl).to eq(:private)
     expect(bucket.encoding).to eq('binary')
     expect(bucket.instance_variable_get(:@prefix)).to be_nil
     bucket.close
 
-    bucket = BFS.resolve("s3://#{sandbox[:bucket]}/a/b/")
+    bucket = BFS.resolve("s3://#{bucket_name}/a/b/")
     expect(bucket).to be_instance_of(described_class)
-    expect(bucket.name).to eq(sandbox[:bucket])
+    expect(bucket.name).to eq(bucket_name)
     expect(bucket.instance_variable_get(:@prefix)).to eq('a/b')
     bucket.close
   end
 
-  it 'should enumerate over a large number of files' do
-    bucket = described_class.new sandbox[:bucket], prefix: 'm/'
+  it 'enumerates over a large number of files' do
+    bucket = described_class.new bucket_name, prefix: 'm/'
     expect(bucket.ls('**/*').count).to eq(2121)
     bucket.close
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bfs-s3
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.8.3
 platform: ruby
 authors:
 - Dimitrij Denissenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-02 00:00:00.000000000 Z
+date: 2021-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.7.5
+        version: 0.8.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.7.5
+        version: 0.8.3
 description: https://github.com/bsm/bfs.rb
 email: dimitrij@blacksquaremedia.com
 executables: []
@@ -67,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.15
 signing_key: 
 specification_version: 4
 summary: S3 bucket adapter for bfs