bfs-s3 0.7.6 → 0.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7b0a87bb99ee39fefb3d297a891b96c740793529a187a8f19a3a9f37219b1e5
-  data.tar.gz: 4d38e77a1b7f0a1f3a9591dfd4f00a038aec0c43f9dda60203049d9036df8004
+  metadata.gz: 759ac37f6057cd72d3a8126cbd2e733745efd8d2a5e0845b0fe1ffa7e4555eba
+  data.tar.gz: f988f4496cc91538a70326eba61896b2309b18e70a52604d7389a4857df22a2c
 SHA512:
-  metadata.gz: d79da1d49939071bd04072e3120d4e4d4dd7c35c78cae2eba0010e6e7a0900052cc92970ba6e56c17bb1b6595b0a5553d3d5fb274140429fd47ca9b57ba5eb17
-  data.tar.gz: 38831316b1684be8981982309b4d85e03b2c57e6ec29ed6e245cb8ddaa042166b5a4ca0479af621e5fa56eb0cf2740d2274029d8a4311a658428074e62f699c5
+  metadata.gz: 0db4458fc49076e9c9780c69864d53b21298f286360b4441635210162ad47be74e0c537faac51060a00aaca033aeae063909e84b8a519e0bea5dc47fab56d293
+  data.tar.gz: a44ed9a7876cca15c41209207dd8045666bd691f120573d8a04e98b41e40b3911c7766c794e2db21fe9cf4de64bc7037c5a36cf21ad5bfe6117f3b1dcb5cf944

data/lib/bfs/bucket/s3.rb

@@ -33,22 +33,19 @@ module BFS
 
       # Lists the contents of a bucket using a glob pattern
       def ls(pattern = '**/*', **opts)
-        prefix = pattern[%r{^[^*?\{\}\[\]]+/}]
-        prefix = File.join(*[@prefix, prefix].compact) if @prefix
-
-        opts = opts.merge(bucket: name, prefix: @prefix)
-        opts[:prefix] = prefix if prefix
+        Enumerator.new do |acc|
+          walk(pattern, **opts) do |path, _|
+            acc << path
+          end
+        end
+      end
 
-        next_token = nil
-        Enumerator.new do |y|
-          loop do
-            resp = @client.list_objects_v2 opts.merge(continuation_token: next_token)
-            resp.contents.each do |obj|
-              name = trim_prefix(obj.key)
-              y << name if File.fnmatch?(pattern, name, File::FNM_PATHNAME)
-            end
-            next_token = resp.next_continuation_token.to_s
-            break if next_token.empty?
+      # Iterates over the contents of a bucket using a glob pattern
+      def glob(pattern = '**/*', **opts)
+        Enumerator.new do |acc|
+          walk(pattern, **opts) do |path, obj|
+            info = BFS::FileInfo.new(path: path, size: obj.size, mtime: obj.last_modified)
+            acc << info
           end
         end
       end
@@ -60,7 +57,7 @@ module BFS
         info = @client.head_object(**opts)
         raise BFS::FileNotFound, path unless info
 
-        BFS::FileInfo.new(path: path, size: info.content_length, mtime: info.last_modified, content_type: info.content_type, metadata: norm_meta(info.metadata))
+        BFS::FileInfo.new path: path, size: info.content_length, mtime: info.last_modified, content_type: info.content_type, metadata: norm_meta(info.metadata)
       rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound
         raise BFS::FileNotFound, path
       end
@@ -82,7 +79,7 @@ module BFS
         opts[:server_side_encryption] ||= @sse if @sse
         opts[:storage_class] ||= @storage_class if @storage_class
 
-        BFS::TempWriter.new(path, encoding: encoding, perm: perm) do |t|
+        BFS::Writer.new(path, encoding: encoding, perm: perm) do |t|
           File.open(t, encoding: encoding) do |file|
             @client.put_object(opts.merge(body: file))
           end
@@ -144,9 +141,38 @@ module BFS
         config[:credentials] = opts[:credentials] if opts[:credentials]
         config[:credentials] ||= Aws::Credentials.new(opts[:access_key_id].to_s, opts[:secret_access_key].to_s) if opts[:access_key_id]
         config[:credentials] ||= Aws::SharedCredentials.new(profile_name: opts[:profile_name]) if opts[:profile_name]
+        config[:credentials] = assume_role_credentials(opts[:assume_role], config[:credentials]) if opts[:assume_role]
 
         Aws::S3::Client.new(config)
       end
+
+      def assume_role_credentials(role_arn, credentials = nil)
+        opts = {
+          role_arn: role_arn,
+          role_session_name: SecureRandom.urlsafe_base64(12),
+        }
+        opts[:client] = Aws::STS::Client.new(credentials: credentials) if credentials
+        Aws::AssumeRoleCredentials.new(**opts)
+      end
+
+      def walk(pattern, **opts)
+        prefix = pattern[%r{^[^*?\{\}\[\]]+/}]
+        prefix = File.join(*[@prefix, prefix].compact) if @prefix
+
+        opts = opts.merge(bucket: name, prefix: @prefix)
+        opts[:prefix] = prefix if prefix
+
+        next_token = nil
+        loop do
+          resp = @client.list_objects_v2 opts.merge(continuation_token: next_token)
+          resp.contents.each do |obj|
+            path = trim_prefix(obj.key)
+            yield(path, obj) if File.fnmatch?(pattern, path, File::FNM_PATHNAME)
+          end
+          next_token = resp.next_continuation_token.to_s
+          break if next_token.empty?
+        end
+      end
     end
   end
 end
@@ -154,7 +180,7 @@ end
 BFS.register('s3') do |url, opts, block|
   prefix = BFS.norm_path(opts[:prefix] || url.path)
   opts[:prefix] = prefix.empty? ? nil : prefix
-  opts = opts.slice(:prefix, :region, :sse, :access_key_id, :secret_access_key, :acl, :storage_class, :encoding)
+  opts = opts.slice(:prefix, :region, :sse, :access_key_id, :secret_access_key, :acl, :storage_class, :encoding, :assume_role)
 
   BFS::Bucket::S3.open url.host, **opts, &block
 end

data/spec/bfs/bucket/s3_spec.rb

@@ -3,20 +3,20 @@ require 'spec_helper'
 bucket_name = 'bsm-bfs-unittest'
 
 RSpec.describe BFS::Bucket::S3, s3: true do
-  let(:prefix) { "x/#{SecureRandom.uuid}/" }
-
   subject do
     described_class.new bucket_name, prefix: prefix
   end
 
-  after :all do
-    bucket = described_class.new bucket_name, prefix: 'x/'
+  let(:prefix) { "x/#{SecureRandom.uuid}/" }
+
+  after do
+    bucket = described_class.new bucket_name, prefix: prefix
     bucket.ls.each {|name| bucket.rm(name) }
   end
 
   it_behaves_like 'a bucket'
 
-  it 'should resolve from URL' do
+  it 'resolves from URL' do
     bucket = BFS.resolve("s3://#{bucket_name}/?acl=private&encoding=binary")
     expect(bucket).to be_instance_of(described_class)
     expect(bucket.name).to eq(bucket_name)
@@ -32,7 +32,7 @@ RSpec.describe BFS::Bucket::S3, s3: true do
     bucket.close
   end
 
-  it 'should enumerate over a large number of files' do
+  it 'enumerates over a large number of files' do
     bucket = described_class.new bucket_name, prefix: 'm/'
     expect(bucket.ls('**/*').count).to eq(2121)
     bucket.close

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bfs-s3
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.8.4
 platform: ruby
 authors:
 - Dimitrij Denissenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-10 00:00:00.000000000 Z
+date: 2021-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.7.6
+        version: 0.8.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.7.6
+        version: 0.8.4
 description: https://github.com/bsm/bfs.rb
 email: dimitrij@blacksquaremedia.com
 executables: []
@@ -67,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.15
 signing_key: 
 specification_version: 4
 summary: S3 bucket adapter for bfs