bfs-s3 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e4e2d75bba619dbe3e7a68ff669b91cb5e031acbac453683f6197d11a7cc98d
-  data.tar.gz: 1b92cdd4b1cf3bfbd2b173498de47d7b27528f590010fe5ab5256f6bd391d373
+  metadata.gz: 4d6df20270797fc308401a911b078872076ce9cb248bec9fd7c725fec0d034a3
+  data.tar.gz: 27cc25f8e2d2f4a854d83decf3fffdff7771c67b9bd69c41d87f0b23a79903fd
 SHA512:
-  metadata.gz: d5848551c6fe1b2cc699c3d0cdb47010e2806296cc3cb7734c8a40ede206b566d49855110e70fca2917b65cd92f943497e6acec00b6c17713f3f7a04efb7142d
-  data.tar.gz: 4e410ead67a7cf7886b14224d216311f759f1be92dc42d85e4ccf11772c935e22fbd723db3f26274f729ac445b15cb56f847c76fa3444799e5d80b7e9cfed398
+  metadata.gz: 3c370605ebd6d535da63e198dfc5141e7343eb1b3a55ee3d34c9714d7a83157d2a922089cc875f84b369e6e87181849c1a3ab3ec22b35bcda168614329cfa982
+  data.tar.gz: ad263a8e5ee1c2e6423d476b7a65b7d2d227a39002c6d495d8d672bdb22a909bc50c802b306e6ef1a2c7f45ba45246fe24de7daf070a26524319b05f4da07dd9

data/lib/bfs/bucket/s3.rb

@@ -33,22 +33,19 @@ module BFS
 
       # Lists the contents of a bucket using a glob pattern
       def ls(pattern = '**/*', **opts)
-        prefix = pattern[%r{^[^*?\{\}\[\]]+/}]
-        prefix = File.join(*[@prefix, prefix].compact) if @prefix
-
-        opts = opts.merge(bucket: name, prefix: @prefix)
-        opts[:prefix] = prefix if prefix
+        Enumerator.new do |acc|
+          walk(pattern, **opts) do |path, _|
+            acc << path
+          end
+        end
+      end
 
-        next_token = nil
-        Enumerator.new do |y|
-          loop do
-            resp = @client.list_objects_v2 opts.merge(continuation_token: next_token)
-            resp.contents.each do |obj|
-              name = trim_prefix(obj.key)
-              y << name if File.fnmatch?(pattern, name, File::FNM_PATHNAME)
-            end
-            next_token = resp.next_continuation_token.to_s
-            break if next_token.empty?
+      # Iterates over the contents of a bucket using a glob pattern
+      def glob(pattern = '**/*', **opts)
+        Enumerator.new do |acc|
+          walk(pattern, **opts) do |path, obj|
+            info = BFS::FileInfo.new(path: path, size: obj.size, mtime: obj.last_modified)
+            acc << info
           end
         end
       end
@@ -60,7 +57,7 @@ module BFS
         info = @client.head_object(**opts)
         raise BFS::FileNotFound, path unless info
 
-        BFS::FileInfo.new(path: path, size: info.content_length, mtime: info.last_modified, content_type: info.content_type, metadata: norm_meta(info.metadata))
+        BFS::FileInfo.new path: path, size: info.content_length, mtime: info.last_modified, content_type: info.content_type, metadata: norm_meta(info.metadata)
       rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound
         raise BFS::FileNotFound, path
       end
@@ -144,9 +141,38 @@ module BFS
         config[:credentials] = opts[:credentials] if opts[:credentials]
         config[:credentials] ||= Aws::Credentials.new(opts[:access_key_id].to_s, opts[:secret_access_key].to_s) if opts[:access_key_id]
         config[:credentials] ||= Aws::SharedCredentials.new(profile_name: opts[:profile_name]) if opts[:profile_name]
+        config[:credentials] = assume_role_credentials(opts[:assume_role], config[:credentials]) if opts[:assume_role]
 
         Aws::S3::Client.new(config)
       end
+
+      def assume_role_credentials(role_arn, credentials = nil)
+        opts = {
+          role_arn: role_arn,
+          role_session_name: SecureRandom.urlsafe_base64(12),
+        }
+        opts[:client] = Aws::STS::Client.new(credentials: credentials) if credentials
+        Aws::AssumeRoleCredentials.new(**opts)
+      end
+
+      def walk(pattern, **opts)
+        prefix = pattern[%r{^[^*?\{\}\[\]]+/}]
+        prefix = File.join(*[@prefix, prefix].compact) if @prefix
+
+        opts = opts.merge(bucket: name, prefix: @prefix)
+        opts[:prefix] = prefix if prefix
+
+        next_token = nil
+        loop do
+          resp = @client.list_objects_v2 opts.merge(continuation_token: next_token)
+          resp.contents.each do |obj|
+            path = trim_prefix(obj.key)
+            yield(path, obj) if File.fnmatch?(pattern, path, File::FNM_PATHNAME)
+          end
+          next_token = resp.next_continuation_token.to_s
+          break if next_token.empty?
+        end
+      end
     end
   end
 end
@@ -154,7 +180,7 @@ end
 BFS.register('s3') do |url, opts, block|
   prefix = BFS.norm_path(opts[:prefix] || url.path)
   opts[:prefix] = prefix.empty? ? nil : prefix
-  opts = opts.slice(:prefix, :region, :sse, :access_key_id, :secret_access_key, :acl, :storage_class, :encoding)
+  opts = opts.slice(:prefix, :region, :sse, :access_key_id, :secret_access_key, :acl, :storage_class, :encoding, :assume_role)
 
   BFS::Bucket::S3.open url.host, **opts, &block
 end

data/spec/bfs/bucket/s3_spec.rb

@@ -3,20 +3,20 @@ require 'spec_helper'
 bucket_name = 'bsm-bfs-unittest'
 
 RSpec.describe BFS::Bucket::S3, s3: true do
-  let(:prefix) { "x/#{SecureRandom.uuid}/" }
-
   subject do
     described_class.new bucket_name, prefix: prefix
   end
 
-  after :all do
-    bucket = described_class.new bucket_name, prefix: 'x/'
+  let(:prefix) { "x/#{SecureRandom.uuid}/" }
+
+  after do
+    bucket = described_class.new bucket_name, prefix: prefix
     bucket.ls.each {|name| bucket.rm(name) }
   end
 
   it_behaves_like 'a bucket'
 
-  it 'should resolve from URL' do
+  it 'resolves from URL' do
     bucket = BFS.resolve("s3://#{bucket_name}/?acl=private&encoding=binary")
     expect(bucket).to be_instance_of(described_class)
     expect(bucket.name).to eq(bucket_name)
@@ -32,7 +32,7 @@ RSpec.describe BFS::Bucket::S3, s3: true do
     bucket.close
   end
 
-  it 'should enumerate over a large number of files' do
+  it 'enumerates over a large number of files' do
     bucket = described_class.new bucket_name, prefix: 'm/'
     expect(bucket.ls('**/*').count).to eq(2121)
     bucket.close

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bfs-s3
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Dimitrij Denissenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-01 00:00:00.000000000 Z
+date: 2021-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.0
 description: https://github.com/bsm/bfs.rb
 email: dimitrij@blacksquaremedia.com
 executables: []
@@ -67,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.15
 signing_key: 
 specification_version: 4
 summary: S3 bucket adapter for bfs