bfs-s3 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 770bbfff15bd69519f2349867f883a6c47c839de21547fc9d323d5809e76dfd0
-  data.tar.gz: 47ee1a747cb2a821833bab758b06c38160b06bb11f4b9dc7b68d43bf706f53e3
+  metadata.gz: 4103371b6039d46aba27c96dc7ec740935ffa8353a1d52d09b1225ba6f2e152d
+  data.tar.gz: 179845d43b49fd2681230f3a3a064496983e5156382575e7c234aeda654bfc04
 SHA512:
-  metadata.gz: c5c5037ec228f4945c4d530c14d3f4c62982088462ea91d84dc06c66eec9fa2de6d1e1ce19f8b4957fd7d1a8ba2c32b8df1aaadd1b170383d860857b06407499
-  data.tar.gz: 8b5596aefdc9333fdb5f4b0f242cea295d2d7b80302c20fa2c971ababbce1d795325eb935c5f75633d9dfb4ec543a7bf4b32b689faa4d3140fa1c12bd7eb7ad2
+  metadata.gz: 54a4afbfaae7f55b40fdeeab4dba2990a133525e2d513c65c13036e15703d65394e0cc581141448cdaed79c9f5435bc4087d78d5f56963c69c70992059989445
+  data.tar.gz: 84d9af9c6f5abc03751d158e9716aaa58841ded7f182750c11c5f06744fca3931e1f7e8e7ae7715fb88ef735b03958369a792fe943eaf717237e78666df4d918

data/bfs-s3.gemspec

@@ -18,5 +18,5 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.2.0'
 
   s.add_dependency 'aws-sdk-s3'
-  s.add_dependency 'bfs'
+  s.add_dependency 'bfs', s.version
 end

data/lib/bfs/bucket/s3.rb

@@ -36,9 +36,17 @@ module BFS
 
       # Lists the contents of a bucket using a glob pattern
       def ls(pattern='**/*', opts={})
-        @client.list_objects_v2(opts.merge(bucket: name)).contents.select do |obj|
-          File.fnmatch?(pattern, obj.key, File::FNM_PATHNAME)
-        end.map(&:key)
+        next_token = nil
+        Enumerator.new do |y|
+          loop do
+            resp = @client.list_objects_v2 opts.merge(bucket: name, continuation_token: next_token)
+            resp.contents.each do |obj|
+              y << obj.key if File.fnmatch?(pattern, obj.key, File::FNM_PATHNAME)
+            end
+            next_token = resp.next_continuation_token.to_s
+            break if next_token.empty?
+          end
+        end
       end
 
       # Info returns the object info
@@ -85,7 +93,7 @@ module BFS
       # Opens an existing file for reading
       def open(path, opts={}, &block)
         path = norm_path(path)
-        temp = Tempfile.new(File.basename(path))
+        temp = Tempfile.new(File.basename(path), binmode: true)
         temp.close
 
         opts = opts.merge(

data/spec/bfs/bucket/s3_spec.rb

@@ -1,59 +1,42 @@
 require 'spec_helper'
 
 RSpec.describe BFS::Bucket::S3 do
-  let(:client)  { double('Aws::S3::Client') }
-  let(:files)   { {} }
-  subject { described_class.new('mock-bucket', client: client) }
+  let(:files) { {} }
+  let :mock_client do
+    client = double Aws::S3::Client
 
-  # stub put_object calls and store file data
-  before do
     allow(client).to receive(:put_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
       files[opts[:key]] = opts[:body].read
       nil
     end
-  end
 
-  # stub get_object calls
-  before do
     allow(client).to receive(:get_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
       data = files[opts[:key]]
       raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless data
 
-      File.open(opts[:response_target], 'w') {|f| f.write(data) }
+      File.open(opts[:response_target], 'wb') {|f| f.write(data) }
       nil
     end
-  end
 
-  before do
     allow(client).to receive(:delete_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
       raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless files.key?(opts[:key])
 
       files.delete(opts[:key])
       nil
     end
-  end
 
-  # stub list_objects_v2 calls
-  before do
-    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket') do |*|
+    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', continuation_token: nil) do |*|
       contents = files.keys.map {|key| Aws::S3::Types::Object.new(key: key) }
-      double 'ListObjectsV2Response', contents: contents
+      double 'ListObjectsV2Response', contents: contents, next_continuation_token: ''
+    end
+    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', max_keys: 1, prefix: 'a/b/c.txt') do |*|
+      obj = Aws::S3::Types::Object.new(key: 'a/b/c.txt', size: 10, last_modified: Time.now)
+      double 'ListObjectsV2Response', contents: [obj]
+    end
+    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', max_keys: 1, prefix: 'missing.txt') do |*|
+      double 'ListObjectsV2Response', contents: []
     end
-  end
-
-  # stub list_objects_v2, single object calls
-  before do
-    match = double 'ListObjectsV2Response', contents: [
-      Aws::S3::Types::Object.new(key: 'a/b/c.txt', size: 10, last_modified: Time.now),
-    ]
-    no_match = double 'ListObjectsV2Response', contents: []
-
-    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', max_keys: 1, prefix: 'a/b/c.txt').and_return(match)
-    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', max_keys: 1, prefix: 'missing.txt').and_return(no_match)
-  end
 
-  # stub copy_object calls
-  before do
     allow(client).to receive(:copy_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
       src = opts[:copy_source].sub('/mock-bucket/', '')
       raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless files.key?(src)
@@ -61,8 +44,11 @@ RSpec.describe BFS::Bucket::S3 do
       files[opts[:key]] = files[src]
       nil
     end
+
+    client
   end
 
+  subject { described_class.new('mock-bucket', client: mock_client) }
   it_behaves_like 'a bucket'
 
   it 'should resolve from URL' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bfs-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Dimitrij Denissenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-18 00:00:00.000000000 Z
+date: 2018-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -28,16 +28,16 @@ dependencies:
   name: bfs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.0
 description: https://github.com/bsm/bfs.rb
 email: dimitrij@blacksquaremedia.com
 executables: []