bfs-gs 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22b3892b324405cf0bb342b72dba4e6bf3cd8c16943af7b225f8d2367c7a5ba4
-  data.tar.gz: 1b6902a3577e89bbed1a0357a26327dca0cebabc8663322014501c82b7ae687b
+  metadata.gz: 15b66903e512e68c645067504a13c6e67b49892291b0c5c6e8efe620f0add955
+  data.tar.gz: 5ac5868a81d4631356ab1980deb860ef39dd59c9b39b15e4e681db8855c6b854
 SHA512:
-  metadata.gz: edd42acc0466e51285005fdeaf66aceb6de7287c9ea430f80a245221c2a224f75e11aeed7e937ad12d9e5a1cd69428e3d9d7840f7ffce52d957011f2541fac1c
-  data.tar.gz: ec15157af7129148f231b3e91a771cea2bbb4b7c4f5b3cd47e9d92a808bf2210d56fea54cdb0a6288a07d8184fb8a900a2f949d39327c65ec53e052c45423a95
+  metadata.gz: 39bf4f7cda53d992997b65c6a8fc4adf8015616e99787c3c981e03020df2969a4d9bf6c2914d872074ab420d76748fd06a91342ef44e32c956fa8dccacc02fd3
+  data.tar.gz: '01924ee8d89b523203dc4aa02ebb2cb0b25d82233d5503af95274524a65634d8660abe5632c1ddbbf88a11befa54beaabab4650152071448f9f91912fa7c8e24'

data/lib/bfs/bucket/gs.rb

@@ -15,6 +15,7 @@ module BFS
       # @option opts [String] :project_id project ID. Defaults to GCP_PROJECT  env var. Required.
       # @option opts [String, Hash, Google::Auth::Credentials] :credentials
       #   the path to the keyfile as a String, the contents of the keyfile as a Hash, or a Google::Auth::Credentials object.
+      # @option opts [String] :prefix custom namespace within the bucket
       # @option opts [Integer] :retries number of times to retry requests. Default: 3.
       # @option opts [Integer] :timeout request timeout, in seconds.
       # @option opts [String] :acl set the default ACL.
@@ -27,8 +28,9 @@ module BFS
         end
         opts[:project_id] ||= ENV['GCP_PROJECT'] || ENV['GCLOUD_PROJECT']
 
-        acl    = opts.delete(:acl)
-        client = opts.delete(:client) || Google::Cloud::Storage.new(opts)
+        @prefix = opts.delete(:prefix)
+        acl     = opts.delete(:acl)
+        client  = opts.delete(:client) || Google::Cloud::Storage.new(opts)
 
         @name   = name.to_s
         @bucket = client.bucket(@name)
@@ -37,25 +39,31 @@ module BFS
 
       # Lists the contents of a bucket using a glob pattern
       def ls(pattern='**/*', opts={})
+        prefix = pattern[%r{^[^\*\?\{\}\[\]]+/}]
+        prefix = File.join(*[@prefix, prefix].compact) if @prefix
+        opts   = opts.merge(prefix: prefix) if prefix
+
         Enumerator.new do |y|
-          @bucket.files(opts).each do |file|
-            y << file.name if File.fnmatch?(pattern, file.name, File::FNM_PATHNAME)
+          @bucket.files(opts).all do |file|
+            name = trim_prefix(file.name)
+            y << name if File.fnmatch?(pattern, name, File::FNM_PATHNAME)
           end
         end
       end
 
       # Info returns the object info
       def info(path, _opts={})
-        path = norm_path(path)
+        path = full_path(path)
         file = @bucket.file(path)
-        raise BFS::FileNotFound, path unless file
+        raise BFS::FileNotFound, trim_prefix(path) unless file
 
-        BFS::FileInfo.new(path, file.size, file.updated_at, file.content_type, file.metadata)
+        name = trim_prefix(file.name)
+        BFS::FileInfo.new(name, file.size, file.updated_at.to_time, file.content_type, file.metadata)
       end
 
       # Creates a new file and opens it for writing
       def create(path, opts={}, &block)
-        path = norm_path(path)
+        path = full_path(path)
         temp = BFS::TempWriter.new(path) do |t|
           File.open(t, binmode: true) do |file|
             @bucket.create_file(file, path, opts)
@@ -72,9 +80,9 @@ module BFS
 
       # Opens an existing file for reading
       def open(path, opts={}, &block)
-        path = norm_path(path)
+        path = full_path(path)
         file = @bucket.file(path)
-        raise BFS::FileNotFound, path unless file
+        raise BFS::FileNotFound, trim_prefix(path) unless file
 
         temp = Tempfile.new(File.basename(path), binmode: true)
         temp.close
@@ -85,18 +93,18 @@ module BFS
 
       # Deletes a file.
       def rm(path, opts={})
-        path = norm_path(path)
+        path = full_path(path)
         file = @bucket.file(path)
         file.delete(opts) if file
       end
 
       # Copies a file.
       def cp(src, dst, opts={})
-        src  = norm_path(src)
+        src  = full_path(src)
         file = @bucket.file(src)
-        raise BFS::FileNotFound, src unless file
+        raise BFS::FileNotFound, trim_prefix(src) unless file
 
-        file.copy(norm_path(dst), opts)
+        file.copy(full_path(dst), opts)
       end
     end
   end

data/spec/bfs/bucket/gs_spec.rb

@@ -1,55 +1,41 @@
 require 'spec_helper'
 
-RSpec.describe BFS::Bucket::GS do
-  let(:contents)    { {} }
-  let(:mock_client) { double Google::Cloud::Storage, bucket: mock_bucket }
+# silence warnings
+module Google::Auth::CredentialsLoader
+  def warn_if_cloud_sdk_credentials(*); end
+  module_function :warn_if_cloud_sdk_credentials # rubocop:disable Style/AccessModifierDeclarations
+end
 
-  let :mock_bucket do
-    bucket = double Google::Cloud::Storage::Bucket,
-      default_acl: double(Google::Cloud::Storage::Bucket::Acl)
-    allow(bucket).to receive(:create_file) do |io, name, _|
-      contents[name] = double_file(name, io.read)
-    end
-    allow(bucket).to receive(:files) do |_|
-      contents.values
-    end
-    allow(bucket).to receive(:file) do |name|
-      contents[name]
-    end
-    bucket
+sandbox  = { project: 'bsm-affiliates', bucket: 'bsm-bfs-unittest' }.freeze
+run_spec = \
+  begin
+    s = Google::Cloud::Storage.new(project_id: sandbox[:project])
+    !s.bucket(sandbox[:bucket]).nil?
+  rescue Signet::AuthorizationError
+    false
   end
 
-  def double_file(name, data)
-    file = double Google::Cloud::Storage::File,
-      name: name,
-      data: data,
-      size: data.bytesize,
-      content_type: 'text/plain',
-      metadata: {},
-      updated_at: Time.now
-    allow(file).to receive(:download) do |path, _|
-      File.open(path, 'wb') {|f| f.write file.data }
-    end
-    allow(file).to receive(:delete) do |_|
-      contents.delete(file.name)
-      true
-    end
-    allow(file).to receive(:copy) do |dst, _|
-      contents[dst] = double_file(dst, file.data)
-    end
-    file
+RSpec.describe BFS::Bucket::GS, if: run_spec do
+  let(:prefix) { "x/#{SecureRandom.uuid}/" }
+
+  subject do
+    described_class.new sandbox[:bucket], project_id: sandbox[:project], prefix: prefix
+  end
+  after :all do
+    bucket = described_class.new sandbox[:bucket], project_id: sandbox[:project], prefix: 'x/'
+    bucket.ls.each {|name| bucket.rm(name) }
   end
 
-  subject { described_class.new 'mock-bucket', client: mock_client }
   it_behaves_like 'a bucket'
 
   it 'should resolve from URL' do
-    expect(Google::Cloud::Storage).to receive(:new).with(project_id: 'my-project').and_return(mock_client)
-    expect(mock_client).to receive(:bucket).with('mock-bucket').and_return(mock_bucket)
-    expect(mock_bucket.default_acl).to receive(:private!).with(no_args)
-
-    bucket = BFS.resolve('gs://mock-bucket?acl=private&project_id=my-project')
+    bucket = BFS.resolve("gs://#{sandbox[:bucket]}?acl=private&project_id=#{sandbox[:project]}")
     expect(bucket).to be_instance_of(described_class)
-    expect(bucket.name).to eq('mock-bucket')
+    expect(bucket.name).to eq(sandbox[:bucket])
+  end
+
+  it 'should enumerate over a large number of files' do
+    bucket = described_class.new sandbox[:bucket], project_id: sandbox[:project], prefix: 'm/'
+    expect(bucket.ls('**/*').count).to eq(2121)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bfs-gs
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Dimitrij Denissenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-26 00:00:00.000000000 Z
+date: 2018-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.3.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.3.4
 - !ruby/object:Gem::Dependency
   name: google-cloud-storage
   requirement: !ruby/object:Gem::Requirement
@@ -68,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: GS bucket adapter for bfs