bucket_store 0.5.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14fe28facb1db7af27414a3d1ce64ad24974c79e72c5b9faca796860f679c3d8
-  data.tar.gz: 4b6a4fe4e68bb81b4e7df6d76d2f8e14dc6210508955263fe579b067c7b1af61
+  metadata.gz: 0a1d87ab8bc67271f2a7dcc656d78a8b0a57083d2251534a851a882067efc48d
+  data.tar.gz: 32d35e884c672c6b45d815bc022c2c2937ff741686ce52ea24b022b493f0b65a
 SHA512:
-  metadata.gz: 60b864052d68196fd7bda4a67a67bb66a82ef9bb828f01fb9364ce8aeca67ea414f8f2c34259982fd3dd76a3545b9c72172400e3acbb277fe3cc907d5d449e36
-  data.tar.gz: 5aab2c949368e330f73470f5a9c705c681662ec7209d8e8600c3ea971f852c79d2ce6f6af52350760c9f585cc25ab94f51431001cbb4ff053276333515bf8fd1
+  metadata.gz: f3b67bfee44e2b627522acd7dc6b068038ebed5e472cdd82f32f5cac1774b762e165d65107c5faf2f5bda7926e4e630206323d293ee40ad2537f6114680da523
+  data.tar.gz: ba9679d3ed934f439195893ad8d5d6b9d18b4a638d167bb385416c3b12c3f4da28df74cf076cd2a5cd6065e982ec80356fd3893c0d645e396eec1ee45e47ae00

data/README.md

@@ -135,18 +135,34 @@ in mind:
 
 ## Examples
 
-### Uploading a file to a bucket
+### Uploading a string to a bucket
 ```ruby
 BucketStore.for("inmemory://bucket/path/file.xml").upload!("hello world")
 => "inmemory://bucket/path/file.xml"
 ```
 
-### Accessing a file in a bucket
+### Accessing a string in a bucket
 ```ruby
 BucketStore.for("inmemory://bucket/path/file.xml").download
 => {:bucket=>"bucket", :key=>"path/file.xml", :content=>"hello world"}
 ```
 
+### Uploading a file-like object to a bucket
+```ruby
+buffer = StringIO.new("This could also be an actual file")
+BucketStore.for("inmemory://bucket/path/file.xml").stream.upload!(file: buffer)
+=> "inmemory://bucket/path/file.xml"
+```
+
+### Downloading to a file-like object from a bucket
+```ruby
+buffer = StringIO.new
+BucketStore.for("inmemory://bucket/path/file.xml").stream.download(file: buffer)
+=> {:bucket=>"bucket", :key=>"path/file.xml", :file=>buffer}
+buffer.string 
+=> "This could also be an actual file"
+```
+
 ### Listing all keys under a prefix
 ```ruby
 BucketStore.for("inmemory://bucket/path/").list
@@ -163,10 +179,14 @@ BucketStore.for("inmemory://bucket/path/file.xml").delete!
 
 ### Running tests
 BucketStore comes with both unit and integration tests. While unit tests can be run by simply
-executing `bundle exec rspec`, integration tests require running minio locally. We provide an
-helper script (`scripts/run-minio.sh`) that spins up a pre-configured docker container with
-a single test bucket. Once minio has started, integration tests can be executed with
-`bundle exec rspec --tag integration`.
+executing `bundle exec rspec`, integration tests require running minio locally. We provide a
+docker-compose file that spins up pre-configured simulator instances for S3 and GCS with
+test buckets. Running the integration tests is as easy as:
+
+```
+docker-compose up
+bundle exec rspec --tag integration
+```
 
 ## License & Contributing
 

data/lib/bucket_store/disk.rb

@@ -13,23 +13,22 @@ module BucketStore
       @base_dir = File.expand_path(base_dir)
     end
 
-    def upload!(bucket:, key:, content:)
-      File.open(key_path(bucket, key), "w") do |file|
-        file.write(content)
+    def upload!(bucket:, key:, file:)
+      File.open(key_path(bucket, key), "w") do |output_file|
+        output_file.write(file.read)
+        output_file.rewind
       end
+
       {
         bucket: bucket,
         key: key,
       }
     end
 
-    def download(bucket:, key:)
-      File.open(key_path(bucket, key), "r") do |file|
-        {
-          bucket: bucket,
-          key: key,
-          content: file.read,
-        }
+    def download(bucket:, key:, file:)
+      File.open(key_path(bucket, key), "r") do |saved_file|
+        file.write(saved_file.read)
+        file.rewind
       end
     end
 

data/lib/bucket_store/gcs.rb

@@ -3,25 +3,42 @@
 require "stringio"
 require "uri"
 
-require "google/cloud/storage"
-
 module BucketStore
   class Gcs
     DEFAULT_TIMEOUT_SECONDS = 30
 
+    def self.load_client_library
+      @load_client_library ||= require "google/cloud/storage"
+    end
+
     def self.build(timeout_seconds = DEFAULT_TIMEOUT_SECONDS)
-      Gcs.new(timeout_seconds)
+      new(timeout_seconds)
     end
 
     def initialize(timeout_seconds)
-      @storage = Google::Cloud::Storage.new(
+      self.class.load_client_library
+
+      # Ruby's GCS library does not natively support setting up a simulator, but it allows
+      # for a specific endpoint to be passed down which has the same effect. The simulator
+      # needs to be special cased as in that case we want to bypass authentication,
+      # which we can only do by accessing the `.anonymous` version of the Storage class.
+      simulator_endpoint = ENV["STORAGE_EMULATOR_HOST"]
+      is_simulator = !simulator_endpoint.nil?
+
+      args = {
+        endpoint: simulator_endpoint,
         timeout: timeout_seconds,
-      )
+      }.compact
+
+      @storage = if is_simulator
+                   Google::Cloud::Storage.anonymous(**args)
+                 else
+                   Google::Cloud::Storage.new(**args)
+                 end
     end
 
-    def upload!(bucket:, key:, content:)
-      buffer = StringIO.new(content)
-      get_bucket(bucket).create_file(buffer, key)
+    def upload!(bucket:, key:, file:)
+      get_bucket(bucket).create_file(file, key)
 
       {
         bucket: bucket,
@@ -29,17 +46,14 @@ module BucketStore
       }
     end
 
-    def download(bucket:, key:)
-      file = get_bucket(bucket).file(key)
+    def download(bucket:, key:, file:)
+      file.tap do |f|
+        get_bucket(bucket).
+          file(key).
+          download(f)
 
-      buffer = StringIO.new
-      file.download(buffer)
-
-      {
-        bucket: bucket,
-        key: key,
-        content: buffer.string,
-      }
+        f.rewind
+      end
     end
 
     def list(bucket:, key:, page_size:)

data/lib/bucket_store/in_memory.rb

@@ -24,8 +24,10 @@ module BucketStore
       @buckets = Hash.new { |hash, key| hash[key] = {} }
     end
 
-    def upload!(bucket:, key:, content:)
-      @buckets[bucket][key] = content
+    def upload!(bucket:, key:, file:)
+      file.tap do |f|
+        @buckets[bucket][key] = f.read
+      end
 
       {
         bucket: bucket,
@@ -33,12 +35,11 @@ module BucketStore
       }
     end
 
-    def download(bucket:, key:)
-      {
-        bucket: bucket,
-        key: key,
-        content: @buckets[bucket].fetch(key),
-      }
+    def download(bucket:, key:, file:)
+      file.tap do |f|
+        f.write(@buckets[bucket].fetch(key))
+        f.rewind
+      end
     end
 
     def list(bucket:, key:, page_size:)

data/lib/bucket_store/key_storage.rb

@@ -15,6 +15,90 @@ module BucketStore
       disk: Disk,
     }.freeze
 
+    # Defines a streaming interface for download and upload operations.
+    #
+    # Note that individual adapters may require additional configuration for the correct
+    # behavior of the streaming interface.
+    class KeyStreamer
+      attr_reader :bucket, :key, :adapter_type
+
+      def initialize(adapter:, adapter_type:, bucket:, key:)
+        @adapter = adapter
+        @adapter_type = adapter_type
+        @bucket = bucket
+        @key = key
+      end
+
+      # Streams the content of the reference key into a file-like object
+      # @param [IO] file a writeable IO instance, or a file-like object such as `StringIO`
+      # @return hash containing the bucket, the key and file like object passed in as input
+      #
+      # @see KeyStorage#download
+      # @example Download a key
+      #   buffer = StringIO.new
+      #   BucketStore.for("inmemory://bucket/file.xml").stream.download(file: buffer)
+      #   buffer.string == "Imagine I'm a 2GB file"
+      def download(file:)
+        BucketStore.logger.info(event: "key_storage.download_started")
+
+        start = BucketStore::Timing.monotonic_now
+        adapter.download(
+          bucket: bucket,
+          key: key,
+          file: file,
+        )
+
+        BucketStore.logger.info(event: "key_storage.download_finished",
+                                duration: BucketStore::Timing.monotonic_now - start)
+
+        {
+          bucket: bucket,
+          key: key,
+          file: file,
+        }
+      end
+
+      # Performs a streaming upload to the backing object store
+      # @param [IO] file a readable IO instance, or a file-like object such as `StringIO`
+      # @return the generated key for the new object
+      #
+      # @see KeyStorage#upload!
+      # @example Upload a key
+      #   buffer = StringIO.new("Imagine I'm a 2GB file")
+      #   BucketStore.for("inmemory://bucket/file.xml").stream.upload!(file: buffer)
+      def upload!(file:)
+        raise ArgumentError, "Key cannot be empty" if key.empty?
+
+        BucketStore.logger.info(event: "key_storage.upload_started",
+                                **log_context)
+
+        start = BucketStore::Timing.monotonic_now
+        adapter.upload!(
+          bucket: bucket,
+          key: key,
+          file: file,
+        )
+
+        BucketStore.logger.info(event: "key_storage.upload_finished",
+                                duration: BucketStore::Timing.monotonic_now - start,
+                                **log_context)
+
+        "#{adapter_type}://#{bucket}/#{key}"
+      end
+
+      private
+
+      attr_reader :adapter
+
+      def log_context
+        {
+          bucket: bucket,
+          key: key,
+          adapter_type: adapter_type,
+        }.compact
+      end
+    end
+
     attr_reader :bucket, :key, :adapter_type
 
     def initialize(adapter:, bucket:, key:)
@@ -40,45 +124,32 @@ module BucketStore
     # @example Download a key
     #   BucketStore.for("inmemory://bucket/file.xml").download
     def download
-      raise ArgumentError, "Key cannot be empty" if key.empty?
-
-      BucketStore.logger.info(event: "key_storage.download_started")
-
-      start = BucketStore::Timing.monotonic_now
-      result = adapter.download(bucket: bucket, key: key)
-
-      BucketStore.logger.info(event: "key_storage.download_finished",
-                              duration: BucketStore::Timing.monotonic_now - start)
-
-      result
+      buffer = StringIO.new
+      stream.download(file: buffer).tap do |result|
+        result.delete(:file)
+        result[:content] = buffer.string
+      end
     end
 
-    # Uploads the given content to the reference key location.
+    # Uploads the given file to the reference key location.
     #
     # If the `key` already exists, its content will be replaced by the one in input.
     #
-    # @param [String] content The content to upload
+    # @param [String] content Contents of the file
     # @return [String] The final `key` where the content has been uploaded
     # @example Upload a file
-    #   BucketStore.for("inmemory://bucket/file.xml").upload("hello world")
+    #   BucketStore.for("inmemory://bucket/file.xml").upload!("hello world")
     def upload!(content)
-      raise ArgumentError, "Key cannot be empty" if key.empty?
-
-      BucketStore.logger.info(event: "key_storage.upload_started",
-                              **log_context)
-
-      start = BucketStore::Timing.monotonic_now
-      result = adapter.upload!(
-        bucket: bucket,
-        key: key,
-        content: content,
-      )
+      stream.upload!(file: StringIO.new(content))
+    end
 
-      BucketStore.logger.info(event: "key_storage.upload_finished",
-                              duration: BucketStore::Timing.monotonic_now - start,
-                              **log_context)
+    # Returns an interface for streaming operations
+    #
+    # @return [KeyStreamer] An interface for streaming operations
+    def stream
+      raise ArgumentError, "Key cannot be empty" if key.empty?
 
-      "#{adapter_type}://#{result[:bucket]}/#{result[:key]}"
+      KeyStreamer.new(adapter: adapter, adapter_type: adapter_type, bucket: bucket, key: key)
     end
 
     # Lists all keys for the current adapter that have the reference key as prefix
@@ -159,13 +230,5 @@ module BucketStore
     private
 
     attr_reader :adapter
-
-    def log_context
-      {
-        bucket: bucket,
-        key: key,
-        adapter_type: adapter_type,
-      }.compact
-    end
   end
 end

data/lib/bucket_store/s3.rb

@@ -2,29 +2,33 @@
 
 require "uri"
 
-require "aws-sdk-s3"
-
 module BucketStore
   class S3
     DEFAULT_TIMEOUT_SECONDS = 30
 
+    def self.load_client_library
+      @load_client_library ||= require "aws-sdk-s3"
+    end
+
     def self.build(open_timeout_seconds = DEFAULT_TIMEOUT_SECONDS,
                    read_timeout_seconds = DEFAULT_TIMEOUT_SECONDS)
-      S3.new(open_timeout_seconds, read_timeout_seconds)
+      new(open_timeout_seconds, read_timeout_seconds)
     end
 
     def initialize(open_timeout_seconds, read_timeout_seconds)
+      self.class.load_client_library
+
       @storage = Aws::S3::Client.new(
         http_open_timeout: open_timeout_seconds,
         http_read_timeout: read_timeout_seconds,
       )
     end
 
-    def upload!(bucket:, key:, content:)
+    def upload!(bucket:, key:, file:)
       storage.put_object(
         bucket: bucket,
         key: key,
-        body: content,
+        body: file,
       )
 
       {
@@ -33,17 +37,12 @@ module BucketStore
       }
     end
 
-    def download(bucket:, key:)
-      file = storage.get_object(
+    def download(bucket:, key:, file:)
+      storage.get_object(
+        response_target: file,
         bucket: bucket,
         key: key,
       )
-
-      {
-        bucket: bucket,
-        key: key,
-        content: file.body.read,
-      }
     end
 
     def list(bucket:, key:, page_size:)

data/lib/bucket_store/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BucketStore
-  VERSION = "0.5.0"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bucket_store
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.0
 platform: ruby
 authors:
 - GoCardless Engineering
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-09 00:00:00.000000000 Z
+date: 2024-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -16,58 +16,44 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.104'
+        version: '1.147'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.104'
+        version: '1.147'
 - !ruby/object:Gem::Dependency
   name: google-cloud-storage
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.34'
+        version: '1.50'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.34'
+        version: '1.50'
 - !ruby/object:Gem::Dependency
   name: gc_ruboconfig
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.29'
+        version: 4.4.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.29'
+        version: 4.4.2
 - !ruby/object:Gem::Dependency
   name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.9'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.9'
-- !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -81,61 +67,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.10'
 - !ruby/object:Gem::Dependency
-  name: rspec_junit_formatter
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.4.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.4.1
-- !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: '3.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: '3.13'
 - !ruby/object:Gem::Dependency
-  name: rubocop-performance
+  name: rspec-github
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: 2.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
-  name: rubocop-rspec
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '1.63'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '1.63'
 description: "    A helper library to access cloud storage services such as Google
   Cloud Storage or S3.\n"
 email:
@@ -159,7 +131,8 @@ files:
 homepage: https://github.com/gocardless/bucket-store
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -168,14 +141,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: A helper library to access cloud storage services