bucket_store 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e251b7658b3f9e155f3e9e329d2022e6f4791e502de43bba2b8a0563634ba499
-  data.tar.gz: f66fbec21d4a07e2ff0feb0ad45383eccb71daa13d733cc92f2a0ae54d6cd9f4
+  metadata.gz: 4a5f88aa8903c932c2de1c7ed531bf78f58f72e9785ba8732a4cca7e44605549
+  data.tar.gz: f2d8e01ad1f54acf7c44a71df268576131df013e656dfced91f700b80927f036
 SHA512:
-  metadata.gz: cb3aeab11c33767d349fb4c7efe744000c9a129131cfff3054c308de7971da12e1ea3c32fafeaa8722ddf07049d4e2f0e1cf6ba85b24d6c997ac1f3fe4eb14af
-  data.tar.gz: 5f2d1e70e8f0bc27ccfbf998ec3c4e1c89fb9ed0e179b8b2a808a7345bf4cda38dcc45fbadc842a142a0eb42d51fdb44247159c1fe58f5717cf302a6145ecff3
+  metadata.gz: 22ba164e42115a064b4aa6f7833d3d5c86931fe7c35d2da873725fa2488ab6849ce2481bf09bc9d93f7eab085a0132d957e1d81fad8855e89060663895aa5428
+  data.tar.gz: 9c88e6bf881ecab6686255cabfa416be09985a8f48b200726f89e7589e5925dd5ab13700a865aa69c49b4050d654447c09e802b71852b62ef331c5e80568b369

data/README.md

@@ -135,18 +135,34 @@ in mind:
 
 ## Examples
 
-### Uploading a file to a bucket
+### Uploading a string to a bucket
 ```ruby
 BucketStore.for("inmemory://bucket/path/file.xml").upload!("hello world")
 => "inmemory://bucket/path/file.xml"
 ```
 
-### Accessing a file in a bucket
+### Accessing a string in a bucket
 ```ruby
 BucketStore.for("inmemory://bucket/path/file.xml").download
 => {:bucket=>"bucket", :key=>"path/file.xml", :content=>"hello world"}
 ```
 
+### Uploading a file-like object to a bucket
+```ruby
+buffer = StringIO.new("This could also be an actual file")
+BucketStore.for("inmemory://bucket/path/file.xml").stream.upload!(file: buffer)
+=> "inmemory://bucket/path/file.xml"
+```
+
+### Downloading to a file-like object from a bucket
+```ruby
+buffer = StringIO.new
+BucketStore.for("inmemory://bucket/path/file.xml").stream.download(file: buffer)
+=> {:bucket=>"bucket", :key=>"path/file.xml", :file=>buffer}
+buffer.string 
+=> "This could also be an actual file"
+```
+
 ### Listing all keys under a prefix
 ```ruby
 BucketStore.for("inmemory://bucket/path/").list
@@ -159,6 +175,19 @@ BucketStore.for("inmemory://bucket/path/file.xml").delete!
 => true
 ```
 
+## Development
+
+### Running tests
+BucketStore comes with both unit and integration tests. While unit tests can be run by simply
+executing `bundle exec rspec`, integration tests require running minio locally. We provide a
+docker-compose file that spins up pre-configured simulator instances for S3 and GCS with
+test buckets. Running the integration tests is as easy as:
+
+```
+docker-compose up
+bundle exec rspec --tag integration
+```
+
 ## License & Contributing
 
 * BucketStore is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/lib/bucket_store/disk.rb

@@ -13,23 +13,22 @@ module BucketStore
       @base_dir = File.expand_path(base_dir)
     end
 
-    def upload!(bucket:, key:, content:)
-      File.open(key_path(bucket, key), "w") do |file|
-        file.write(content)
+    def upload!(bucket:, key:, file:)
+      File.open(key_path(bucket, key), "w") do |output_file|
+        output_file.write(file.read)
+        output_file.rewind
       end
+
       {
         bucket: bucket,
         key: key,
       }
     end
 
-    def download(bucket:, key:)
-      File.open(key_path(bucket, key), "r") do |file|
-        {
-          bucket: bucket,
-          key: key,
-          content: file.read,
-        }
+    def download(bucket:, key:, file:)
+      File.open(key_path(bucket, key), "r") do |saved_file|
+        file.write(saved_file.read)
+        file.rewind
       end
     end
 
@@ -78,7 +77,7 @@ module BucketStore
     end
 
     def sanitize_filename(filename)
-      filename.gsub(%r{[^0-9A-z.\-/]}, "_")
+      filename.gsub(%r{[^0-9A-z.\- /]}, "_")
     end
   end
 end

data/lib/bucket_store/gcs.rb

@@ -14,14 +14,27 @@ module BucketStore
     end
 
     def initialize(timeout_seconds)
-      @storage = Google::Cloud::Storage.new(
+      # Ruby's GCS library does not natively support setting up a simulator, but it allows
+      # for a specific endpoint to be passed down which has the same effect. The simulator
+      # needs to be special cased as in that case we want to bypass authentication,
+      # which we can only do by accessing the `.anonymous` version of the Storage class.
+      simulator_endpoint = ENV["STORAGE_EMULATOR_HOST"]
+      is_simulator = !simulator_endpoint.nil?
+
+      args = {
+        endpoint: simulator_endpoint,
         timeout: timeout_seconds,
-      )
+      }.compact
+
+      @storage = if is_simulator
+                   Google::Cloud::Storage.anonymous(**args)
+                 else
+                   Google::Cloud::Storage.new(**args)
+                 end
     end
 
-    def upload!(bucket:, key:, content:)
-      buffer = StringIO.new(content)
-      get_bucket(bucket).create_file(buffer, key)
+    def upload!(bucket:, key:, file:)
+      get_bucket(bucket).create_file(file, key)
 
       {
         bucket: bucket,
@@ -29,17 +42,14 @@ module BucketStore
       }
     end
 
-    def download(bucket:, key:)
-      file = get_bucket(bucket).file(key)
-
-      buffer = StringIO.new
-      file.download(buffer)
+    def download(bucket:, key:, file:)
+      file.tap do |f|
+        get_bucket(bucket).
+          file(key).
+          download(f)
 
-      {
-        bucket: bucket,
-        key: key,
-        content: buffer.string,
-      }
+        f.rewind
+      end
     end
 
     def list(bucket:, key:, page_size:)

data/lib/bucket_store/in_memory.rb

@@ -24,8 +24,10 @@ module BucketStore
       @buckets = Hash.new { |hash, key| hash[key] = {} }
     end
 
-    def upload!(bucket:, key:, content:)
-      @buckets[bucket][key] = content
+    def upload!(bucket:, key:, file:)
+      file.tap do |f|
+        @buckets[bucket][key] = f.read
+      end
 
       {
         bucket: bucket,
@@ -33,12 +35,11 @@ module BucketStore
       }
     end
 
-    def download(bucket:, key:)
-      {
-        bucket: bucket,
-        key: key,
-        content: @buckets[bucket].fetch(key),
-      }
+    def download(bucket:, key:, file:)
+      file.tap do |f|
+        f.write(@buckets[bucket].fetch(key))
+        f.rewind
+      end
     end
 
     def list(bucket:, key:, page_size:)

data/lib/bucket_store/key_context.rb

@@ -19,18 +19,35 @@ module BucketStore
     end
 
     def self.parse(raw_key)
-      uri = URI(raw_key)
+      uri = URI(escape(raw_key))
+
+      scheme = unescape(uri.scheme)
+      bucket = unescape(uri.host)
 
       # A key should never be `nil` but can be empty. Depending on the operation, this may
       # or may not be a valid configuration (e.g. an empty key is likely valid on a
       # `list`, but not during a `download`).
-      key = uri.path.sub!(%r{/}, "") || ""
+      key = unescape(uri.path).sub!(%r{/}, "") || ""
 
-      raise KeyParseException if [uri.scheme, uri.host, key].map(&:nil?).any?
+      raise KeyParseException if [scheme, bucket, key].map(&:nil?).any?
 
-      KeyContext.new(adapter: uri.scheme,
-                     bucket: uri.host,
+      KeyContext.new(adapter: scheme,
+                     bucket: bucket,
                      key: key)
     end
+
+    def self.escape(key)
+      return key if key.nil?
+
+      URI::DEFAULT_PARSER.escape(key)
+    end
+    private_class_method :escape
+
+    def self.unescape(key)
+      return key if key.nil?
+
+      URI::DEFAULT_PARSER.unescape(key)
+    end
+    private_class_method :unescape
   end
 end

data/lib/bucket_store/key_storage.rb

@@ -15,6 +15,90 @@ module BucketStore
       disk: Disk,
     }.freeze
 
+    # Defines a streaming interface for download and upload operations.
+    #
+    # Note that individual adapters may require additional configuration for the correct
+    # behavior of the streaming interface.
+    class KeyStreamer
+      attr_reader :bucket, :key, :adapter_type
+
+      def initialize(adapter:, adapter_type:, bucket:, key:)
+        @adapter = adapter
+        @adapter_type = adapter_type
+        @bucket = bucket
+        @key = key
+      end
+
+      # Streams the content of the reference key into a file-like object
+      # @param [IO] file a writeable IO instance, or a file-like object such as `StringIO`
+      # @return hash containing the bucket, the key and file like object passed in as input
+      #
+      # @see KeyStorage#download
+      # @example Download a key
+      #   buffer = StringIO.new
+      #   BucketStore.for("inmemory://bucket/file.xml").stream.download(file: buffer)
+      #   buffer.string == "Imagine I'm a 2GB file"
+      def download(file:)
+        BucketStore.logger.info(event: "key_storage.download_started")
+
+        start = BucketStore::Timing.monotonic_now
+        adapter.download(
+          bucket: bucket,
+          key: key,
+          file: file,
+        )
+
+        BucketStore.logger.info(event: "key_storage.download_finished",
+                                duration: BucketStore::Timing.monotonic_now - start)
+
+        {
+          bucket: bucket,
+          key: key,
+          file: file,
+        }
+      end
+
+      # Performs a streaming upload to the backing object store
+      # @param [IO] file a readable IO instance, or a file-like object such as `StringIO`
+      # @return the generated key for the new object
+      #
+      # @see KeyStorage#upload!
+      # @example Upload a key
+      #   buffer = StringIO.new("Imagine I'm a 2GB file")
+      #   BucketStore.for("inmemory://bucket/file.xml").stream.upload!(file: buffer)
+      def upload!(file:)
+        raise ArgumentError, "Key cannot be empty" if key.empty?
+
+        BucketStore.logger.info(event: "key_storage.upload_started",
+                                **log_context)
+
+        start = BucketStore::Timing.monotonic_now
+        adapter.upload!(
+          bucket: bucket,
+          key: key,
+          file: file,
+        )
+
+        BucketStore.logger.info(event: "key_storage.upload_finished",
+                                duration: BucketStore::Timing.monotonic_now - start,
+                                **log_context)
+
+        "#{adapter_type}://#{bucket}/#{key}"
+      end
+
+      private
+
+      attr_reader :adapter
+
+      def log_context
+        {
+          bucket: bucket,
+          key: key,
+          adapter_type: adapter_type,
+        }.compact
+      end
+    end
+
     attr_reader :bucket, :key, :adapter_type
 
     def initialize(adapter:, bucket:, key:)
@@ -40,45 +124,32 @@ module BucketStore
     # @example Download a key
     #   BucketStore.for("inmemory://bucket/file.xml").download
     def download
-      raise ArgumentError, "Key cannot be empty" if key.empty?
-
-      BucketStore.logger.info(event: "key_storage.download_started")
-
-      start = BucketStore::Timing.monotonic_now
-      result = adapter.download(bucket: bucket, key: key)
-
-      BucketStore.logger.info(event: "key_storage.download_finished",
-                              duration: BucketStore::Timing.monotonic_now - start)
-
-      result
+      buffer = StringIO.new
+      stream.download(file: buffer).tap do |result|
+        result.delete(:file)
+        result[:content] = buffer.string
+      end
     end
 
-    # Uploads the given content to the reference key location.
+    # Uploads the given file to the reference key location.
     #
     # If the `key` already exists, its content will be replaced by the one in input.
     #
-    # @param [String] content The content to upload
+    # @param [String] content Contents of the file
     # @return [String] The final `key` where the content has been uploaded
     # @example Upload a file
-    #   BucketStore.for("inmemory://bucket/file.xml").upload("hello world")
+    #   BucketStore.for("inmemory://bucket/file.xml").upload!("hello world")
     def upload!(content)
-      raise ArgumentError, "Key cannot be empty" if key.empty?
-
-      BucketStore.logger.info(event: "key_storage.upload_started",
-                              **log_context)
-
-      start = BucketStore::Timing.monotonic_now
-      result = adapter.upload!(
-        bucket: bucket,
-        key: key,
-        content: content,
-      )
+      stream.upload!(file: StringIO.new(content))
+    end
 
-      BucketStore.logger.info(event: "key_storage.upload_finished",
-                              duration: BucketStore::Timing.monotonic_now - start,
-                              **log_context)
+    # Returns an interface for streaming operations
+    #
+    # @return [KeyStreamer] An interface for streaming operations
+    def stream
+      raise ArgumentError, "Key cannot be empty" if key.empty?
 
-      "#{adapter_type}://#{result[:bucket]}/#{result[:key]}"
+      KeyStreamer.new(adapter: adapter, adapter_type: adapter_type, bucket: bucket, key: key)
     end
 
     # Lists all keys for the current adapter that have the reference key as prefix
@@ -153,19 +224,11 @@ module BucketStore
     #
     # @return [bool] `true` if the given key exists, `false` if not
     def exists?
-      list.first == "#{adapter_type}://#{bucket}/#{key}"
+      list(page_size: 1).first == "#{adapter_type}://#{bucket}/#{key}"
     end
 
     private
 
     attr_reader :adapter
-
-    def log_context
-      {
-        bucket: bucket,
-        key: key,
-        adapter_type: adapter_type,
-      }.compact
-    end
   end
 end

data/lib/bucket_store/s3.rb

@@ -20,11 +20,11 @@ module BucketStore
       )
     end
 
-    def upload!(bucket:, key:, content:)
+    def upload!(bucket:, key:, file:)
       storage.put_object(
         bucket: bucket,
         key: key,
-        body: content,
+        body: file,
       )
 
       {
@@ -33,17 +33,12 @@ module BucketStore
       }
     end
 
-    def download(bucket:, key:)
-      file = storage.get_object(
+    def download(bucket:, key:, file:)
+      storage.get_object(
+        response_target: file,
         bucket: bucket,
         key: key,
       )
-
-      {
-        bucket: bucket,
-        key: key,
-        content: file.body.read,
-      }
     end
 
     def list(bucket:, key:, page_size:)

data/lib/bucket_store/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BucketStore
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/bucket_store.rb

@@ -4,6 +4,7 @@ require "bucket_store/version"
 require "bucket_store/configuration"
 require "bucket_store/key_context"
 require "bucket_store/key_storage"
+require "bucket_store/uri_builder"
 
 # An abstraction layer on the top of file cloud storage systems such as Google Cloud
 # Storage or S3. This module exposes a generic interface that allows interoperability
@@ -41,8 +42,8 @@ module BucketStore
     # Given a `key` in the format of `adapter://bucket/key` returns the corresponding
     # adapter that will allow to manipulate (e.g. download, upload or list) such key.
     #
-    # Currently supported adapters are `gs` (Google Cloud Storage), `inmemory` (an
-    # in-memory key-value storage) and `disk` (a disk-backed key-value store).
+    # Currently supported adapters are `gs` (Google Cloud Storage), `s3` (AWS S3),
+    # `inmemory` (an in-memory key-value storage) and `disk` (a disk-backed key-value store).
     #
     # @param [String] key The reference key
     # @return [KeyStorage] An interface to the adapter that can handle requests on the given key

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bucket_store
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - GoCardless Engineering
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-29 00:00:00.000000000 Z
+date: 2023-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.29'
+        version: 4.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.29'
+        version: 4.3.0
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -81,61 +81,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.10'
 - !ruby/object:Gem::Dependency
-  name: rspec_junit_formatter
+  name: rspec-github
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 2.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.22'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.22'
-- !ruby/object:Gem::Dependency
-  name: rubocop-performance
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
-- !ruby/object:Gem::Dependency
-  name: rubocop-rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '1.49'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '1.49'
 description: "    A helper library to access cloud storage services such as Google
   Cloud Storage or S3.\n"
 email:
@@ -159,7 +131,8 @@ files:
 homepage: https://github.com/gocardless/bucket-store
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -168,14 +141,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: A helper library to access cloud storage services