canistor 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af06960c106c4d6a66f2e157cf5392df11bd27182c9014f934658b1981f095c8
-  data.tar.gz: 9c8d3c968c879d31bf6b471d599cc132b1843ed7cf7c8282cc0b21a27b04b351
+  metadata.gz: a6c462186c93c3bac4903417214765c33f95816760224eec56c1fb704cf7d5f2
+  data.tar.gz: 29ae2992af7256c0b4d86de177c61626f5ebf413b51f04c0ed69ba80e7df2db6
 SHA512:
-  metadata.gz: 2275495c50f747fe474ff02c745976dcf1f05a87a8073002ef53fae3b71a3256aff2ebb8d2fc2f3ed18167c064ed6557271839c83d9dd68600b0e1dff306a6c3
-  data.tar.gz: 4fd8d042c1acb1bbcfbda6be5a8a865d24bb0d1160d971a09cb6742b4c4175139e9b4ca442a7c6206d1e7249749f1b2b1ed3c1b7f5b135a92889351ae9e3c87e
+  metadata.gz: 53b6b59d4f5260cc6393996a48b7c20a533bc0f2d5723cba82722044be0bd093184bc666b81e0a6363bf60418a536b62a1d4eb2d5c1859118309ecc2792b2eb5
+  data.tar.gz: 5341e4bef20532532f6f2e7b3669b1fd5130b6bc46fa3f45a851d9fd260d2cf4533f488b1675275f57977e1eb306fdc56e8027d884d8690ed7712a3a48c691a3

data/lib/canistor.rb

@@ -23,28 +23,42 @@ require "thread"
 # credentials to be useful. It can be configured using either the
 # config method on the instance or by specifying the buckets one by one.
 #
-# In the example below Canistor will have two accounts and three buckets. It
+# In the example below Canistor will have two accounts and four buckets. It
 # also specifies which accounts can access the buckets.
 #
 #   Canistor.config(
 #     logger: Rails.logger,
 #     credentials: {
-#       'global' => {
+#       {
 #         access_key_id: 'AKIAIXXXXXXXXXXXXXX1',
 #         secret_access_key: 'phRL+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx1'
 #       },
-#       'accounting' => {
+#       {
 #         access_key_id: 'AKIAIXXXXXXXXXXXXXX2',
 #         secret_access_key: 'phRL+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx2'
 #       }
 #     },
 #     buckets: {
 #       'us-east-1' => {
-#         'com-procore-production-images' => ['global'],
-#         'com-procore-production-books' => ['global', 'accounting']
+#         'io-canistor-production-images' => {
+#           allow_access_keys: ['AKIAIXXXXXXXXXXXXXX1'],
+#           replicate_to: [
+#             'eu-central-1:io-canistor-production-images-replica'
+#           ],
+#           versioned: true
+#         },
+#         'io-canistor-production-books' => {
+#           allow_access_keys: ['AKIAIXXXXXXXXXXXXXX1', 'AKIAIXXXXXXXXXXXXXX2']
+#         }
 #       },
 #       'eu-central-1' => {
-#         'com-procore-production-sales' => ['global']
+#         'io-canistor-production-sales' => {
+#           allow_access_keys: ['AKIAIXXXXXXXXXXXXXX1']
+#         },
+#         'io-canistor-production-images-replica' => {
+#           allow_access_keys: ['AKIAIXXXXXXXXXXXXXX1'],
+#           versioned: true
+#         }
 #       }
 #     }
 #   )
@@ -53,6 +67,10 @@ require "thread"
 # verifies authentication information. It does not implement control lists so
 # all accounts have full access to the buckets and objects.
 #
+# It's possible to turn on replication and versioning. Note that replication is
+# instant in the mock. On actual S3 it takes a while for objects to replicate.
+# Not the entire replication and versioning API is implemented.
+# 
 # The mock can simulate a number of failures. These are triggered by setting
 # the operation which needs to fail on the mock. For more information see
 # [Canistor.fail].
@@ -60,6 +78,9 @@ require "thread"
 # In most cases you should configure the suite to clear the mock before running 
 # each example with [Canistor.clear].
 module Canistor
+  # Raised when Canistor's own configuration is somehow unusable.
+  class ConfigurationError < StandardError; end
+
   class << self
     attr_accessor :logger
     attr_accessor :store
@@ -102,15 +123,15 @@ module Canistor
 
   def self.buckets=(buckets)
     buckets.each do |region, attributes|
-      attributes.each do |bucket, access_key_ids|
+      attributes.each do |bucket, options|
         bucket = create_bucket(region, bucket)
-        bucket.allow_access_to(access_key_ids)
+        bucket.update_settings(options)
         bucket
       end
     end
   end
 
-  # Configures a bucket in the mock implementation. Use #allow_access_to on
+  # Configures a bucket in the mock implementation. Use #update_settings on
   # the Container object returned by this method to configure who may access
   # the bucket.
   def self.create_bucket(region, bucket_name)

data/lib/canistor/storage.rb

@@ -6,6 +6,8 @@ require 'nokogiri'
 require 'securerandom'
 
 require_relative "storage/bucket"
+require_relative "storage/delete_marker"
 require_relative "storage/object"
+require_relative "storage/objects"
 require_relative "storage/part"
 require_relative "storage/upload"

data/lib/canistor/storage/bucket.rb

@@ -6,6 +6,8 @@ require 'nokogiri'
 require 'singleton'
 require 'securerandom'
 
+require_relative "bucket/settings"
+
 module Canistor
   module Storage
     # Holds information about a bucket and implements interaction with it.
@@ -13,18 +15,24 @@ module Canistor
       attr_accessor :region
       attr_accessor :name
 
-      attr_reader :access_keys
+      attr_reader :settings
       attr_reader :objects
       attr_reader :uploads
 
       def initialize(**attributes)
-        @access_keys = Set.new
+        @settings = Settings.new
         clear
         attributes.each do |name, value|
           public_send("#{name}=", value)
         end
       end
 
+      # Update bucket settings, see Canistor::Storage::Bucket::Settings for
+      # supported configuration.
+      def update_settings(settings)
+        @settings.update(settings)
+      end
+
       def [](name)
         @objects[name]
       end
@@ -38,7 +46,7 @@ module Canistor
       end
 
       def head(context, access_key_id, subject)
-        if !access_keys.include?(access_key_id)
+        if !settings.access_keys.include?(access_key_id)
           Canistor::ErrorHandler.serve_access_denied(context, subject)
         elsif object = objects[subject.key]
           object.head(context, subject)
@@ -50,7 +58,7 @@ module Canistor
       def get(context, access_key_id, subject)
         params = CGI::parse(context.http_request.endpoint.query.to_s)
         catch(:rendered_error) do
-          if !access_keys.include?(access_key_id)
+          if !settings.access_keys.include?(access_key_id)
             Canistor::ErrorHandler.serve_access_denied(context, subject)
           elsif params.has_key?('uploads')
             list_bucket_uploads(context)
@@ -67,7 +75,7 @@ module Canistor
       end
 
       def put(context, access_key_id, subject)
-        if access_keys.include?(access_key_id)
+        if settings.access_keys.include?(access_key_id)
           Canistor.take_fail(:store) { return }
           params = CGI::parse(context.http_request.endpoint.query.to_s)
           catch(:rendered_error) do
@@ -85,7 +93,7 @@ module Canistor
       end
 
       def post(context, access_key_id, subject)
-        if access_keys.include?(access_key_id)
+        if settings.access_keys.include?(access_key_id)
           Canistor.take_fail(:store) { return }
           params = CGI::parse(context.http_request.endpoint.query.to_s)
           catch(:rendered_error) do
@@ -105,10 +113,10 @@ module Canistor
       end
 
       def delete(context, access_key_id, subject)
-        if !access_keys.include?(access_key_id)
+        if !settings.access_keys.include?(access_key_id)
           Canistor::ErrorHandler.serve_access_denied(context, subject)
-        elsif object = objects[subject.key]
-          @objects.delete(object.key)
+        elsif objects[subject.key]
+          object = @objects.delete(subject.key)
           object.delete(context, subject)
         else
           Canistor::ErrorHandler.serve_no_such_key(context, subject)
@@ -116,7 +124,9 @@ module Canistor
       end
 
       def clear
-        @objects = {}
+        @objects = Canistor::Storage::Objects.new(
+          versioned: settings.versioned?
+        )
         @uploads = {}
       end
 
@@ -133,22 +143,41 @@ module Canistor
         access_keys.merge(access_key_ids)
       end
 
-      def headers
-        {}
+      def store_replica(object)
+        replica = object.copy
+        replica.versioned = settings.versioned?
+        self[object.key] = replica
       end
 
       private
 
-      def build_upload(context, subject)
-        Canistor::Storage::Upload.new(
+      def build_object(subject)
+        Canistor::Storage::Object.new(
           region: subject.region,
           bucket: subject.bucket,
-          key: subject.key
+          key: subject.key,
+          versioned: settings.versioned?
         )
       end
 
-      def find_or_build_object(context, subject)
-        objects[subject.key] || Canistor::Storage::Object.new(
+      def put_object(context, subject)
+        object = build_object(subject)
+        object.versioned = settings.versioned?
+        object.put(context, subject)
+        self[subject.key] = object
+        replicate(object)
+      end
+
+      def replicate(object)
+        if settings.replicated?
+          settings.replicate_to_buckets.each do |bucket|
+            bucket.store_replica(object)
+          end
+        end
+      end
+
+      def build_upload(subject)
+        Canistor::Storage::Upload.new(
           region: subject.region,
           bucket: subject.bucket,
           key: subject.key
@@ -156,7 +185,7 @@ module Canistor
       end
 
       def post_upload(context, subject)
-        upload = build_upload(context, subject)
+        upload = build_upload(subject)
         @uploads[upload.id] = upload
         upload.put(context, subject)
       end
@@ -180,12 +209,6 @@ module Canistor
         end
       end
 
-      def put_object(context, subject)
-        object = find_or_build_object(context, subject)
-        self[subject.key] = object
-        object.put(context, subject)
-      end
-
       # Iterate over all objects in the bucket using the filter and pagination
       # options which exist in S3.
       def each(prefix:, marker:, max_keys:, &block)

data/lib/canistor/storage/bucket/settings.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Canistor
+  module Storage
+    class Bucket
+      # Stores settings for the bucket.
+      #
+      #   +allow_access_keys+: Allow full access to the bucket using these
+      #   access keys.
+      #   +replicate_to+: Replicate all updates to the bucket to this bucket.
+      #   +versioned+: Use versioning on the bucket.
+      class Settings
+        def initialize(settings = {})
+          clear
+          update(settings)
+        end
+
+        def update(settings)
+          case settings
+          # Old style setup allowed settings to be an array of allowed access
+          # keys.
+          when Set, Array
+            allow_access_keys(settings)
+          else
+            settings.each do |name, value|
+              public_send("#{name}", value)
+            end
+          end
+        end
+
+        def access_keys
+          @access_keys || []
+        end
+
+        def replicates_to
+          @replicates_to || []
+        end
+
+        def versioned?
+          !!@versioned
+        end
+
+        def allow_access_keys(access_keys)
+          @access_keys = access_keys
+        end
+
+        def replicated?
+          @replicates_to && !@replicates_to.empty?
+        end
+
+        def replicate_to(replicates_to)
+          @replicates_to = replicates_to
+        end
+
+        def replicate_to_buckets
+          return [] unless replicated?
+          @replicates_to.map do |location|
+            if bucket = Canistor.store.dig(*location.split(':'))
+              bucket
+            else
+              raise(
+                Canistor::ConfigurationError,
+                "Can't locate bucket `#{location}' when trying to replicate " \
+                "object. Please make sure the replication location is in the " \
+                "form of region:bucket and configured in Canistor."
+              )
+            end
+          end
+        end
+
+        def versioned(versioned)
+          @versioned = versioned
+        end
+
+        def clear
+          @access_keys = []
+          @replicates_to = []
+          @versioned = false
+        end
+      end
+    end
+  end
+end

data/lib/canistor/storage/delete_marker.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Canistor
+  module Storage
+    # Represents the non-existence of an object on a versioned bucket.
+    class DeleteMarker
+      attr_reader :version_id
+
+      def initialize
+        @version_id = generate_version_id
+      end
+
+      def headers
+        {
+          'x-amz-delete-marker' => 'true',
+          'x-amz-version-id' => version_id
+        }
+      end
+
+      def head(context, subject)
+        Canistor::ErrorHandler.serve_no_such_key(context, subject)
+      end
+
+      def get(context, subject)
+        Canistor::ErrorHandler.serve_no_such_key(context, subject)
+      end
+
+      def delete(context, subject)
+        context.http_response.signal_headers(200, headers)
+      end
+
+      private
+
+      def generate_version_id
+        Base64.strict_encode64(SecureRandom.hex(16)).gsub('=', '')
+      end
+    end
+  end
+end

data/lib/canistor/storage/object.rb

@@ -1,10 +1,38 @@
 # frozen_string_literal: true
 
-require "digest/sha1"
+require "digest/md5"
 
 module Canistor
   module Storage
     class Object
+      # Renders the result of a copy request.
+      class CopyObjectResult
+        def initialize(source_object, object)
+          @source_object, @object = source_object, object
+        end
+
+        def headers
+          @object.identity_headers.merge(
+            'x-amz-copy-source-version-id' => @source_object.version_id
+          )
+        end
+
+        def to_xml
+          Nokogiri::XML::Builder.new do |xml|
+            xml.CopyObjectResult do
+              xml.LastModified @object.last_modified
+              xml.ETag @object.etag
+            end
+          end.to_xml
+        end
+
+        def self.serve(context, subject, source_object, object)
+          result = new(source_object, object)
+          context.http_response.signal_headers(200, result.headers)
+          context.http_response.signal_data(result.to_xml)
+        end
+      end
+
       attr_accessor :region
       attr_accessor :bucket
       attr_accessor :key
@@ -13,6 +41,7 @@ module Canistor
       attr_reader :last_modified
 
       def initialize(**attributes)
+        @versioned = false
         @headers = {}
         @data = ''
         attributes.each do |name, value|
@@ -21,6 +50,20 @@ module Canistor
         @digest = nil
       end
 
+      def versioned=(versioned)
+        @versioned = versioned
+      end
+
+      def versioned?
+        !!@versioned
+      end
+
+      def version_id
+        if versioned?
+          @version_id ||= generate_version_id
+        end
+      end
+
       def size
         data&.size
       end
@@ -35,7 +78,7 @@ module Canistor
       end
 
       def digest
-        @digest ||= Digest::SHA1.hexdigest(data)
+        @digest ||= Digest::MD5.hexdigest(data)
       end
 
       def etag
@@ -65,11 +108,12 @@ module Canistor
 
       def identity_headers
         {
-          'x-amz-request-id' => Base64.strict_encode64(SecureRandom.hex(16)),
+          'etag' => etag,
           'x-amz-id' => digest[0, 16],
           'x-amz-id-2' => Base64.strict_encode64(digest),
-          'etag' => etag
-        }
+          'x-amz-request-id' => Base64.strict_encode64(SecureRandom.hex(16)),
+          'x-amz-version-id' => version_id
+        }.compact
       end
 
       def head(context, subject)
@@ -82,23 +126,38 @@ module Canistor
       end
 
       def put(context, subject)
-        catch(:rendered_error) do
-          source_object = source_object(context, subject)
-          write(
-            object_headers(context, source_object),
-            object_data(context, source_object)
-          )
-          context.http_response.signal_headers(200, identity_headers)
+        if source_object = find_source(context, subject)
+          apply_source_object(context, subject, source_object)
+        else
+          apply_request(context, subject)
         end
       end
 
+      def copy
+        object = self.class.new(region: region, bucket: bucket, key: key)
+        object.write(headers, data)
+        object
+      end
+
       def delete(context, subject)
         context.http_response.signal_headers(200, {})
       end
 
+      def copy_headers(context)
+        directive = context.http_request.headers['x-amz-metadata-directive']
+        case directive
+        when 'COPY'
+          identity_headers
+        when 'REPLACE', nil
+          context.http_request.headers
+        else
+          raise ArgumentError, "Unsupported metadata directive: `#{directive}'"
+        end
+      end
+
       private
 
-      def source_object(context, subject)
+      def find_source(context, subject)
         if source = context.http_request.headers['x-amz-copy-source']
           bucket_name, key = source.split('/', 2)
           if bucket = Canistor.store.dig(region, bucket_name)
@@ -115,24 +174,18 @@ module Canistor
         end
       end
 
-      def object_data(context, source_object)
-        if source_object
-          source_object.data
-        else
-          context.http_request.body
-        end
+      def apply_request(context, subject)
+        write(context.http_request.headers, context.http_request.body)
+        context.http_response.signal_headers(200, identity_headers)
       end
 
-      def object_headers(context, source_object)
-        directive = context.http_request.headers['x-amz-metadata-directive']
-        case directive
-        when 'COPY'
-          source_object.headers
-        when 'REPLACE', nil
-          context.http_request.headers
-        else
-          raise ArgumentError, "Unsupported metadata directive: `#{directive}'"
-        end
+      def apply_source_object(context, subject, source_object)
+        write(source_object.copy_headers(context), source_object.data)
+        CopyObjectResult.serve(context, subject, source_object, self)
+      end
+
+      def generate_version_id
+        Base64.strict_encode64(SecureRandom.hex(16)).gsub('=', '')
       end
 
       META_HEADERS = %w(
@@ -143,7 +196,7 @@ module Canistor
       def headers=(headers)
         return if headers.nil?
         headers.each do |name, value|
-          if META_HEADERS.include?(name)
+          if META_HEADERS.include?(name) || name.start_with?('x-amz-meta')
             @headers[name] = value
           end
         end

data/lib/canistor/storage/objects.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Canistor
+  module Storage
+    # Manages a lookup table of versioned objects.
+    class Objects
+      def initialize(versioned: true)
+        @versioned = versioned
+        @objects = {}
+      end
+
+      def versioned?
+        @versioned
+      end
+
+      def [](key)
+        if stack = @objects[key]
+          stack.last
+        end
+      end
+
+      def []=(key, object)
+        @objects[key] ||= []
+        if versioned?
+          @objects[key].push(object)
+        else
+          @objects[key] = [object]
+        end
+      end
+
+      def delete(key)
+        if versioned?
+          self[key] = Canistor::Storage::DeleteMarker.new
+        else
+          @objects.delete(key).last
+        end
+      end
+
+      def dig(*segments)
+        @objects.dig(*segments)&.last
+      end
+
+      def each
+        @objects.each do |key, stack|
+          yield [key, stack.last]
+        end
+      end
+    end
+  end
+end

data/lib/canistor/storage/part.rb

@@ -10,7 +10,6 @@ module Canistor
       attr_accessor :number
 
       attr_reader :data
-      attr_reader :etag
 
       def initialize(**attributes)
         attributes.each do |name, value|

data/lib/canistor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Canistor
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canistor
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Manfred Stienstra
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-24 00:00:00.000000000 Z
+date: 2018-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -98,7 +98,10 @@ files:
 - lib/canistor/plugin.rb
 - lib/canistor/storage.rb
 - lib/canistor/storage/bucket.rb
+- lib/canistor/storage/bucket/settings.rb
+- lib/canistor/storage/delete_marker.rb
 - lib/canistor/storage/object.rb
+- lib/canistor/storage/objects.rb
 - lib/canistor/storage/part.rb
 - lib/canistor/storage/upload.rb
 - lib/canistor/subject.rb
@@ -123,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Canistor is mock for Aws::S3 defined by the AWS SDK gem.