canistor 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2621d73e017b7bbfb296384b7bceb87699d7f71942fa9b85c346d824103f8fc0
-  data.tar.gz: ff98d15d36834943e177a66c4db1c7fee64f96b4cb7398ad4f4c4a179bbfc2c9
+  metadata.gz: af06960c106c4d6a66f2e157cf5392df11bd27182c9014f934658b1981f095c8
+  data.tar.gz: 9c8d3c968c879d31bf6b471d599cc132b1843ed7cf7c8282cc0b21a27b04b351
 SHA512:
-  metadata.gz: 1b92e5cbae6bb7d863b3e437df73cd0a23fda1ce3b6533a00499f26ad4eef07c2bd305d263ff68930efc5ab77aeb763c91ba561c0a8dc8aadd608488779e7bf7
-  data.tar.gz: 3460dd2aad6248bddd71cd1cd8595da3b80a871a5568edd47b8455307e552e9e2b3228e5cb4cd45f8e4d28b41b07c68519f155704dd7748463d670101dca4d24
+  metadata.gz: 2275495c50f747fe474ff02c745976dcf1f05a87a8073002ef53fae3b71a3256aff2ebb8d2fc2f3ed18167c064ed6557271839c83d9dd68600b0e1dff306a6c3
+  data.tar.gz: 4fd8d042c1acb1bbcfbda6be5a8a865d24bb0d1160d971a09cb6742b4c4175139e9b4ca442a7c6206d1e7249749f1b2b1ed3c1b7f5b135a92889351ae9e3c87e

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) Manfred Stienstra <manfred@fngtps.com>
+Copyright (c) Procore <info@procore.com>
 
 MIT License
 

data/lib/canistor/error_handler.rb

@@ -68,6 +68,20 @@ module Canistor
       end.to_xml)
     end
 
+    def serve_no_such_upload(subject)
+      serve_error(404, Nokogiri::XML::Builder.new do |xml|
+        xml.Error do
+          xml.Code 'NoSuchUpload'
+          xml.Message 'The specified upload does not exist. The upload ID ' \
+                      'may be invalid, or the upload may have been aborted ' \
+                      'or completed.'
+          xml.Key subject.key
+          xml.RequestId request_id
+          xml.HostId host_id
+        end
+      end.to_xml)
+    end
+
     def serve_no_such_key(subject)
       serve_error(404, Nokogiri::XML::Builder.new do |xml|
         xml.Error do
@@ -80,6 +94,35 @@ module Canistor
       end.to_xml)
     end
 
+    def serve_invalid_part(upload_id, part_number, part_etag)
+      serve_error(400, Nokogiri::XML::Builder.new do |xml|
+        xml.Error do
+          xml.Code 'InvalidPart'
+          xml.Message 'One or more of the specified parts could not be found. '\
+                      'The part may not have been uploaded, or the specified ' \
+                      ' entity tag may not match the part\'s entity tag.'
+          xml.UploadId upload_id
+          xml.PartNumber part_number
+          xml.ETag part_etag
+          xml.RequestId request_id
+          xml.HostId host_id
+        end
+      end.to_xml)
+    end
+
+    def serve_invalid_part_order(upload_id)
+      serve_error(400, Nokogiri::XML::Builder.new do |xml|
+        xml.Error do
+          xml.Code 'InvalidPartOrder'
+          xml.Message 'The list of parts was not in ascending order. Parts ' \
+                      'must be ordered by part number.'
+          xml.UploadId upload_id
+          xml.RequestId request_id
+          xml.HostId host_id
+        end
+      end.to_xml)
+    end
+
     def serve_access_denied(subject)
       serve_error(403, Nokogiri::XML::Builder.new do |xml|
         xml.Error do
@@ -139,10 +182,22 @@ module Canistor
       new(context).serve_no_such_bucket(subject)
     end
 
+    def self.serve_no_such_upload(context, subject)
+      new(context).serve_no_such_upload(subject)
+    end
+
     def self.serve_no_such_key(context, subject)
       new(context).serve_no_such_key(subject)
     end
 
+    def self.serve_invalid_part(context, upload_id, part_number, part_etag)
+      new(context).serve_invalid_part(upload_id, part_number, part_etag)
+    end
+
+    def self.serve_invalid_part_order(context, upload_id)
+      new(context).serve_invalid_part_order(upload_id)
+    end
+
     def self.access_denied(context, subject)
       new(context).access_denied(subject)
     end

data/lib/canistor/storage.rb

@@ -7,4 +7,5 @@ require 'securerandom'
 
 require_relative "storage/bucket"
 require_relative "storage/object"
+require_relative "storage/part"
 require_relative "storage/upload"

data/lib/canistor/storage/bucket.rb

@@ -8,6 +8,7 @@ require 'securerandom'
 
 module Canistor
   module Storage
+    # Holds information about a bucket and implements interaction with it.
     class Bucket
       attr_accessor :region
       attr_accessor :name
@@ -47,23 +48,37 @@ module Canistor
       end
 
       def get(context, access_key_id, subject)
-        if !access_keys.include?(access_key_id)
-          Canistor::ErrorHandler.serve_access_denied(context, subject)
-        elsif subject.key.nil? || subject.key == ''
-          list_bucket(context)
-        elsif object = objects[subject.key]
-          object.get(context, subject)
-        else
-          Canistor::ErrorHandler.serve_no_such_key(context, subject)
+        params = CGI::parse(context.http_request.endpoint.query.to_s)
+        catch(:rendered_error) do
+          if !access_keys.include?(access_key_id)
+            Canistor::ErrorHandler.serve_access_denied(context, subject)
+          elsif params.has_key?('uploads')
+            list_bucket_uploads(context)
+          elsif params.has_key?('uploadId')
+            list_bucket_upload_parts(context, subject, params)
+          elsif subject.key.nil? || subject.key == ''
+            list_bucket(context)
+          elsif object = objects[subject.key]
+            object.get(context, subject)
+          else
+            Canistor::ErrorHandler.serve_no_such_key(context, subject)
+          end
         end
       end
 
       def put(context, access_key_id, subject)
         if access_keys.include?(access_key_id)
           Canistor.take_fail(:store) { return }
-          object = find_or_build_object(subject, context)
-          self[subject.key] = object
-          object.put(context, subject)
+          params = CGI::parse(context.http_request.endpoint.query.to_s)
+          catch(:rendered_error) do
+            if params.has_key?('uploadId')
+              # Client wants to create a new upload part when uploadId is
+              # present in the query.
+              put_upload_part(context, subject, params)
+            else
+              put_object(context, subject)
+            end
+          end
         else
           Canistor::ErrorHandler.serve_access_denied(context, subject)
         end
@@ -72,9 +87,18 @@ module Canistor
       def post(context, access_key_id, subject)
         if access_keys.include?(access_key_id)
           Canistor.take_fail(:store) { return }
-          upload = build_upload(subject, context)
-          self[upload.id] = upload
-          upload.get(context, subject)
+          params = CGI::parse(context.http_request.endpoint.query.to_s)
+          catch(:rendered_error) do
+            if params.has_key?('uploads')
+              # Client wants to create a new upload when uploads is present in
+              # the query.
+              post_upload(context, subject)
+            elsif params.has_key?('uploadId')
+              # Client wants to complete the upload when uploadId is present in
+              # the query.
+              complete_upload(context, subject, params)
+            end
+          end
         else
           Canistor::ErrorHandler.serve_access_denied(context, subject)
         end
@@ -115,7 +139,7 @@ module Canistor
 
       private
 
-      def build_upload(subject, context)
+      def build_upload(context, subject)
         Canistor::Storage::Upload.new(
           region: subject.region,
           bucket: subject.bucket,
@@ -123,7 +147,7 @@ module Canistor
         )
       end
 
-      def find_or_build_object(subject, context)
+      def find_or_build_object(context, subject)
         objects[subject.key] || Canistor::Storage::Object.new(
           region: subject.region,
           bucket: subject.bucket,
@@ -131,6 +155,37 @@ module Canistor
         )
       end
 
+      def post_upload(context, subject)
+        upload = build_upload(context, subject)
+        @uploads[upload.id] = upload
+        upload.put(context, subject)
+      end
+
+      def put_upload_part(context, subject, params)
+        if upload = uploads.dig(params['uploadId'][0])
+          upload.put(context, subject)
+        else
+          Canistor::ErrorHandler.serve_no_such_upload(context, subject)
+          throw :rendered_error
+        end
+      end
+
+      def complete_upload(context, subject, params)
+        if upload = uploads.dig(params['uploadId'][0])
+          object = upload.post(context, subject)
+          self[subject.key] = object
+        else
+          Canistor::ErrorHandler.serve_no_such_upload(context, subject)
+          throw :rendered_error
+        end
+      end
+
+      def put_object(context, subject)
+        object = find_or_build_object(context, subject)
+        self[subject.key] = object
+        object.put(context, subject)
+      end
+
       # Iterate over all objects in the bucket using the filter and pagination
       # options which exist in S3.
       def each(prefix:, marker:, max_keys:, &block)
@@ -146,6 +201,28 @@ module Canistor
         end
       end
 
+      def upload_matches?(upload, upload_id_marker:, key_marker:)
+        if upload_id_marker && !upload.id.start_with?(upload_id_marker)
+          return false
+        end
+        if key_marker && !upload.key.start_with?(key_marker)
+          return false
+        end
+        true
+      end
+
+      def each_upload(upload_id_marker:, key_marker:, &block)
+        uploads.each do |upload_id, upload|
+          if upload_matches?(
+            upload,
+            upload_id_marker: upload_id_marker,
+            key_marker: key_marker
+          )
+            yield upload
+          end
+        end
+      end
+
       def list_bucket(context)
         context.http_response.signal_headers(
           200,
@@ -157,8 +234,27 @@ module Canistor
         end
       end
 
+      def list_bucket_uploads(context)
+        context.http_response.signal_headers(
+          200,
+          'date' => Time.now.httpdate,
+          'x-amz-request-id' => SecureRandom.hex(8).upcase
+        )
+        context.http_response.signal_data(to_uploads_xml(context))
+      end
+
+      def list_bucket_upload_parts(context, subject, params)
+        upload = uploads.dig(params['uploadId'][0])
+        if upload && upload.key == subject.key
+          upload.get(context)
+        else
+          Canistor::ErrorHandler.serve_no_such_upload(context, subject)
+          throw :rendered_error
+        end
+      end
+
       def to_xml(context)
-        # Only returns objects with keys that start with the prefix.
+        # Only return objects with keys that start with the prefix.
         prefix = context.params[:prefix]
         # Return objects until we find a key that matches the marker. Can
         # be used to group objects.
@@ -182,6 +278,31 @@ module Canistor
           end
         end.to_xml
       end
+
+      def to_uploads_xml(context)
+        # Only return uploads with ID's that start with id marker
+        upload_id_marker = context.params[:upload_id_marker]
+        # Only return uploads with keys that start with key marker
+        key_marker = context.params[:key_marker]
+        Nokogiri::XML::Builder.new do |xml|
+          xml.ListMultipartUploadsResult(
+            xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
+          ) do
+            xml.Bucket name
+            xml.KeyMarker key_marker
+            xml.UploadIdMarker upload_id_marker
+            each_upload(
+              upload_id_marker: upload_id_marker,
+              key_marker: key_marker
+            ) do |upload|
+              xml.Upload do
+                xml.Key upload.key
+                xml.UploadId upload.id
+              end
+            end
+          end
+        end.to_xml
+      end
     end
   end
 end

data/lib/canistor/storage/object.rb

@@ -42,6 +42,18 @@ module Canistor
         '"' + digest + '"'
       end
 
+      def endpoint
+        Aws::Partitions::EndpointProvider.resolve(region, 's3')
+      end
+
+      def location
+        [
+          endpoint,
+          bucket,
+          key,
+        ].join('/')
+      end
+
       def headers
         @headers.merge(identity_headers).merge(
           'date' => Time.now.httpdate,
@@ -72,8 +84,10 @@ module Canistor
       def put(context, subject)
         catch(:rendered_error) do
           source_object = source_object(context, subject)
-          self.data = object_data(context, source_object)
-          self.headers = object_headers(context, source_object)
+          write(
+            object_headers(context, source_object),
+            object_data(context, source_object)
+          )
           context.http_response.signal_headers(200, identity_headers)
         end
       end

data/lib/canistor/storage/part.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Canistor
+  module Storage
+    class Part
+      attr_accessor :region
+      attr_accessor :bucket
+      attr_accessor :key
+      attr_accessor :upload_id
+      attr_accessor :number
+
+      attr_reader :data
+      attr_reader :etag
+
+      def initialize(**attributes)
+        attributes.each do |name, value|
+          send("#{name}=", value)
+        end
+      end
+
+      def put(context, subject)
+        write(context.http_request.body.read)
+        context.http_response.signal_headers(200, identity_headers)
+      end
+
+      def etag
+        '"' + digest + '"'
+      end
+
+      def size
+        data&.size
+      end
+
+      def attributes
+        {
+          region: region,
+          bucket: bucket,
+          key: key,
+          upload_id: upload_id,
+          number: number,
+          digest: digest,
+          etag: etag
+        }
+      end
+
+    private
+
+      attr_writer :data
+
+      def write(data)
+        self.data = data
+      end
+
+      def digest
+        @digest ||= Digest::SHA1.hexdigest(data)
+      end
+
+      def identity_headers
+        {
+          'x-amz-request-id' => Base64.strict_encode64(SecureRandom.hex(16)),
+          'x-amz-id' => digest[0, 16],
+          'x-amz-id-2' => Base64.strict_encode64(digest),
+          'etag' => etag
+        }
+      end
+    end
+  end
+end

data/lib/canistor/storage/upload.rb

@@ -7,32 +7,157 @@ module Canistor
       attr_accessor :region
       attr_accessor :bucket
       attr_accessor :key
+      attr_accessor :parts
+
+      attr_reader :headers
 
       def initialize(**attributes)
         @id = SecureRandom.hex(64)
+        @parts = {}
+        @headers = {}
         attributes.each do |name, value|
           send("#{name}=", value)
         end
       end
 
-      def get(context, subject)
-        show_upload(context)
+      def write(headers)
+        self.headers = headers
+      end
+
+      def get(context)
+        list_parts(context)
+      end
+
+      def put(context, subject)
+        params = CGI::parse(context.http_request.endpoint.query.to_s)
+        if params.has_key?('uploads')
+          write(context.http_request.headers)
+          show_initiate_upload(context)
+        elsif params.has_key?('partNumber')
+          part_number = Integer(params['partNumber'][0])
+          part = find_or_build_part(subject, context, part_number)
+          parts[part_number] = part
+          part.put(context, subject)
+        else
+          raise(
+            RuntimeError,
+            "Implementation should never attempt to PUT an upload when there " \
+            "is no uploads or partNumber parameter present in the request."
+          )
+        end
+      end
+
+      def post(context, subject)
+        catch(:rendered_error) do
+          object = Canistor::Storage::Object.new(
+            region: region,
+            bucket: bucket,
+            key: key
+          )
+          object.write(headers, String.new)
+          each_part_in_body(context) do |part|
+            object.data << part.data
+          end
+          show_complete_upload(context, object)
+          object
+        end
       end
 
       private
 
-      def show_upload(context)
+      META_HEADERS = %w(
+        content-disposition
+        content-type
+      )
+
+      def headers=(headers)
+        return if headers.nil?
+        headers.each do |name, value|
+          if META_HEADERS.include?(name)
+            @headers[name] = value
+          end
+        end
+      end
+
+      def find_or_build_part(subject, context, part_number)
+        parts[part_number] || Canistor::Storage::Part.new(
+          region: subject.region,
+          bucket: subject.bucket,
+          key: subject.key,
+          upload_id: id,
+          number: part_number
+        )
+      end
+
+      def show_initiate_upload(context)
         context.http_response.signal_headers(
           200,
           'date' => Time.now.httpdate,
           'x-amz-request-id' => SecureRandom.hex(8).upcase
         )
         unless context.http_request.http_method == 'HEAD'
-          context.http_response.signal_data(to_xml(context))
+          context.http_response.signal_data(to_initiate_xml(context))
+        end
+      end
+
+      def show_complete_upload(context, etag)
+        context.http_response.signal_headers(
+          200,
+          'date' => Time.now.httpdate,
+          'x-amz-request-id' => SecureRandom.hex(8).upcase
+        )
+        context.http_response.signal_data(to_complete_xml(context, etag))
+      end
+
+      def each_part_in_body(context)
+        document = Nokogiri::XML.parse(context.http_request.body.read)
+        number = 0
+        document.css('Part').each do |element|
+          part_number = Integer(element.css('PartNumber').text)
+          if part_number > number
+            part_etag = element.css('ETag').text
+            found = find_part(part_number, part_etag)
+            if found
+              yield found
+            else
+              Canistor::ErrorHandler.serve_invalid_part(
+                context, id, part_number, part_etag
+              )
+              throw :rendered_error
+            end
+            number = part_number
+          else
+            Canistor::ErrorHandler.serve_invalid_part_order(context, id)
+            throw :rendered_error
+          end
+        end
+      end
+
+      def find_part(part_number, part_etag)
+        if part = parts[part_number]
+          if part.etag == part_etag
+            return part
+          end
+        end
+        nil
+      end
+
+      def each_part(&block)
+        parts.keys.sort.each do |number|
+          yield parts[number]
         end
       end
 
-      def to_xml(context)
+      def list_parts(context)
+        context.http_response.signal_headers(
+          200,
+          'date' => Time.now.httpdate,
+          'x-amz-request-id' => SecureRandom.hex(8).upcase
+        )
+        context.http_response.signal_data(to_upload_parts_xml(context))
+      end
+
+      def to_initiate_xml(context)
         Nokogiri::XML::Builder.new do |xml|
           xml.InitiateMultipartUploadResult(
             xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
@@ -43,6 +168,38 @@ module Canistor
           end
         end.to_xml
       end
+
+      def to_complete_xml(context, object)
+        Nokogiri::XML::Builder.new do |xml|
+          xml.CompleteMultipartUploadResult(
+            xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
+          ) do
+            xml.Location object.location
+            xml.Bucket object.bucket
+            xml.Key object.key
+            xml.ETag object.etag
+          end
+        end.to_xml
+      end
+
+      def to_upload_parts_xml(context)
+        Nokogiri::XML::Builder.new do |xml|
+          xml.ListPartsResult(
+            xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
+          ) do
+            xml.Bucket bucket
+            xml.Key key
+            xml.UploadId id
+            each_part do |part|
+              xml.Part do
+                xml.PartNumber part.number
+                xml.ETag part.etag
+                xml.Size part.size
+              end
+            end
+          end
+        end.to_xml
+      end
     end
   end
 end

data/lib/canistor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Canistor
-  VERSION = '0.3.2'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canistor
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Manfred Stienstra
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-26 00:00:00.000000000 Z
+date: 2018-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -99,10 +99,11 @@ files:
 - lib/canistor/storage.rb
 - lib/canistor/storage/bucket.rb
 - lib/canistor/storage/object.rb
+- lib/canistor/storage/part.rb
 - lib/canistor/storage/upload.rb
 - lib/canistor/subject.rb
 - lib/canistor/version.rb
-homepage: https://erm.im/canistor
+homepage: https://github.com/procore/canistor
 licenses:
 - MIT
 metadata: {}