aws-sdk-s3 1.75.0

data/lib/aws-sdk-s3/event_streams.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# WARNING ABOUT GENERATED CODE
+#
+# This file is generated. See the contributing guide for more information:
+# https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md
+#
+# WARNING ABOUT GENERATED CODE
+
+module Aws::S3
+  module EventStreams
+    class SelectObjectContentEventStream
+
+      def initialize
+        @event_emitter = Aws::EventEmitter.new
+      end
+
+      def on_records_event(&block)
+        @event_emitter.on(:records, block) if block_given?
+      end
+
+      def on_stats_event(&block)
+        @event_emitter.on(:stats, block) if block_given?
+      end
+
+      def on_progress_event(&block)
+        @event_emitter.on(:progress, block) if block_given?
+      end
+
+      def on_cont_event(&block)
+        @event_emitter.on(:cont, block) if block_given?
+      end
+
+      def on_end_event(&block)
+        @event_emitter.on(:end, block) if block_given?
+      end
+
+      def on_error_event(&block)
+        @event_emitter.on(:error, block) if block_given?
+      end
+
+      def on_initial_response_event(&block)
+        @event_emitter.on(:initial_response, block) if block_given?
+      end
+
+      def on_unknown_event(&block)
+        @event_emitter.on(:unknown_event, block) if block_given?
+      end
+
+      def on_event(&block)
+        on_records_event(&block)
+        on_stats_event(&block)
+        on_progress_event(&block)
+        on_cont_event(&block)
+        on_end_event(&block)
+        on_error_event(&block)
+        on_initial_response_event(&block)
+        on_unknown_event(&block)
+      end
+
+      # @api private
+      # @return Aws::EventEmitter
+      attr_reader :event_emitter
+
+    end
+
+  end
+end
+

data/lib/aws-sdk-s3/file_downloader.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'thread'
+require 'set'
+require 'tmpdir'
+
+module Aws
+  module S3
+    # @api private
+    class FileDownloader
+
+      MIN_CHUNK_SIZE = 5 * 1024 * 1024
+      MAX_PARTS = 10_000
+      THREAD_COUNT = 10
+
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      def download(destination, options = {})
+        @path = destination
+        @mode = options[:mode] || 'auto'
+        @thread_count = options[:thread_count] || THREAD_COUNT
+        @chunk_size = options[:chunk_size]
+        @params = {
+          bucket: options[:bucket],
+          key: options[:key],
+        }
+        @params[:version_id] = options[:version_id] if options[:version_id]
+
+        case @mode
+        when 'auto' then multipart_download
+        when 'single_request' then single_request
+        when 'get_range'
+          if @chunk_size
+            resp = @client.head_object(@params)
+            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+          else
+            msg = 'In :get_range mode, :chunk_size must be provided'
+            raise ArgumentError, msg
+          end
+        else
+          msg = "Invalid mode #{@mode} provided, "\
+                'mode should be :single_request, :get_range or :auto'
+          raise ArgumentError, msg
+        end
+      end
+
+      private
+
+      def multipart_download
+        resp = @client.head_object(@params.merge(part_number: 1))
+        count = resp.parts_count
+        if count.nil? || count <= 1
+          resp.content_length < MIN_CHUNK_SIZE ?
+            single_request :
+            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+        else
+          # partNumber is an option
+          resp = @client.head_object(@params)
+          resp.content_length < MIN_CHUNK_SIZE ?
+            single_request :
+            compute_mode(resp.content_length, count)
+        end
+      end
+
+      def compute_mode(file_size, count)
+        chunk_size = compute_chunk(file_size)
+        part_size = (file_size.to_f / count.to_f).ceil
+        if chunk_size < part_size
+          multithreaded_get_by_ranges(construct_chunks(file_size))
+        else
+          multithreaded_get_by_parts(count)
+        end
+      end
+
+      def construct_chunks(file_size)
+        offset = 0
+        default_chunk_size = compute_chunk(file_size)
+        chunks = []
+        while offset <= file_size
+          progress = offset + default_chunk_size
+          chunks << "bytes=#{offset}-#{progress < file_size ? progress : file_size}"
+          offset = progress + 1
+        end
+        chunks
+      end
+
+      def compute_chunk(file_size)
+        if @chunk_size && @chunk_size > file_size
+          raise ArgumentError, ":chunk_size shouldn't exceed total file size."
+        else
+          @chunk_size || [(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE].max.to_i
+        end
+      end
+
+      def batches(chunks, mode)
+        chunks = (1..chunks) if mode.eql? 'part_number'
+        chunks.each_slice(@thread_count).to_a
+      end
+
+      def multithreaded_get_by_ranges(chunks)
+        thread_batches(chunks, 'range')
+      end
+
+      def multithreaded_get_by_parts(parts)
+        thread_batches(parts, 'part_number')
+      end
+
+      def thread_batches(chunks, param)
+        batches(chunks, param).each do |batch|
+          threads = []
+          batch.each do |chunk|
+            threads << Thread.new do
+              resp = @client.get_object(
+                @params.merge(param.to_sym => chunk)
+              )
+              write(resp)
+            end
+          end
+          threads.each(&:join)
+        end
+      end
+
+      def write(resp)
+        range, _ = resp.content_range.split(' ').last.split('/')
+        head, _ = range.split('-').map {|s| s.to_i}
+        IO.write(@path, resp.body.read, head)
+      end
+
+      def single_request
+        @client.get_object(
+          @params.merge(response_target: @path)
+        )
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/file_part.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+
+    # A utility class that provides an IO-like interface to a portion of a file
+    # on disk.
+    # @api private
+    class FilePart
+
+      # @option options [required, String, Pathname, File, Tempfile] :source
+      #   The file to upload.
+      #
+      # @option options [required, Integer] :offset The file part will read
+      #   starting at this byte offset.
+      #
+      # @option options [required, Integer] :size The maximum number of bytes to
+      #   read from the `:offset`.
+      def initialize(options = {})
+        @source = options[:source]
+        @first_byte = options[:offset]
+        @last_byte = @first_byte + options[:size]
+        @size = options[:size]
+        @file = nil
+      end
+
+      # @return [String, Pathname, File, Tempfile]
+      attr_reader :source
+
+      # @return [Integer]
+      attr_reader :first_byte
+
+      # @return [Integer]
+      attr_reader :last_byte
+
+      # @return [Integer]
+      attr_reader :size
+
+      def read(bytes = nil, output_buffer = nil)
+        open_file unless @file
+        read_from_file(bytes, output_buffer)
+      end
+
+      def rewind
+        if @file
+          @file.seek(@first_byte)
+          @position = @first_byte
+        end
+        0
+      end
+
+      def close
+        @file.close if @file
+      end
+
+      private
+
+      def open_file
+        @file = File.open(@source, 'rb')
+        rewind
+      end
+
+      def read_from_file(bytes, output_buffer)
+        length = [remaining_bytes, *bytes].min
+        data   = @file.read(length, output_buffer)
+
+        @position += data ? data.bytesize : 0
+
+        data.to_s unless bytes && (data.nil? || data.empty?)
+      end
+
+      def remaining_bytes
+        @last_byte - @position
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-s3/file_uploader.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Aws
+  module S3
+    # @api private
+    class FileUploader
+
+      FIFTEEN_MEGABYTES = 15 * 1024 * 1024
+
+      # @param [Hash] options
+      # @option options [Client] :client
+      # @option options [Integer] :multipart_threshold (15728640)
+      def initialize(options = {})
+        @options = options
+        @client = options[:client] || Client.new
+        @multipart_threshold = options[:multipart_threshold] ||
+                               FIFTEEN_MEGABYTES
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @return [Integer] Files larger than this in bytes are uploaded
+      #   using a {MultipartFileUploader}.
+      attr_reader :multipart_threshold
+
+      # @param [String, Pathname, File, Tempfile] source The file to upload.
+      # @option options [required, String] :bucket The bucket to upload to.
+      # @option options [required, String] :key The key for the object.
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
+      # @return [void]
+      def upload(source, options = {})
+        if File.size(source) >= multipart_threshold
+          MultipartFileUploader.new(@options).upload(source, options)
+        else
+          put_object(source, options)
+        end
+      end
+
+      private
+
+      def open_file(source)
+        if String === source || Pathname === source
+          File.open(source, 'rb') { |file| yield(file) }
+        else
+          yield(source)
+        end
+      end
+
+      def put_object(source, options)
+        if (callback = options.delete(:progress_callback))
+          options[:on_chunk_sent] = single_part_progress(callback)
+        end
+        open_file(source) do |file|
+          @client.put_object(options.merge(body: file))
+        end
+      end
+
+      def single_part_progress(progress_callback)
+        proc do |_chunk, bytes_read, total_size|
+          progress_callback.call([bytes_read], [total_size])
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/legacy_signer.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'time'
+require 'openssl'
+require 'cgi'
+require 'webrick/httputils'
+require 'aws-sdk-core/query'
+
+module Aws
+  module S3
+    # @api private
+    class LegacySigner
+
+      SIGNED_QUERYSTRING_PARAMS = Set.new(%w(
+
+        acl delete cors lifecycle location logging notification partNumber
+        policy requestPayment restore tagging torrent uploadId uploads
+        versionId versioning versions website replication requestPayment
+        accelerate
+
+        response-content-type response-content-language
+        response-expires response-cache-control
+        response-content-disposition response-content-encoding
+
+      ))
+
+      def self.sign(context)
+        new(
+          context.config.credentials,
+          context.params,
+          context.config.force_path_style
+        ).sign(context.http_request)
+      end
+
+      # @param [CredentialProvider] credentials
+      def initialize(credentials, params, force_path_style)
+        @credentials = credentials.credentials
+        @params = Query::ParamList.new
+        params.each_pair do |param_name, param_value|
+          @params.set(param_name, param_value)
+        end
+        @force_path_style = force_path_style
+      end
+
+      attr_reader :credentials, :params
+
+      def sign(request)
+        if token = credentials.session_token
+          request.headers["X-Amz-Security-Token"] = token
+        end
+        request.headers['Authorization'] = authorization(request)
+      end
+
+      def authorization(request)
+        "AWS #{credentials.access_key_id}:#{signature(request)}"
+      end
+
+      def signature(request)
+        string_to_sign = string_to_sign(request)
+        signature = digest(credentials.secret_access_key, string_to_sign)
+        uri_escape(signature)
+      end
+
+      def digest(secret, string_to_sign)
+        Base64.encode64(hmac(secret, string_to_sign)).strip
+      end
+
+      def hmac(key, value)
+        OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value)
+      end
+
+      # From the S3 developer guide:
+      #
+      #     StringToSign =
+      #       HTTP-Verb ` "\n" `
+      #       content-md5 ` "\n" `
+      #       content-type ` "\n" `
+      #       date ` "\n" `
+      #       CanonicalizedAmzHeaders + CanonicalizedResource;
+      #
+      def string_to_sign(request)
+        [
+          request.http_method,
+          request.headers.values_at('Content-Md5', 'Content-Type').join("\n"),
+          signing_string_date(request),
+          canonicalized_headers(request),
+          canonicalized_resource(request.endpoint),
+        ].flatten.compact.join("\n")
+      end
+
+      def signing_string_date(request)
+        # if a date is provided via x-amz-date then we should omit the
+        # Date header from the signing string (should appear as a blank line)
+        if request.headers.detect{|k,v| k.to_s =~ /^x-amz-date$/i }
+          ''
+        else
+          request.headers['Date'] = Time.now.httpdate
+        end
+      end
+
+      # CanonicalizedAmzHeaders
+      #
+      # See the developer guide for more information on how this element
+      # is generated.
+      #
+      def canonicalized_headers(request)
+        x_amz = request.headers.select{|k, v| k =~ /^x-amz-/i }
+        x_amz = x_amz.collect{|k, v| [k.downcase, v] }
+        x_amz = x_amz.sort_by{|k, v| k }
+        x_amz = x_amz.collect{|k, v| "#{k}:#{v.to_s.strip}" }.join("\n")
+        x_amz == '' ? nil : x_amz
+      end
+
+      # From the S3 developer guide
+      #
+      #     CanonicalizedResource =
+      #       [ "/" ` Bucket ] `
+      #       <HTTP-Request-URI, protocol name up to the querystring> +
+      #       [ sub-resource, if present. e.g. "?acl", "?location",
+      #       "?logging", or "?torrent"];
+      #
+      # @api private
+      def canonicalized_resource(endpoint)
+
+        parts = []
+
+        # virtual hosted-style requests require the hostname to appear
+        # in the canonicalized resource prefixed by a forward slash.
+        if bucket = params[:bucket]
+          bucket = bucket.value
+          ssl = endpoint.scheme == 'https'
+          if Plugins::BucketDns.dns_compatible?(bucket, ssl) && !@force_path_style
+            parts << "/#{bucket}"
+          end
+        end
+
+        # append the path name (no querystring)
+        parts << endpoint.path
+
+        # lastly any sub resource querystring params need to be appened
+        # in lexigraphical ordered joined by '&' and prefixed by '?'
+        params = signed_querystring_params(endpoint)
+
+        unless params.empty?
+          parts << '?'
+          parts << params.sort.collect{|p| p.to_s }.join('&')
+        end
+
+        parts.join
+      end
+
+      def signed_querystring_params(endpoint)
+        endpoint.query.to_s.split('&').select do |p|
+          SIGNED_QUERYSTRING_PARAMS.include?(p.split('=')[0])
+        end.map { |p| CGI.unescape(p) }
+      end
+
+      def uri_escape(s)
+
+        #URI.escape(s)
+
+        # URI.escape is deprecated, replacing it with escape from webrick
+        # to squelch the massive number of warnings generated from Ruby.
+        # The following script was used to determine the differences
+        # between the various escape methods available. The webrick
+        # escape only had two differences and it is available in the
+        # standard lib.
+        #
+        #     (0..255).each {|c|
+        #       s = [c].pack("C")
+        #       e = [
+        #         CGI.escape(s),
+        #         ERB::Util.url_encode(s),
+        #         URI.encode_www_form_component(s),
+        #         WEBrick::HTTPUtils.escape_form(s),
+        #         WEBrick::HTTPUtils.escape(s),
+        #         URI.escape(s),
+        #       ]
+        #       next if e.uniq.length == 1
+        #       puts("%5s %5s %5s %5s %5s %5s %5s" % ([s.inspect] + e))
+        #     }
+        #
+        WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']')
+      end
+
+    end
+  end
+end