aws-sdk-core 2.11.212 → 3.170.0

data/lib/aws-sdk-core/plugins/checksum_algorithm.rb

@@ -0,0 +1,340 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    # @api private
+    class ChecksumAlgorithm < Seahorse::Client::Plugin
+      CHUNK_SIZE = 1 * 1024 * 1024 # one MB
+
+      # determine the set of supported client side checksum algorithms
+      # CRC32c requires aws-crt (optional sdk dependency) for support
+      CLIENT_ALGORITHMS = begin
+        supported = %w[SHA256 SHA1 CRC32]
+        begin
+          require 'aws-crt'
+          supported << 'CRC32C'
+        rescue LoadError
+        end
+        supported
+      end.freeze
+
+      # priority order of checksum algorithms to validate responses against
+      # Remove any algorithms not supported by client (ie, depending on CRT availability)
+      CHECKSUM_ALGORITHM_PRIORITIES = %w[CRC32C SHA1 CRC32 SHA256] & CLIENT_ALGORITHMS
+
+      # byte size of checksums, used in computing the trailer length
+      CHECKSUM_SIZE = {
+        'CRC32' => 16,
+        'CRC32C' => 16,
+        'SHA1' => 36,
+        'SHA256' => 52
+      }
+
+      # Interface for computing digests on request/response bodies
+      # which may be files, strings or IO like objects
+      # Applies only to digest functions that produce 32 bit integer checksums
+      # (eg CRC32)
+      class Digest32
+
+        attr_reader :value
+
+        # @param [Object] digest_fn
+        def initialize(digest_fn)
+          @digest_fn = digest_fn
+          @value = 0
+        end
+
+        def update(chunk)
+          @value = @digest_fn.call(chunk, @value)
+        end
+
+        def base64digest
+          Base64.encode64([@value].pack('N')).chomp
+        end
+      end
+
+      def add_handlers(handlers, _config)
+        handlers.add(OptionHandler, step: :initialize)
+        # priority set low to ensure checksum is computed AFTER the request is
+        # built but before it is signed
+        handlers.add(ChecksumHandler, priority: 15, step: :build)
+      end
+
+      private
+
+      def self.request_algorithm_selection(context)
+        return unless context.operation.http_checksum
+
+        input_member = context.operation.http_checksum['requestAlgorithmMember']
+        context.params[input_member.to_sym]&.upcase if input_member
+      end
+
+      def self.request_validation_mode(context)
+        return unless context.operation.http_checksum
+
+        input_member = context.operation.http_checksum['requestValidationModeMember']
+        context.params[input_member.to_sym] if input_member
+      end
+
+      def self.operation_response_algorithms(context)
+        return unless context.operation.http_checksum
+
+        context.operation.http_checksum['responseAlgorithms']
+      end
+
+
+      # @api private
+      class OptionHandler < Seahorse::Client::Handler
+        def call(context)
+          context[:http_checksum] ||= {}
+
+          # validate request configuration
+          if (request_input = ChecksumAlgorithm.request_algorithm_selection(context))
+            unless CLIENT_ALGORITHMS.include? request_input
+              if (request_input == 'CRC32C')
+                raise ArgumentError, "CRC32C requires crt support - install the aws-crt gem for support."
+              else
+                raise ArgumentError, "#{request_input} is not a supported checksum algorithm."
+              end
+            end
+          end
+
+        # validate response configuration
+          if (ChecksumAlgorithm.request_validation_mode(context))
+            # Compute an ordered list as the union between priority supported and the
+            # operation's modeled response algorithms.
+            validation_list = CHECKSUM_ALGORITHM_PRIORITIES &
+              ChecksumAlgorithm.operation_response_algorithms(context)
+            context[:http_checksum][:validation_list] = validation_list
+          end
+
+          @handler.call(context)
+        end
+      end
+
+      # @api private
+      class ChecksumHandler < Seahorse::Client::Handler
+
+        def call(context)
+          if should_calculate_request_checksum?(context)
+            request_algorithm_input = ChecksumAlgorithm.request_algorithm_selection(context)
+            context[:checksum_algorithms] = request_algorithm_input
+
+            request_checksum_property = {
+              'algorithm' => request_algorithm_input,
+              'in' => checksum_request_in(context),
+              'name' => "x-amz-checksum-#{request_algorithm_input.downcase}"
+            }
+
+            calculate_request_checksum(context, request_checksum_property)
+          end
+
+          if should_verify_response_checksum?(context)
+            add_verify_response_checksum_handlers(context)
+          end
+
+          @handler.call(context)
+        end
+
+        private
+
+        def should_calculate_request_checksum?(context)
+          context.operation.http_checksum &&
+            ChecksumAlgorithm.request_algorithm_selection(context)
+        end
+
+        def should_verify_response_checksum?(context)
+          context[:http_checksum][:validation_list] && !context[:http_checksum][:validation_list].empty?
+        end
+
+        def calculate_request_checksum(context, checksum_properties)
+          case checksum_properties['in']
+          when 'header'
+            header_name = checksum_properties['name']
+            body = context.http_request.body_contents
+            if body
+              context.http_request.headers[header_name] ||=
+                ChecksumAlgorithm.calculate_checksum(checksum_properties['algorithm'], body)
+            end
+          when 'trailer'
+            apply_request_trailer_checksum(context, checksum_properties)
+          end
+        end
+
+        def apply_request_trailer_checksum(context, checksum_properties)
+          location_name = checksum_properties['name']
+
+          # set required headers
+          headers = context.http_request.headers
+          headers['Content-Encoding'] = 'aws-chunked'
+          headers['X-Amz-Content-Sha256'] = 'STREAMING-UNSIGNED-PAYLOAD-TRAILER'
+          headers['X-Amz-Trailer'] = location_name
+
+          # We currently always compute the size in the modified body wrapper - allowing us
+          # to set the Content-Length header (set by content_length plugin).
+          # This means we cannot use Transfer-Encoding=chunked
+
+          if !context.http_request.body.respond_to?(:size)
+            raise Aws::Errors::ChecksumError, 'Could not determine length of the body'
+          end
+          headers['X-Amz-Decoded-Content-Length'] = context.http_request.body.size
+
+          context.http_request.body = AwsChunkedTrailerDigestIO.new(
+            context.http_request.body,
+            checksum_properties['algorithm'],
+            location_name
+          )
+        end
+
+        # Add events to the http_response to verify the checksum as its read
+        # This prevents the body from being read multiple times
+        # verification is done only once a successful response has completed
+        def add_verify_response_checksum_handlers(context)
+          http_response = context.http_response
+          checksum_context = { }
+          http_response.on_headers do |_status, headers|
+            header_name, algorithm = response_header_to_verify(headers, context[:http_checksum][:validation_list])
+            if header_name
+              expected = headers[header_name]
+
+              unless context[:http_checksum][:skip_on_suffix] && /-[\d]+$/.match(expected)
+                checksum_context[:algorithm] = algorithm
+                checksum_context[:header_name] = header_name
+                checksum_context[:digest] = ChecksumAlgorithm.digest_for_algorithm(algorithm)
+                checksum_context[:expected] = expected
+              end
+            end
+          end
+
+          http_response.on_data do |chunk|
+            checksum_context[:digest].update(chunk) if checksum_context[:digest]
+          end
+
+          http_response.on_success do
+            if checksum_context[:digest] &&
+              (computed = checksum_context[:digest].base64digest)
+
+              if computed != checksum_context[:expected]
+                raise Aws::Errors::ChecksumError,
+                      "Checksum validation failed on #{checksum_context[:header_name]} "\
+                      "computed: #{computed}, expected: #{checksum_context[:expected]}"
+              end
+
+              context[:http_checksum][:validated] = checksum_context[:algorithm]
+            end
+          end
+        end
+
+        # returns nil if no headers to verify
+        def response_header_to_verify(headers, validation_list)
+          validation_list.each do |algorithm|
+            header_name = "x-amz-checksum-#{algorithm}"
+            return [header_name, algorithm] if headers[header_name]
+          end
+          nil
+        end
+
+        # determine where (header vs trailer) a request checksum should be added
+        def checksum_request_in(context)
+          if context.operation['authtype'].eql?('v4-unsigned-body')
+            'trailer'
+          else
+            'header'
+          end
+        end
+
+      end
+
+      def self.calculate_checksum(algorithm, body)
+        digest = ChecksumAlgorithm.digest_for_algorithm(algorithm)
+        if body.respond_to?(:read)
+          ChecksumAlgorithm.update_in_chunks(digest, body)
+        else
+          digest.update(body)
+        end
+        digest.base64digest
+      end
+
+      def self.digest_for_algorithm(algorithm)
+        case algorithm
+        when 'CRC32'
+          Digest32.new(Zlib.method(:crc32))
+        when 'CRC32C'
+          # this will only be used if input algorithm is CRC32C AND client supports it (crt available)
+          Digest32.new(Aws::Crt::Checksums.method(:crc32c))
+        when 'SHA1'
+          Digest::SHA1.new
+        when 'SHA256'
+          Digest::SHA256.new
+        end
+      end
+
+      # The trailer size (in bytes) is the overhead + the trailer name +
+      # the length of the base64 encoded checksum
+      def self.trailer_length(algorithm, location_name)
+        CHECKSUM_SIZE[algorithm] + location_name.size
+      end
+
+      def self.update_in_chunks(digest, io)
+        loop do
+          chunk = io.read(CHUNK_SIZE)
+          break unless chunk
+          digest.update(chunk)
+        end
+        io.rewind
+      end
+
+      # Wrapper for request body that implements application-layer
+      # chunking with Digest computed on chunks + added as a trailer
+      class AwsChunkedTrailerDigestIO
+        CHUNK_SIZE = 16384
+
+        def initialize(io, algorithm, location_name)
+          @io = io
+          @location_name = location_name
+          @algorithm = algorithm
+          @digest = ChecksumAlgorithm.digest_for_algorithm(algorithm)
+          @trailer_io = nil
+        end
+
+        # the size of the application layer aws-chunked + trailer body
+        def size
+          # compute the number of chunks
+          # a full chunk has 4 + 4 bytes overhead, a partial chunk is len.to_s(16).size + 4
+          orig_body_size = @io.size
+          n_full_chunks = orig_body_size / CHUNK_SIZE
+          partial_bytes = orig_body_size % CHUNK_SIZE
+          chunked_body_size = n_full_chunks * (CHUNK_SIZE + 8)
+          chunked_body_size += partial_bytes.to_s(16).size + partial_bytes + 4 unless  partial_bytes.zero?
+          trailer_size = ChecksumAlgorithm.trailer_length(@algorithm, @location_name)
+          chunked_body_size + trailer_size
+        end
+
+        def rewind
+          @io.rewind
+        end
+
+        def read(length, buf)
+          # account for possible leftover bytes at the end, if we have trailer bytes, send them
+          if @trailer_io
+            return @trailer_io.read(length, buf)
+          end
+
+          chunk = @io.read(length)
+          if chunk
+            @digest.update(chunk)
+            application_chunked = "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+            return StringIO.new(application_chunked).read(application_chunked.size, buf)
+          else
+            trailers = {}
+            trailers[@location_name] = @digest.base64digest
+            trailers = trailers.map { |k,v| "#{k}:#{v}"}.join("\r\n")
+            @trailer_io = StringIO.new("0\r\n#{trailers}\r\n\r\n")
+            chunk = @trailer_io.read(length, buf)
+          end
+          chunk
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/client_metrics_plugin.rb

@@ -0,0 +1,283 @@
+# frozen_string_literal: true
+
+require 'date'
+require_relative 'retries/error_inspector'
+
+module Aws
+  module Plugins
+    class ClientMetricsPlugin < Seahorse::Client::Plugin
+
+      option(:client_side_monitoring,
+        default: false,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+When `true`, client-side metrics will be collected for all API requests from
+this client.
+      DOCS
+        resolve_client_side_monitoring(cfg)
+      end
+
+      option(:client_side_monitoring_port,
+        default: 31000,
+        doc_type: Integer,
+        docstring: <<-DOCS) do |cfg|
+Required for publishing client metrics. The port that the client side monitoring
+agent is running on, where client metrics will be published via UDP.
+      DOCS
+        resolve_client_side_monitoring_port(cfg)
+      end
+
+      option(:client_side_monitoring_host,
+        default: "127.0.0.1",
+        doc_type: String,
+        docstring: <<-DOCS) do |cfg|
+Allows you to specify the DNS hostname or IPv4 or IPv6 address that the client
+side monitoring agent is running on, where client metrics will be published via UDP.
+      DOCS
+        resolve_client_side_monitoring_host(cfg)
+      end
+
+      option(:client_side_monitoring_publisher,
+        default: ClientSideMonitoring::Publisher,
+        doc_type: Aws::ClientSideMonitoring::Publisher,
+        docstring: <<-DOCS) do |cfg|
+Allows you to provide a custom client-side monitoring publisher class. By default,
+will use the Client Side Monitoring Agent Publisher.
+      DOCS
+        resolve_publisher(cfg)
+      end
+
+      option(:client_side_monitoring_client_id,
+        default: "",
+        doc_type: String,
+        docstring: <<-DOCS) do |cfg|
+Allows you to provide an identifier for this client which will be attached to
+all generated client side metrics. Defaults to an empty string.
+        DOCS
+        resolve_client_id(cfg)
+      end
+
+      def add_handlers(handlers, config)
+        if config.client_side_monitoring && config.client_side_monitoring_port
+          handlers.add(Handler, step: :initialize)
+          publisher = config.client_side_monitoring_publisher
+          publisher.agent_port = config.client_side_monitoring_port
+          publisher.agent_host = config.client_side_monitoring_host
+        end
+      end
+
+      private
+      def self.resolve_publisher(cfg)
+        ClientSideMonitoring::Publisher.new
+      end
+
+      def self.resolve_client_side_monitoring_port(cfg)
+        env_source = ENV["AWS_CSM_PORT"]
+        env_source = nil if env_source == ""
+        cfg_source = Aws.shared_config.csm_port(profile: cfg.profile)
+        if env_source
+          env_source.to_i
+        elsif cfg_source
+          cfg_source.to_i
+        else
+          31000
+        end
+      end
+
+      def self.resolve_client_side_monitoring_host(cfg)
+        env_source = ENV["AWS_CSM_HOST"]
+        env_source = nil if env_source == ""
+        cfg_source = Aws.shared_config.csm_host(profile: cfg.profile)
+        if env_source
+          env_source
+        elsif cfg_source
+          cfg_source
+        else
+          "127.0.0.1"
+        end
+      end
+
+      def self.resolve_client_side_monitoring(cfg)
+        env_source = ENV["AWS_CSM_ENABLED"]
+        env_source = nil if env_source == ""
+        if env_source.is_a?(String) && (env_source.downcase == "false" || env_source.downcase == "f")
+          env_source = false
+        end
+        cfg_source = Aws.shared_config.csm_enabled(profile: cfg.profile)
+        if env_source || cfg_source
+          true
+        else
+          false
+        end
+      end
+
+      def self.resolve_client_id(cfg)
+        default = ""
+        env_source = ENV["AWS_CSM_CLIENT_ID"]
+        env_source = nil if env_source == ""
+        cfg_source = Aws.shared_config.csm_client_id(profile: cfg.profile)
+        env_source || cfg_source || default
+      end
+
+      class Handler < Seahorse::Client::Handler
+        def call(context)
+          publisher = context.config.client_side_monitoring_publisher
+          service_id = context.config.api.metadata["serviceId"]
+          # serviceId not present in all versions, need a fallback
+          service_id ||= _calculate_service_id(context)
+
+          request_metrics = ClientSideMonitoring::RequestMetrics.new(
+            service: service_id,
+            operation: context.operation.name,
+            client_id: context.config.client_side_monitoring_client_id,
+            region: context.config.region,
+            timestamp: DateTime.now.strftime('%Q').to_i,
+          )
+          context.metadata[:client_metrics] = request_metrics
+          start_time = Aws::Util.monotonic_milliseconds
+          final_error_retryable = false
+          final_aws_exception = nil
+          final_aws_exception_message = nil
+          final_sdk_exception = nil
+          final_sdk_exception_message = nil
+          begin
+            @handler.call(context)
+          rescue StandardError => e
+            # Handle SDK Exceptions
+            inspector = Retries::ErrorInspector.new(
+              e,
+              context.http_response.status_code
+            )
+            if inspector.retryable?(context)
+              final_error_retryable = true
+            end
+            if request_metrics.api_call_attempts.empty?
+              attempt = request_metrics.build_call_attempt
+              attempt.sdk_exception = e.class.to_s
+              attempt.sdk_exception_msg = e.message
+              request_metrics.add_call_attempt(attempt)
+            elsif request_metrics.api_call_attempts.last.aws_exception.nil?
+              # Handle exceptions during response handlers
+              attempt = request_metrics.api_call_attempts.last
+              attempt.sdk_exception = e.class.to_s
+              attempt.sdk_exception_msg = e.message
+            elsif !e.class.to_s.match(request_metrics.api_call_attempts.last.aws_exception)
+              # Handle response handling exceptions that happened in addition to
+              # an AWS exception
+              attempt = request_metrics.api_call_attempts.last
+              attempt.sdk_exception = e.class.to_s
+              attempt.sdk_exception_msg = e.message
+            end # Else we don't have an SDK exception and are done.
+            final_attempt = request_metrics.api_call_attempts.last
+            final_aws_exception = final_attempt.aws_exception
+            final_aws_exception_message = final_attempt.aws_exception_msg
+            final_sdk_exception = final_attempt.sdk_exception
+            final_sdk_exception_message = final_attempt.sdk_exception_msg
+            raise e
+          ensure
+            end_time = Aws::Util.monotonic_milliseconds
+            complete_opts = {
+              latency: end_time - start_time,
+              attempt_count: context.retries + 1,
+              user_agent: context.http_request.headers["user-agent"],
+              final_error_retryable: final_error_retryable,
+              final_http_status_code: context.http_response.status_code,
+              final_aws_exception: final_aws_exception,
+              final_aws_exception_message: final_aws_exception_message,
+              final_sdk_exception: final_sdk_exception,
+              final_sdk_exception_message: final_sdk_exception_message
+            }
+            if context.metadata[:redirect_region]
+              complete_opts[:region] = context.metadata[:redirect_region]
+            end
+            request_metrics.api_call.complete(complete_opts)
+            # Report the metrics by passing the complete RequestMetrics object
+            if publisher
+              publisher.publish(request_metrics)
+            end # Else we drop all this on the floor.
+          end
+        end
+
+        private
+        def _calculate_service_id(context)
+          class_name = context.client.class.to_s.match(/(.+)::Client/)[1]
+          class_name.sub!(/^Aws::/, '')
+          _fallback_service_id(class_name)
+        end
+
+        def _fallback_service_id(id)
+          # Need hard-coded exceptions since information needed to
+          # reverse-engineer serviceId is not present in older versions.
+          # This list should not need to grow.
+          exceptions = {
+            "ACMPCA" => "ACM PCA",
+            "APIGateway" => "API Gateway",
+            "AlexaForBusiness" => "Alexa For Business",
+            "ApplicationAutoScaling" => "Application Auto Scaling",
+            "ApplicationDiscoveryService" => "Application Discovery Service",
+            "AutoScaling" => "Auto Scaling",
+            "AutoScalingPlans" => "Auto Scaling Plans",
+            "CloudHSMV2" => "CloudHSM V2",
+            "CloudSearchDomain" => "CloudSearch Domain",
+            "CloudWatchEvents" => "CloudWatch Events",
+            "CloudWatchLogs" => "CloudWatch Logs",
+            "CognitoIdentity" => "Cognito Identity",
+            "CognitoIdentityProvider" => "Cognito Identity Provider",
+            "CognitoSync" => "Cognito Sync",
+            "ConfigService" => "Config Service",
+            "CostExplorer" => "Cost Explorer",
+            "CostandUsageReportService" => "Cost and Usage Report Service",
+            "DataPipeline" => "Data Pipeline",
+            "DatabaseMigrationService" => "Database Migration Service",
+            "DeviceFarm" => "Device Farm",
+            "DirectConnect" => "Direct Connect",
+            "DirectoryService" => "Directory Service",
+            "DynamoDBStreams" => "DynamoDB Streams",
+            "ElasticBeanstalk" => "Elastic Beanstalk",
+            "ElasticLoadBalancing" => "Elastic Load Balancing",
+            "ElasticLoadBalancingV2" => "Elastic Load Balancing v2",
+            "ElasticTranscoder" => "Elastic Transcoder",
+            "ElasticsearchService" => "Elasticsearch Service",
+            "IoTDataPlane" => "IoT Data Plane",
+            "IoTJobsDataPlane" => "IoT Jobs Data Plane",
+            "IoT1ClickDevicesService" => "IoT 1Click Devices Service",
+            "IoT1ClickProjects" => "IoT 1Click Projects",
+            "KinesisAnalytics" => "Kinesis Analytics",
+            "KinesisVideo" => "Kinesis Video",
+            "KinesisVideoArchivedMedia" => "Kinesis Video Archived Media",
+            "KinesisVideoMedia" => "Kinesis Video Media",
+            "LambdaPreview" => "Lambda",
+            "Lex" => "Lex Runtime Service",
+            "LexModelBuildingService" => "Lex Model Building Service",
+            "Lightsail" => "Lightsail",
+            "MQ" => "mq",
+            "MachineLearning" => "Machine Learning",
+            "MarketplaceCommerceAnalytics" => "Marketplace Commerce Analytics",
+            "MarketplaceEntitlementService" => "Marketplace Entitlement Service",
+            "MarketplaceMetering" => "Marketplace Metering",
+            "MediaStoreData" => "MediaStore Data",
+            "MigrationHub" => "Migration Hub",
+            "ResourceGroups" => "Resource Groups",
+            "ResourceGroupsTaggingAPI" => "Resource Groups Tagging API",
+            "Route53" => "Route 53",
+            "Route53Domains" => "Route 53 Domains",
+            "SecretsManager" => "Secrets Manager",
+            "SageMakerRuntime" => "SageMaker Runtime",
+            "ServiceCatalog" => "Service Catalog",
+            "ServiceDiscovery" => "ServiceDiscovery",
+            "Signer" => "signer",
+            "States" => "SFN",
+            "StorageGateway" => "Storage Gateway",
+            "TranscribeService" => "Transcribe Service",
+            "WAFRegional" => "WAF Regional",
+          }
+          if exceptions[id]
+            exceptions[id]
+          else
+            id
+          end
+        end
+      end
+    end
+  end
+end