aws-sdk-bedrock 1.41.0 → 1.42.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87923521772094c9ec78f8e86a115fe80cc75f9094a6123b8d7d094fe4cbc07d
-  data.tar.gz: 73e72424273ad3dc26502454113fa810107fba80b7e02a4ef69ebb6ba2730e05
+  metadata.gz: 9d0f164ba5fc2410b7f4f1e90bba72a55935b8968fa451d9a672a117b085a9a4
+  data.tar.gz: 3afc3e64b1a798f8f8755e0f827e82922350794c92e88e4de9f0f04875e2f73f
 SHA512:
-  metadata.gz: 837e2fd591a620addc22e9f1e68947b679af09089f7663ee38e04c2eec6a58589b1828b462f92c1adf4fa3695ca2ccee62d53443680194ab4e4e75433cfeabd9
-  data.tar.gz: 25c1c7e1756a78927a4836a66bdaf27dd443d6f0daa1aedfca9c68938c7569353fa11f0d78d67b31f30629bcc6da40c81d1e55466cf7bffe8b88f4e095304866
+  metadata.gz: 94dc660266ae76090c24be7ebbce195fe4c0745a4bf04187fd09bd107377f4cfb01d5e33b9517ddc9f236c0d21f84dccc74e4f57c21df6c6d921f03d77cbb58a
+  data.tar.gz: d09329fabb979c240e03bdc1f98364fdc033e27428e0b6aedbd3cb7e57211a447ad158b5467129d3ef17d6dca746488198b4cf40110ac3b8507309ce0f12059a

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 Unreleased Changes
 ------------------
 
+1.42.0 (2025-04-17)
+------------------
+
+* Feature - With this release, Bedrock Evaluation will now support custom metrics for evaluation.
+
 1.41.0 (2025-04-07)
 ------------------
 

data/VERSION

@@ -1 +1 @@
-1.41.0
+1.42.0

data/lib/aws-sdk-bedrock/client.rb

@@ -484,6 +484,35 @@ module Aws::Bedrock
     #   * {Types::BatchDeleteEvaluationJobResponse#errors #errors} => Array<Types::BatchDeleteEvaluationJobError>
     #   * {Types::BatchDeleteEvaluationJobResponse#evaluation_jobs #evaluation_jobs} => Array<Types::BatchDeleteEvaluationJobItem>
     #
+    #
+    # @example Example: Delete evaluation jobs
+    #
+    #   # The following example shows a request to delete two model evaluation jobs, where one of the jobs is not found.
+    #
+    #   resp = client.batch_delete_evaluation_job({
+    #     job_identifiers: [
+    #       "arn:aws:bedrock:us-east-2:123456789012:evaluation-job/12rnxmplqv0v", 
+    #       "arn:aws:bedrock:us-east-2:123456789012:evaluation-job/rispxmpl12rn", 
+    #     ], 
+    #   })
+    #
+    #   resp.to_h outputs the following:
+    #   {
+    #     errors: [
+    #       {
+    #         code: "404", 
+    #         job_identifier: "arn:aws:bedrock:us-east-2:123456789012:evaluation-job/rispxmpl12rn", 
+    #         message: "Unable to locate this job to delete.", 
+    #       }, 
+    #     ], 
+    #     evaluation_jobs: [
+    #       {
+    #         job_identifier: "arn:aws:bedrock:us-east-2:123456789012:evaluation-job/12rnxmplqv0v", 
+    #         job_status: "Deleting", 
+    #       }, 
+    #     ], 
+    #   }
+    #
     # @example Request syntax with placeholder values
     #
     #   resp = client.batch_delete_evaluation_job({
@@ -612,6 +641,32 @@ module Aws::Bedrock
     #             },
     #           ],
     #         },
+    #         custom_metric_config: {
+    #           custom_metrics: [ # required
+    #             {
+    #               custom_metric_definition: {
+    #                 name: "MetricName", # required
+    #                 instructions: "CustomMetricInstructions", # required
+    #                 rating_scale: [
+    #                   {
+    #                     definition: "RatingScaleItemDefinition", # required
+    #                     value: { # required
+    #                       string_value: "RatingScaleItemValueStringValueString",
+    #                       float_value: 1.0,
+    #                     },
+    #                   },
+    #                 ],
+    #               },
+    #             },
+    #           ],
+    #           evaluator_model_config: { # required
+    #             bedrock_evaluator_models: [ # required
+    #               {
+    #                 model_identifier: "EvaluatorModelIdentifier", # required
+    #               },
+    #             ],
+    #           },
+    #         },
     #       },
     #       human: {
     #         human_workflow_config: {
@@ -2367,6 +2422,15 @@ module Aws::Bedrock
     #   resp.evaluation_config.automated.dataset_metric_configs[0].metric_names[0] #=> String
     #   resp.evaluation_config.automated.evaluator_model_config.bedrock_evaluator_models #=> Array
     #   resp.evaluation_config.automated.evaluator_model_config.bedrock_evaluator_models[0].model_identifier #=> String
+    #   resp.evaluation_config.automated.custom_metric_config.custom_metrics #=> Array
+    #   resp.evaluation_config.automated.custom_metric_config.custom_metrics[0].custom_metric_definition.name #=> String
+    #   resp.evaluation_config.automated.custom_metric_config.custom_metrics[0].custom_metric_definition.instructions #=> String
+    #   resp.evaluation_config.automated.custom_metric_config.custom_metrics[0].custom_metric_definition.rating_scale #=> Array
+    #   resp.evaluation_config.automated.custom_metric_config.custom_metrics[0].custom_metric_definition.rating_scale[0].definition #=> String
+    #   resp.evaluation_config.automated.custom_metric_config.custom_metrics[0].custom_metric_definition.rating_scale[0].value.string_value #=> String
+    #   resp.evaluation_config.automated.custom_metric_config.custom_metrics[0].custom_metric_definition.rating_scale[0].value.float_value #=> Float
+    #   resp.evaluation_config.automated.custom_metric_config.evaluator_model_config.bedrock_evaluator_models #=> Array
+    #   resp.evaluation_config.automated.custom_metric_config.evaluator_model_config.bedrock_evaluator_models[0].model_identifier #=> String
     #   resp.evaluation_config.human.human_workflow_config.flow_definition_arn #=> String
     #   resp.evaluation_config.human.human_workflow_config.instructions #=> String
     #   resp.evaluation_config.human.custom_metrics #=> Array
@@ -3357,6 +3421,8 @@ module Aws::Bedrock
     #   resp.job_summaries[0].rag_identifiers[0] #=> String
     #   resp.job_summaries[0].evaluator_model_identifiers #=> Array
     #   resp.job_summaries[0].evaluator_model_identifiers[0] #=> String
+    #   resp.job_summaries[0].custom_metrics_evaluator_model_identifiers #=> Array
+    #   resp.job_summaries[0].custom_metrics_evaluator_model_identifiers[0] #=> String
     #   resp.job_summaries[0].inference_config_summary.model_config_summary.bedrock_model_identifiers #=> Array
     #   resp.job_summaries[0].inference_config_summary.model_config_summary.bedrock_model_identifiers[0] #=> String
     #   resp.job_summaries[0].inference_config_summary.model_config_summary.precomputed_inference_source_identifiers #=> Array
@@ -4029,7 +4095,7 @@ module Aws::Bedrock
     #
     #   * Failed – This job has failed. Check the failure message for any
     #     further details. For further assistance, reach out to the [Amazon
-    #     Web Services Support Center][3].
+    #     Web ServicesSupport Center][3].
     #
     #   * Stopped – This job was stopped by a user.
     #
@@ -4866,7 +4932,7 @@ module Aws::Bedrock
         tracer: tracer
       )
       context[:gem_name] = 'aws-sdk-bedrock'
-      context[:gem_version] = '1.41.0'
+      context[:gem_version] = '1.42.0'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-bedrock/client_api.rb

@@ -23,6 +23,9 @@ module Aws::Bedrock
     ApplicationType = Shapes::StringShape.new(name: 'ApplicationType')
     Arn = Shapes::StringShape.new(name: 'Arn')
     AutomatedEvaluationConfig = Shapes::StructureShape.new(name: 'AutomatedEvaluationConfig')
+    AutomatedEvaluationCustomMetricConfig = Shapes::StructureShape.new(name: 'AutomatedEvaluationCustomMetricConfig')
+    AutomatedEvaluationCustomMetricSource = Shapes::UnionShape.new(name: 'AutomatedEvaluationCustomMetricSource')
+    AutomatedEvaluationCustomMetrics = Shapes::ListShape.new(name: 'AutomatedEvaluationCustomMetrics')
     BaseModelIdentifier = Shapes::StringShape.new(name: 'BaseModelIdentifier')
     BatchDeleteEvaluationJobError = Shapes::StructureShape.new(name: 'BatchDeleteEvaluationJobError')
     BatchDeleteEvaluationJobErrors = Shapes::ListShape.new(name: 'BatchDeleteEvaluationJobErrors')
@@ -65,6 +68,11 @@ module Aws::Bedrock
     CreatePromptRouterResponse = Shapes::StructureShape.new(name: 'CreatePromptRouterResponse')
     CreateProvisionedModelThroughputRequest = Shapes::StructureShape.new(name: 'CreateProvisionedModelThroughputRequest')
     CreateProvisionedModelThroughputResponse = Shapes::StructureShape.new(name: 'CreateProvisionedModelThroughputResponse')
+    CustomMetricBedrockEvaluatorModel = Shapes::StructureShape.new(name: 'CustomMetricBedrockEvaluatorModel')
+    CustomMetricBedrockEvaluatorModels = Shapes::ListShape.new(name: 'CustomMetricBedrockEvaluatorModels')
+    CustomMetricDefinition = Shapes::StructureShape.new(name: 'CustomMetricDefinition')
+    CustomMetricEvaluatorModelConfig = Shapes::StructureShape.new(name: 'CustomMetricEvaluatorModelConfig')
+    CustomMetricInstructions = Shapes::StringShape.new(name: 'CustomMetricInstructions')
     CustomModelArn = Shapes::StringShape.new(name: 'CustomModelArn')
     CustomModelName = Shapes::StringShape.new(name: 'CustomModelName')
     CustomModelSummary = Shapes::StructureShape.new(name: 'CustomModelSummary')
@@ -149,6 +157,7 @@ module Aws::Bedrock
     FilterKey = Shapes::StringShape.new(name: 'FilterKey')
     FilterValue = Shapes::DocumentShape.new(name: 'FilterValue', document: true)
     FineTuningJobStatus = Shapes::StringShape.new(name: 'FineTuningJobStatus')
+    Float = Shapes::FloatShape.new(name: 'Float')
     FoundationModelArn = Shapes::StringShape.new(name: 'FoundationModelArn')
     FoundationModelDetails = Shapes::StructureShape.new(name: 'FoundationModelDetails')
     FoundationModelLifecycle = Shapes::StructureShape.new(name: 'FoundationModelLifecycle')
@@ -350,6 +359,7 @@ module Aws::Bedrock
     MaxTokens = Shapes::IntegerShape.new(name: 'MaxTokens')
     Message = Shapes::StringShape.new(name: 'Message')
     MetricFloat = Shapes::FloatShape.new(name: 'MetricFloat')
+    MetricName = Shapes::StringShape.new(name: 'MetricName')
     ModelArchitecture = Shapes::StringShape.new(name: 'ModelArchitecture')
     ModelArn = Shapes::StringShape.new(name: 'ModelArn')
     ModelCopyJobArn = Shapes::StringShape.new(name: 'ModelCopyJobArn')
@@ -421,6 +431,11 @@ module Aws::Bedrock
     RAGStopSequences = Shapes::ListShape.new(name: 'RAGStopSequences')
     RAGStopSequencesMemberString = Shapes::StringShape.new(name: 'RAGStopSequencesMemberString')
     RagConfigs = Shapes::ListShape.new(name: 'RagConfigs')
+    RatingScale = Shapes::ListShape.new(name: 'RatingScale')
+    RatingScaleItem = Shapes::StructureShape.new(name: 'RatingScaleItem')
+    RatingScaleItemDefinition = Shapes::StringShape.new(name: 'RatingScaleItemDefinition')
+    RatingScaleItemValue = Shapes::UnionShape.new(name: 'RatingScaleItemValue')
+    RatingScaleItemValueStringValueString = Shapes::StringShape.new(name: 'RatingScaleItemValueStringValueString')
     RegisterMarketplaceModelEndpointRequest = Shapes::StructureShape.new(name: 'RegisterMarketplaceModelEndpointRequest')
     RegisterMarketplaceModelEndpointResponse = Shapes::StructureShape.new(name: 'RegisterMarketplaceModelEndpointResponse')
     RequestMetadataBaseFilters = Shapes::StructureShape.new(name: 'RequestMetadataBaseFilters')
@@ -509,8 +524,21 @@ module Aws::Bedrock
 
     AutomatedEvaluationConfig.add_member(:dataset_metric_configs, Shapes::ShapeRef.new(shape: EvaluationDatasetMetricConfigs, required: true, location_name: "datasetMetricConfigs"))
     AutomatedEvaluationConfig.add_member(:evaluator_model_config, Shapes::ShapeRef.new(shape: EvaluatorModelConfig, location_name: "evaluatorModelConfig"))
+    AutomatedEvaluationConfig.add_member(:custom_metric_config, Shapes::ShapeRef.new(shape: AutomatedEvaluationCustomMetricConfig, location_name: "customMetricConfig"))
     AutomatedEvaluationConfig.struct_class = Types::AutomatedEvaluationConfig
 
+    AutomatedEvaluationCustomMetricConfig.add_member(:custom_metrics, Shapes::ShapeRef.new(shape: AutomatedEvaluationCustomMetrics, required: true, location_name: "customMetrics"))
+    AutomatedEvaluationCustomMetricConfig.add_member(:evaluator_model_config, Shapes::ShapeRef.new(shape: CustomMetricEvaluatorModelConfig, required: true, location_name: "evaluatorModelConfig"))
+    AutomatedEvaluationCustomMetricConfig.struct_class = Types::AutomatedEvaluationCustomMetricConfig
+
+    AutomatedEvaluationCustomMetricSource.add_member(:custom_metric_definition, Shapes::ShapeRef.new(shape: CustomMetricDefinition, location_name: "customMetricDefinition"))
+    AutomatedEvaluationCustomMetricSource.add_member(:unknown, Shapes::ShapeRef.new(shape: nil, location_name: 'unknown'))
+    AutomatedEvaluationCustomMetricSource.add_member_subclass(:custom_metric_definition, Types::AutomatedEvaluationCustomMetricSource::CustomMetricDefinition)
+    AutomatedEvaluationCustomMetricSource.add_member_subclass(:unknown, Types::AutomatedEvaluationCustomMetricSource::Unknown)
+    AutomatedEvaluationCustomMetricSource.struct_class = Types::AutomatedEvaluationCustomMetricSource
+
+    AutomatedEvaluationCustomMetrics.member = Shapes::ShapeRef.new(shape: AutomatedEvaluationCustomMetricSource)
+
     BatchDeleteEvaluationJobError.add_member(:job_identifier, Shapes::ShapeRef.new(shape: EvaluationJobIdentifier, required: true, location_name: "jobIdentifier"))
     BatchDeleteEvaluationJobError.add_member(:code, Shapes::ShapeRef.new(shape: String, required: true, location_name: "code"))
     BatchDeleteEvaluationJobError.add_member(:message, Shapes::ShapeRef.new(shape: String, location_name: "message"))
@@ -696,6 +724,19 @@ module Aws::Bedrock
     CreateProvisionedModelThroughputResponse.add_member(:provisioned_model_arn, Shapes::ShapeRef.new(shape: ProvisionedModelArn, required: true, location_name: "provisionedModelArn"))
     CreateProvisionedModelThroughputResponse.struct_class = Types::CreateProvisionedModelThroughputResponse
 
+    CustomMetricBedrockEvaluatorModel.add_member(:model_identifier, Shapes::ShapeRef.new(shape: EvaluatorModelIdentifier, required: true, location_name: "modelIdentifier"))
+    CustomMetricBedrockEvaluatorModel.struct_class = Types::CustomMetricBedrockEvaluatorModel
+
+    CustomMetricBedrockEvaluatorModels.member = Shapes::ShapeRef.new(shape: CustomMetricBedrockEvaluatorModel)
+
+    CustomMetricDefinition.add_member(:name, Shapes::ShapeRef.new(shape: MetricName, required: true, location_name: "name"))
+    CustomMetricDefinition.add_member(:instructions, Shapes::ShapeRef.new(shape: CustomMetricInstructions, required: true, location_name: "instructions"))
+    CustomMetricDefinition.add_member(:rating_scale, Shapes::ShapeRef.new(shape: RatingScale, location_name: "ratingScale"))
+    CustomMetricDefinition.struct_class = Types::CustomMetricDefinition
+
+    CustomMetricEvaluatorModelConfig.add_member(:bedrock_evaluator_models, Shapes::ShapeRef.new(shape: CustomMetricBedrockEvaluatorModels, required: true, location_name: "bedrockEvaluatorModels"))
+    CustomMetricEvaluatorModelConfig.struct_class = Types::CustomMetricEvaluatorModelConfig
+
     CustomModelSummary.add_member(:model_arn, Shapes::ShapeRef.new(shape: CustomModelArn, required: true, location_name: "modelArn"))
     CustomModelSummary.add_member(:model_name, Shapes::ShapeRef.new(shape: CustomModelName, required: true, location_name: "modelName"))
     CustomModelSummary.add_member(:creation_time, Shapes::ShapeRef.new(shape: Timestamp, required: true, location_name: "creationTime"))
@@ -873,9 +914,10 @@ module Aws::Bedrock
     EvaluationSummary.add_member(:creation_time, Shapes::ShapeRef.new(shape: Timestamp, required: true, location_name: "creationTime"))
     EvaluationSummary.add_member(:job_type, Shapes::ShapeRef.new(shape: EvaluationJobType, required: true, location_name: "jobType"))
     EvaluationSummary.add_member(:evaluation_task_types, Shapes::ShapeRef.new(shape: EvaluationTaskTypes, required: true, location_name: "evaluationTaskTypes"))
-    EvaluationSummary.add_member(:model_identifiers, Shapes::ShapeRef.new(shape: EvaluationBedrockModelIdentifiers, deprecated: true, location_name: "modelIdentifiers", metadata: {"deprecatedMessage"=>"Inference identifiers should be retrieved from the inferenceConfigSummary"}))
-    EvaluationSummary.add_member(:rag_identifiers, Shapes::ShapeRef.new(shape: EvaluationBedrockKnowledgeBaseIdentifiers, deprecated: true, location_name: "ragIdentifiers", metadata: {"deprecatedMessage"=>"Inference identifiers should be retrieved from the inferenceConfigSummary"}))
+    EvaluationSummary.add_member(:model_identifiers, Shapes::ShapeRef.new(shape: EvaluationBedrockModelIdentifiers, deprecated: true, location_name: "modelIdentifiers", metadata: {"deprecatedMessage"=>"Inference identifiers should be retrieved from the inferenceConfigSummary", "deprecatedSince"=>"2025-03-07"}))
+    EvaluationSummary.add_member(:rag_identifiers, Shapes::ShapeRef.new(shape: EvaluationBedrockKnowledgeBaseIdentifiers, deprecated: true, location_name: "ragIdentifiers", metadata: {"deprecatedMessage"=>"Inference identifiers should be retrieved from the inferenceConfigSummary", "deprecatedSince"=>"2025-03-07"}))
     EvaluationSummary.add_member(:evaluator_model_identifiers, Shapes::ShapeRef.new(shape: EvaluatorModelIdentifiers, location_name: "evaluatorModelIdentifiers"))
+    EvaluationSummary.add_member(:custom_metrics_evaluator_model_identifiers, Shapes::ShapeRef.new(shape: EvaluatorModelIdentifiers, location_name: "customMetricsEvaluatorModelIdentifiers"))
     EvaluationSummary.add_member(:inference_config_summary, Shapes::ShapeRef.new(shape: EvaluationInferenceConfigSummary, location_name: "inferenceConfigSummary"))
     EvaluationSummary.add_member(:application_type, Shapes::ShapeRef.new(shape: ApplicationType, location_name: "applicationType"))
     EvaluationSummary.struct_class = Types::EvaluationSummary
@@ -1824,6 +1866,20 @@ module Aws::Bedrock
 
     RagConfigs.member = Shapes::ShapeRef.new(shape: RAGConfig)
 
+    RatingScale.member = Shapes::ShapeRef.new(shape: RatingScaleItem)
+
+    RatingScaleItem.add_member(:definition, Shapes::ShapeRef.new(shape: RatingScaleItemDefinition, required: true, location_name: "definition"))
+    RatingScaleItem.add_member(:value, Shapes::ShapeRef.new(shape: RatingScaleItemValue, required: true, location_name: "value"))
+    RatingScaleItem.struct_class = Types::RatingScaleItem
+
+    RatingScaleItemValue.add_member(:string_value, Shapes::ShapeRef.new(shape: RatingScaleItemValueStringValueString, location_name: "stringValue"))
+    RatingScaleItemValue.add_member(:float_value, Shapes::ShapeRef.new(shape: Float, location_name: "floatValue"))
+    RatingScaleItemValue.add_member(:unknown, Shapes::ShapeRef.new(shape: nil, location_name: 'unknown'))
+    RatingScaleItemValue.add_member_subclass(:string_value, Types::RatingScaleItemValue::StringValue)
+    RatingScaleItemValue.add_member_subclass(:float_value, Types::RatingScaleItemValue::FloatValue)
+    RatingScaleItemValue.add_member_subclass(:unknown, Types::RatingScaleItemValue::Unknown)
+    RatingScaleItemValue.struct_class = Types::RatingScaleItemValue
+
     RegisterMarketplaceModelEndpointRequest.add_member(:endpoint_identifier, Shapes::ShapeRef.new(shape: Arn, required: true, location: "uri", location_name: "endpointIdentifier"))
     RegisterMarketplaceModelEndpointRequest.add_member(:model_source_identifier, Shapes::ShapeRef.new(shape: ModelSourceIdentifier, required: true, location_name: "modelSourceIdentifier"))
     RegisterMarketplaceModelEndpointRequest.struct_class = Types::RegisterMarketplaceModelEndpointRequest

data/lib/aws-sdk-bedrock/types.rb

@@ -39,15 +39,76 @@ module Aws::Bedrock
     #   This model computes all evaluation related metrics.
     #   @return [Types::EvaluatorModelConfig]
     #
+    # @!attribute [rw] custom_metric_config
+    #   Defines the configuration of custom metrics to be used in an
+    #   evaluation job.
+    #   @return [Types::AutomatedEvaluationCustomMetricConfig]
+    #
     # @see http://docs.aws.amazon.com/goto/WebAPI/bedrock-2023-04-20/AutomatedEvaluationConfig AWS API Documentation
     #
     class AutomatedEvaluationConfig < Struct.new(
       :dataset_metric_configs,
+      :evaluator_model_config,
+      :custom_metric_config)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
+    # Defines the configuration of custom metrics to be used in an
+    # evaluation job. To learn more about using custom metrics in Amazon
+    # Bedrock evaluation jobs, see [Create a prompt for a custom metrics
+    # (LLM-as-a-judge model evaluations)][1] and [Create a prompt for a
+    # custom metrics (RAG evaluations)][2].
+    #
+    #
+    #
+    # [1]: https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation-custom-metrics-prompt-formats.html
+    # [2]: https://docs.aws.amazon.com/bedrock/latest/userguide/kb-evaluation-custom-metrics-prompt-formats.html
+    #
+    # @!attribute [rw] custom_metrics
+    #   Defines a list of custom metrics to be used in an Amazon Bedrock
+    #   evaluation job.
+    #   @return [Array<Types::AutomatedEvaluationCustomMetricSource>]
+    #
+    # @!attribute [rw] evaluator_model_config
+    #   Configuration of the evaluator model you want to use to evaluate
+    #   custom metrics in an Amazon Bedrock evaluation job.
+    #   @return [Types::CustomMetricEvaluatorModelConfig]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/bedrock-2023-04-20/AutomatedEvaluationCustomMetricConfig AWS API Documentation
+    #
+    class AutomatedEvaluationCustomMetricConfig < Struct.new(
+      :custom_metrics,
       :evaluator_model_config)
       SENSITIVE = []
       include Aws::Structure
     end
 
+    # An array item definining a single custom metric for use in an Amazon
+    # Bedrock evaluation job.
+    #
+    # @note AutomatedEvaluationCustomMetricSource is a union - when making an API calls you must set exactly one of the members.
+    #
+    # @note AutomatedEvaluationCustomMetricSource is a union - when returned from an API call exactly one value will be set and the returned type will be a subclass of AutomatedEvaluationCustomMetricSource corresponding to the set member.
+    #
+    # @!attribute [rw] custom_metric_definition
+    #   The definition of a custom metric for use in an Amazon Bedrock
+    #   evaluation job.
+    #   @return [Types::CustomMetricDefinition]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/bedrock-2023-04-20/AutomatedEvaluationCustomMetricSource AWS API Documentation
+    #
+    class AutomatedEvaluationCustomMetricSource < Struct.new(
+      :custom_metric_definition,
+      :unknown)
+      SENSITIVE = [:custom_metric_definition]
+      include Aws::Structure
+      include Aws::Structure::Union
+
+      class CustomMetricDefinition < AutomatedEvaluationCustomMetricSource; end
+      class Unknown < AutomatedEvaluationCustomMetricSource; end
+    end
+
     # A JSON array that provides the status of the evaluation jobs being
     # deleted.
     #
@@ -1101,6 +1162,95 @@ module Aws::Bedrock
       include Aws::Structure
     end
 
+    # Defines the model you want to evaluate custom metrics in an Amazon
+    # Bedrock evaluation job.
+    #
+    # @!attribute [rw] model_identifier
+    #   The Amazon Resource Name (ARN) of the evaluator model for custom
+    #   metrics. For a list of supported evaluator models, see [Evaluate
+    #   model performance using another LLM as a judge][1] and [Evaluate the
+    #   performance of RAG sources using Amazon Bedrock evaluations][2].
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation-judge.html
+    #   [2]: https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation-kb.html
+    #   @return [String]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/bedrock-2023-04-20/CustomMetricBedrockEvaluatorModel AWS API Documentation
+    #
+    class CustomMetricBedrockEvaluatorModel < Struct.new(
+      :model_identifier)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
+    # The definition of a custom metric for use in an Amazon Bedrock
+    # evaluation job. A custom metric definition includes a metric name,
+    # prompt (instructions) and optionally, a rating scale. Your prompt must
+    # include a task description and input variables. The required input
+    # variables are different for model-as-a-judge and RAG evaluations.
+    #
+    # For more information about how to define a custom metric in Amazon
+    # Bedrock, see [Create a prompt for a custom metrics (LLM-as-a-judge
+    # model evaluations)][1] and [Create a prompt for a custom metrics (RAG
+    # evaluations)][2].
+    #
+    #
+    #
+    # [1]: https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation-custom-metrics-prompt-formats.html
+    # [2]: https://docs.aws.amazon.com/bedrock/latest/userguide/kb-evaluation-custom-metrics-prompt-formats.html
+    #
+    # @!attribute [rw] name
+    #   The name for a custom metric. Names must be unique in your Amazon
+    #   Web Services region.
+    #   @return [String]
+    #
+    # @!attribute [rw] instructions
+    #   The prompt for a custom metric that instructs the evaluator model
+    #   how to rate the model or RAG source under evaluation.
+    #   @return [String]
+    #
+    # @!attribute [rw] rating_scale
+    #   Defines the rating scale to be used for a custom metric. We
+    #   recommend that you always define a ratings scale when creating a
+    #   custom metric. If you don't define a scale, Amazon Bedrock won't
+    #   be able to visually display the results of the evaluation in the
+    #   console or calculate average values of numerical scores. For more
+    #   information on specifying a rating scale, see [Specifying an output
+    #   schema (rating scale)][1].
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation-custom-metrics-prompt-formats.html#model-evaluation-custom-metrics-prompt-formats-schema
+    #   @return [Array<Types::RatingScaleItem>]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/bedrock-2023-04-20/CustomMetricDefinition AWS API Documentation
+    #
+    class CustomMetricDefinition < Struct.new(
+      :name,
+      :instructions,
+      :rating_scale)
+      SENSITIVE = [:name]
+      include Aws::Structure
+    end
+
+    # Configuration of the evaluator model you want to use to evaluate
+    # custom metrics in an Amazon Bedrock evaluation job.
+    #
+    # @!attribute [rw] bedrock_evaluator_models
+    #   Defines the model you want to evaluate custom metrics in an Amazon
+    #   Bedrock evaluation job.
+    #   @return [Array<Types::CustomMetricBedrockEvaluatorModel>]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/bedrock-2023-04-20/CustomMetricEvaluatorModelConfig AWS API Documentation
+    #
+    class CustomMetricEvaluatorModelConfig < Struct.new(
+      :bedrock_evaluator_models)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
     # Summary information for a custom model.
     #
     # @!attribute [rw] model_arn
@@ -1526,7 +1676,7 @@ module Aws::Bedrock
     #
     #   For knowledge base evaluation jobs that evaluate retrieval only,
     #   valid values are "`Builtin.ContextRelevance`",
-    #   "`Builtin.ContextConverage`".
+    #   "`Builtin.ContextCoverage`".
     #
     #   For knowledge base evaluation jobs that evaluate retrieval with
     #   response generation, valid values are "`Builtin.Correctness`",
@@ -1847,6 +1997,11 @@ module Aws::Bedrock
     #   metrics for a knowledge base evaluation job.
     #   @return [Array<String>]
     #
+    # @!attribute [rw] custom_metrics_evaluator_model_identifiers
+    #   The Amazon Resource Names (ARNs) of the models used to compute
+    #   custom metrics in an Amazon Bedrock evaluation job.
+    #   @return [Array<String>]
+    #
     # @!attribute [rw] inference_config_summary
     #   Identifies the models, Knowledge Bases, or other RAG sources
     #   evaluated in a model or Knowledge Base evaluation job.
@@ -1869,6 +2024,7 @@ module Aws::Bedrock
       :model_identifiers,
       :rag_identifiers,
       :evaluator_model_identifiers,
+      :custom_metrics_evaluator_model_identifiers,
       :inference_config_summary,
       :application_type)
       SENSITIVE = []
@@ -3089,7 +3245,7 @@ module Aws::Bedrock
     #
     #   * Failed – This job has failed. Check the failure message for any
     #     further details. For further assistance, reach out to the [Amazon
-    #     Web Services Support Center][3].
+    #     Web ServicesSupport Center][3].
     #
     #   * Stopped – This job was stopped by a user.
     #
@@ -5895,7 +6051,7 @@ module Aws::Bedrock
     #
     #   * Failed – This job has failed. Check the failure message for any
     #     further details. For further assistance, reach out to the [Amazon
-    #     Web Services Support Center][3].
+    #     Web ServicesSupport Center][3].
     #
     #   * Stopped – This job was stopped by a user.
     #
@@ -6651,7 +6807,7 @@ module Aws::Bedrock
     #
     #   * Failed – This job has failed. Check the failure message for any
     #     further details. For further assistance, reach out to the [Amazon
-    #     Web Services Support Center][3].
+    #     Web ServicesSupport Center][3].
     #
     #   * Stopped – This job was stopped by a user.
     #
@@ -7023,6 +7179,58 @@ module Aws::Bedrock
       class Unknown < RAGConfig; end
     end
 
+    # Defines the value and corresponding definition for one rating in a
+    # custom metric rating scale.
+    #
+    # @!attribute [rw] definition
+    #   Defines the definition for one rating in a custom metric rating
+    #   scale.
+    #   @return [String]
+    #
+    # @!attribute [rw] value
+    #   Defines the value for one rating in a custom metric rating scale.
+    #   @return [Types::RatingScaleItemValue]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/bedrock-2023-04-20/RatingScaleItem AWS API Documentation
+    #
+    class RatingScaleItem < Struct.new(
+      :definition,
+      :value)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
+    # Defines the value for one rating in a custom metric rating scale.
+    #
+    # @note RatingScaleItemValue is a union - when making an API calls you must set exactly one of the members.
+    #
+    # @note RatingScaleItemValue is a union - when returned from an API call exactly one value will be set and the returned type will be a subclass of RatingScaleItemValue corresponding to the set member.
+    #
+    # @!attribute [rw] string_value
+    #   A string representing the value for a rating in a custom metric
+    #   rating scale.
+    #   @return [String]
+    #
+    # @!attribute [rw] float_value
+    #   A floating point number representing the value for a rating in a
+    #   custom metric rating scale.
+    #   @return [Float]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/bedrock-2023-04-20/RatingScaleItemValue AWS API Documentation
+    #
+    class RatingScaleItemValue < Struct.new(
+      :string_value,
+      :float_value,
+      :unknown)
+      SENSITIVE = []
+      include Aws::Structure
+      include Aws::Structure::Union
+
+      class StringValue < RatingScaleItemValue; end
+      class FloatValue < RatingScaleItemValue; end
+      class Unknown < RatingScaleItemValue; end
+    end
+
     # @!attribute [rw] endpoint_identifier
     #   The ARN of the Amazon SageMaker endpoint you want to register with
     #   Amazon Bedrock Marketplace.

data/lib/aws-sdk-bedrock.rb

@@ -55,7 +55,7 @@ module Aws::Bedrock
   autoload :EndpointProvider, 'aws-sdk-bedrock/endpoint_provider'
   autoload :Endpoints, 'aws-sdk-bedrock/endpoints'
 
-  GEM_VERSION = '1.41.0'
+  GEM_VERSION = '1.42.0'
 
 end
 

data/sig/client.rbs

@@ -126,6 +126,32 @@ module Aws
                                              model_identifier: ::String
                                            },
                                          ]?
+                                       }?,
+                                       custom_metric_config: {
+                                         custom_metrics: Array[
+                                           {
+                                             custom_metric_definition: {
+                                               name: ::String,
+                                               instructions: ::String,
+                                               rating_scale: Array[
+                                                 {
+                                                   definition: ::String,
+                                                   value: {
+                                                     string_value: ::String?,
+                                                     float_value: ::Float?
+                                                   }
+                                                 },
+                                               ]?
+                                             }?
+                                           },
+                                         ],
+                                         evaluator_model_config: {
+                                           bedrock_evaluator_models: Array[
+                                             {
+                                               model_identifier: ::String
+                                             },
+                                           ]
+                                         }
                                        }?
                                      }?,
                                      human: {

data/sig/types.rbs

@@ -16,9 +16,27 @@ module Aws::Bedrock
     class AutomatedEvaluationConfig
       attr_accessor dataset_metric_configs: ::Array[Types::EvaluationDatasetMetricConfig]
       attr_accessor evaluator_model_config: Types::EvaluatorModelConfig
+      attr_accessor custom_metric_config: Types::AutomatedEvaluationCustomMetricConfig
       SENSITIVE: []
     end
 
+    class AutomatedEvaluationCustomMetricConfig
+      attr_accessor custom_metrics: ::Array[Types::AutomatedEvaluationCustomMetricSource]
+      attr_accessor evaluator_model_config: Types::CustomMetricEvaluatorModelConfig
+      SENSITIVE: []
+    end
+
+    class AutomatedEvaluationCustomMetricSource
+      attr_accessor custom_metric_definition: Types::CustomMetricDefinition
+      attr_accessor unknown: untyped
+      SENSITIVE: [:custom_metric_definition]
+
+      class CustomMetricDefinition < AutomatedEvaluationCustomMetricSource
+      end
+      class Unknown < AutomatedEvaluationCustomMetricSource
+      end
+    end
+
     class BatchDeleteEvaluationJobError
       attr_accessor job_identifier: ::String
       attr_accessor code: ::String
@@ -258,6 +276,23 @@ module Aws::Bedrock
       SENSITIVE: []
     end
 
+    class CustomMetricBedrockEvaluatorModel
+      attr_accessor model_identifier: ::String
+      SENSITIVE: []
+    end
+
+    class CustomMetricDefinition
+      attr_accessor name: ::String
+      attr_accessor instructions: ::String
+      attr_accessor rating_scale: ::Array[Types::RatingScaleItem]
+      SENSITIVE: [:name]
+    end
+
+    class CustomMetricEvaluatorModelConfig
+      attr_accessor bedrock_evaluator_models: ::Array[Types::CustomMetricBedrockEvaluatorModel]
+      SENSITIVE: []
+    end
+
     class CustomModelSummary
       attr_accessor model_arn: ::String
       attr_accessor model_name: ::String
@@ -508,6 +543,7 @@ module Aws::Bedrock
       attr_accessor model_identifiers: ::Array[::String]
       attr_accessor rag_identifiers: ::Array[::String]
       attr_accessor evaluator_model_identifiers: ::Array[::String]
+      attr_accessor custom_metrics_evaluator_model_identifiers: ::Array[::String]
       attr_accessor inference_config_summary: Types::EvaluationInferenceConfigSummary
       attr_accessor application_type: ("ModelEvaluation" | "RagEvaluation")
       SENSITIVE: []
@@ -1656,6 +1692,26 @@ module Aws::Bedrock
       end
     end
 
+    class RatingScaleItem
+      attr_accessor definition: ::String
+      attr_accessor value: Types::RatingScaleItemValue
+      SENSITIVE: []
+    end
+
+    class RatingScaleItemValue
+      attr_accessor string_value: ::String
+      attr_accessor float_value: ::Float
+      attr_accessor unknown: untyped
+      SENSITIVE: []
+
+      class StringValue < RatingScaleItemValue
+      end
+      class FloatValue < RatingScaleItemValue
+      end
+      class Unknown < RatingScaleItemValue
+      end
+    end
+
     class RegisterMarketplaceModelEndpointRequest
       attr_accessor endpoint_identifier: ::String
       attr_accessor model_source_identifier: ::String

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-bedrock
 version: !ruby/object:Gem::Version
-  version: 1.41.0
+  version: 1.42.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-07 00:00:00.000000000 Z
+date: 2025-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core