@google-cloud/discoveryengine 1.4.0 → 1.5.0

package/build/protos/google/cloud/discoveryengine/v1beta/document_processing_config.proto

@@ -0,0 +1,94 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.discoveryengine.v1beta;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+
+option csharp_namespace = "Google.Cloud.DiscoveryEngine.V1Beta";
+option go_package = "cloud.google.com/go/discoveryengine/apiv1beta/discoveryenginepb;discoveryenginepb";
+option java_multiple_files = true;
+option java_outer_classname = "DocumentProcessingConfigProto";
+option java_package = "com.google.cloud.discoveryengine.v1beta";
+option objc_class_prefix = "DISCOVERYENGINE";
+option php_namespace = "Google\\Cloud\\DiscoveryEngine\\V1beta";
+option ruby_package = "Google::Cloud::DiscoveryEngine::V1beta";
+
+// A singleton resource of
+// [DataStore][google.cloud.discoveryengine.v1beta.DataStore]. It's empty when
+// [DataStore][google.cloud.discoveryengine.v1beta.DataStore] is created, which
+// defaults to digital parser. The first call to
+// [DataStoreService.UpdateDocumentProcessingConfig][] method will initialize
+// the config.
+message DocumentProcessingConfig {
+  option (google.api.resource) = {
+    type: "discoveryengine.googleapis.com/DocumentProcessingConfig"
+    pattern: "projects/{project}/locations/{location}/dataStores/{data_store}/documentProcessingConfig"
+    pattern: "projects/{project}/locations/{location}/collections/{collection}/dataStores/{data_store}/documentProcessingConfig"
+  };
+
+  // Related configurations applied to a specific type of document parser.
+  message ParsingConfig {
+    // The digital parsing configurations for documents.
+    message DigitalParsingConfig {}
+
+    // The OCR parsing configurations for documents.
+    message OcrParsingConfig {
+      // Apply additional enhanced OCR processing to a list of document
+      // elements.
+      //
+      // Supported values:
+      // * `table`: advanced table parsing model.
+      repeated string enhanced_document_elements = 1;
+
+      // If true, will use native text instead of OCR text on pages containing
+      // native text.
+      bool use_native_text = 2;
+    }
+
+    // Configs for document processing types.
+    oneof type_dedicated_config {
+      // Configurations applied to digital parser.
+      DigitalParsingConfig digital_parsing_config = 1;
+
+      // Configurations applied to OCR parser. Currently it only applies to
+      // PDFs.
+      OcrParsingConfig ocr_parsing_config = 2;
+    }
+  }
+
+  // The full resource name of the Document Processing Config.
+  // Format:
+  // `projects/*/locations/*/collections/*/dataStores/*/documentProcessingConfig`.
+  string name = 1;
+
+  // Configurations for default Document parser.
+  // If not specified, we will configure it as default DigitalParsingConfig, and
+  // the default parsing config will be applied to all file types for Document
+  // parsing.
+  ParsingConfig default_parsing_config = 4;
+
+  // Map from file type to override the default parsing configuration based on
+  // the file type. Supported keys:
+  // * `pdf`: Override parsing config for PDF files, either digital parsing, ocr
+  // parsing or layout parsing is supported.
+  // * `html`: Override parsing config for HTML files, only digital parsing and
+  // or layout parsing are supported.
+  // * `docx`: Override parsing config for DOCX files, only digital parsing and
+  // or layout parsing are supported.
+  map<string, ParsingConfig> parsing_config_overrides = 5;
+}

package/build/protos/google/cloud/discoveryengine/v1beta/purge_config.proto

@@ -84,6 +84,9 @@ message PurgeDocumentsMetadata {
 
   // Count of entries that encountered errors while processing.
   int64 failure_count = 4;
+
+  // Count of entries that were ignored as entries were not found.
+  int64 ignored_count = 5;
 }
 
 // Request message for

package/build/protos/google/cloud/discoveryengine/v1beta/search_service.proto

@@ -69,6 +69,19 @@ message SearchRequest {
     }
   }
 
+  // A struct to define data stores to filter on in a search call.
+  message DataStoreSpec {
+    // Required. Full resource name of
+    // [DataStore][google.cloud.discoveryengine.v1beta.DataStore], such as
+    // `projects/{project}/locations/{location}/collections/{collection_id}/dataStores/{data_store_id}`.
+    string data_store = 1 [
+      (google.api.field_behavior) = REQUIRED,
+      (google.api.resource_reference) = {
+        type: "discoveryengine.googleapis.com/DataStore"
+      }
+    ];
+  }
+
   // A facet specification to perform faceted search.
   message FacetSpec {
     // Specifies how a facet is computed.
@@ -212,9 +225,8 @@ message SearchRequest {
       // Examples:
       //
       // * To boost documents with document ID "doc_1" or "doc_2", and
-      // color
-      //   "Red" or "Blue":
-      //     * (id: ANY("doc_1", "doc_2")) AND (color: ANY("Red","Blue"))
+      // color "Red" or "Blue":
+      // `(document_id: ANY("doc_1", "doc_2")) AND (color: ANY("Red", "Blue"))`
       string condition = 1;
 
       // Strength of the condition boost, which should be in [-1, 1]. Negative
@@ -347,7 +359,7 @@ message SearchRequest {
       // of results returned is less than `summaryResultCount`, the summary is
       // generated from all of the results.
       //
-      // At most five results can be used to generate a summary.
+      // At most 10 results can be used to generate a summary.
       int32 summary_result_count = 1;
 
       // Specifies whether to include citations in the summary. The default
@@ -547,6 +559,9 @@ message SearchRequest {
   // If this field is negative, an  `INVALID_ARGUMENT`  is returned.
   int32 offset = 6;
 
+  // A list of data store specs to apply on a search call.
+  repeated DataStoreSpec data_store_specs = 32;
+
   // The filter syntax consists of an expression language for constructing a
   // predicate from one or more fields of the documents being filtered. Filter
   // expression is case-sensitive.

package/build/protos/google/cloud/discoveryengine/v1beta/search_tuning_service.proto

@@ -0,0 +1,151 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.discoveryengine.v1beta;
+
+import "google/api/annotations.proto";
+import "google/api/client.proto";
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/cloud/discoveryengine/v1beta/import_config.proto";
+import "google/longrunning/operations.proto";
+import "google/protobuf/timestamp.proto";
+import "google/rpc/status.proto";
+
+option csharp_namespace = "Google.Cloud.DiscoveryEngine.V1Beta";
+option go_package = "cloud.google.com/go/discoveryengine/apiv1beta/discoveryenginepb;discoveryenginepb";
+option java_multiple_files = true;
+option java_outer_classname = "SearchTuningServiceProto";
+option java_package = "com.google.cloud.discoveryengine.v1beta";
+option objc_class_prefix = "DISCOVERYENGINE";
+option php_namespace = "Google\\Cloud\\DiscoveryEngine\\V1beta";
+option ruby_package = "Google::Cloud::DiscoveryEngine::V1beta";
+
+// Service for search tuning.
+service SearchTuningService {
+  option (google.api.default_host) = "discoveryengine.googleapis.com";
+  option (google.api.oauth_scopes) =
+      "https://www.googleapis.com/auth/cloud-platform";
+
+  // Trains a custom model.
+  rpc TrainCustomModel(TrainCustomModelRequest)
+      returns (google.longrunning.Operation) {
+    option (google.api.http) = {
+      post: "/v1beta/{data_store=projects/*/locations/*/collections/*/dataStores/*}:trainCustomModel"
+      body: "*"
+    };
+    option (google.longrunning.operation_info) = {
+      response_type: "google.cloud.discoveryengine.v1beta.TrainCustomModelResponse"
+      metadata_type: "google.cloud.discoveryengine.v1beta.TrainCustomModelMetadata"
+    };
+  }
+}
+
+// Request message for
+// [SearchTuningService.TrainCustomModel][google.cloud.discoveryengine.v1beta.SearchTuningService.TrainCustomModel]
+// method.
+message TrainCustomModelRequest {
+  // Cloud Storage training data input.
+  message GcsTrainingInput {
+    // The Cloud Storage corpus data which could be associated in train data.
+    // The data path format is `gs://<bucket_to_data>/<jsonl_file_name>`.
+    // A newline delimited jsonl/ndjson file.
+    //
+    // For search-tuning model, each line should have the _id, title
+    // and text. Example:
+    // `{"_id": "doc1", title: "relevant doc", "text": "relevant text"}`
+    string corpus_data_path = 1;
+
+    // The gcs query data which could be associated in train data.
+    // The data path format is `gs://<bucket_to_data>/<jsonl_file_name>`.
+    // A newline delimited jsonl/ndjson file.
+    //
+    // For search-tuning model, each line should have the _id
+    // and text. Example: {"_id": "query1",  "text": "example query"}
+    string query_data_path = 2;
+
+    // Cloud Storage training data path whose format should be
+    // `gs://<bucket_to_data>/<tsv_file_name>`. The file should be in tsv
+    // format. Each line should have the doc_id and query_id and score (number).
+    //
+    // For search-tuning model, it should have the query-id corpus-id
+    // score as tsv file header. The score should be a number in `[0, inf+)`.
+    // The larger the number is, the more relevant the pair is. Example:
+    //
+    // * `query-id\tcorpus-id\tscore`
+    // * `query1\tdoc1\t1`
+    string train_data_path = 3;
+
+    // Cloud Storage test data. Same format as train_data_path. If not provided,
+    // a random 80/20 train/test split will be performed on train_data_path.
+    string test_data_path = 4;
+  }
+
+  // Model training input.
+  oneof training_input {
+    // Cloud Storage training input.
+    GcsTrainingInput gcs_training_input = 2;
+  }
+
+  // Required. The resource name of the Data Store, such as
+  // `projects/*/locations/global/collections/default_collection/dataStores/default_data_store`.
+  // This field is used to identify the data store where to train the models.
+  string data_store = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {
+      type: "discoveryengine.googleapis.com/DataStore"
+    }
+  ];
+
+  // Model to be trained. Supported values are:
+  //
+  //  * **search-tuning**: Fine tuning the search system based on data provided.
+  string model_type = 3;
+
+  // The desired location of errors incurred during the data ingestion and
+  // training.
+  ImportErrorConfig error_config = 4;
+}
+
+// Response of the
+// [TrainCustomModelRequest][google.cloud.discoveryengine.v1beta.TrainCustomModelRequest].
+// This message is returned by the google.longrunning.Operations.response field.
+message TrainCustomModelResponse {
+  // A sample of errors encountered while processing the data.
+  repeated google.rpc.Status error_samples = 1;
+
+  // Echoes the destination for the complete errors in the request if set.
+  ImportErrorConfig error_config = 2;
+
+  // The trained model status. Possible values are:
+  //
+  //  * **bad-data**: The training data quality is bad.
+  //  * **no-improvement**: Tuning didn't improve performance. Won't deploy.
+  //  * **in-progress**: Model training is in progress.
+  //  * **ready**: The model is ready for serving.
+  string model_status = 3;
+}
+
+// Metadata related to the progress of the TrainCustomModel operation. This is
+// returned by the google.longrunning.Operation.metadata field.
+message TrainCustomModelMetadata {
+  // Operation create time.
+  google.protobuf.Timestamp create_time = 1;
+
+  // Operation last update time. If the operation is done, this is also the
+  // finish time.
+  google.protobuf.Timestamp update_time = 2;
+}

package/build/protos/google/cloud/discoveryengine/v1beta/serving_config.proto

@@ -109,7 +109,7 @@ message ServingConfig {
   }
 
   // Immutable. Fully qualified name
-  // `projects/{project}/locations/{location}/collections/{collection_id}/dataStores/{data_store_id}/servingConfigs/{serving_config_id}`
+  // `projects/{project}/locations/{location}/collections/{collection_id}/engines/{engine_id}/servingConfigs/{serving_config_id}`
   string name = 1 [(google.api.field_behavior) = IMMUTABLE];
 
   // Required. The human readable serving config display name. Used in Discovery
@@ -165,8 +165,9 @@ message ServingConfig {
   // The ranking expression controls the customized ranking on retrieval
   // documents. To leverage this, document embedding is required. The ranking
   // expression setting in ServingConfig applies to all search requests served
-  // by the serving config. However, if [SearchRequest.ranking_expression][] is
-  // specified, it overrides the ServingConfig ranking expression.
+  // by the serving config. However, if
+  // [SearchRequest.ranking_expression][google.cloud.discoveryengine.v1beta.SearchRequest.ranking_expression]
+  // is specified, it overrides the ServingConfig ranking expression.
   //
   // The ranking expression is a single function or multiple functions that are
   // joined by "+".

package/build/protos/google/cloud/discoveryengine/v1beta/serving_config_service.proto

@@ -32,7 +32,8 @@ option objc_class_prefix = "DISCOVERYENGINE";
 option php_namespace = "Google\\Cloud\\DiscoveryEngine\\V1beta";
 option ruby_package = "Google::Cloud::DiscoveryEngine::V1beta";
 
-// Service for modifying ServingConfig.
+// Service for operations related to
+// [ServingConfig][google.cloud.discoveryengine.v1beta.ServingConfig].
 service ServingConfigService {
   option (google.api.default_host) = "discoveryengine.googleapis.com";
   option (google.api.oauth_scopes) =
@@ -107,7 +108,7 @@ message UpdateServingConfigRequest {
 // Request for GetServingConfig method.
 message GetServingConfigRequest {
   // Required. The resource name of the ServingConfig to get. Format:
-  // `projects/{project_number}/locations/{location}/collections/{collection}/dataStores/{data_store}/servingConfigs/{serving_config_id}`
+  // `projects/{project_number}/locations/{location}/collections/{collection}/engines/{engine}/servingConfigs/{serving_config_id}`
   string name = 1 [
     (google.api.field_behavior) = REQUIRED,
     (google.api.resource_reference) = {
@@ -118,12 +119,12 @@ message GetServingConfigRequest {
 
 // Request for ListServingConfigs method.
 message ListServingConfigsRequest {
-  // Required. The dataStore resource name. Format:
-  // `projects/{project_number}/locations/{location}/collections/{collection}/dataStores/{data_store}`
+  // Required. Full resource name of the parent resource. Format:
+  // `projects/{project_number}/locations/{location}/collections/{collection}/engines/{engine}`
   string parent = 1 [
     (google.api.field_behavior) = REQUIRED,
     (google.api.resource_reference) = {
-      type: "discoveryengine.googleapis.com/DataStore"
+      child_type: "discoveryengine.googleapis.com/ServingConfig"
     }
   ];