@google-cloud/pubsub 4.8.0 → 4.10.0

package/CHANGELOG.md

@@ -4,6 +4,30 @@
 
 [1]: https://www.npmjs.com/package/@google-cloud/pubsub?activeTab=versions
 
+## [4.10.0](https://github.com/googleapis/nodejs-pubsub/compare/v4.9.0...v4.10.0) (2025-01-13)
+
+
+### Features
+
+* Add Kafka-based sources to IngestionDataSourceSettings proto and IngestionFailureEvent proto ([#2007](https://github.com/googleapis/nodejs-pubsub/issues/2007)) ([08d00a1](https://github.com/googleapis/nodejs-pubsub/commit/08d00a1e7d30e2ab47f7582ff61b1eb73ad9d25e))
+
+
+### Bug Fixes
+
+* **deps:** Update dependency @opentelemetry/semantic-conventions to ~1.28.0 ([#2001](https://github.com/googleapis/nodejs-pubsub/issues/2001)) ([c8e5811](https://github.com/googleapis/nodejs-pubsub/commit/c8e581124523edcac13d6d6c48d67342482959f3))
+
+## [4.9.0](https://github.com/googleapis/nodejs-pubsub/compare/v4.8.0...v4.9.0) (2024-11-12)
+
+
+### Features
+
+* Add IngestionFailureEvent to the external proto ([#1984](https://github.com/googleapis/nodejs-pubsub/issues/1984)) ([7075430](https://github.com/googleapis/nodejs-pubsub/commit/70754309fb600c54d0a573f2d49ad4c419577550))
+
+
+### Bug Fixes
+
+* KiB, not MiB for ack size limits ([#1999](https://github.com/googleapis/nodejs-pubsub/issues/1999)) ([798270d](https://github.com/googleapis/nodejs-pubsub/commit/798270db9c5ef71f75c3e24e70d9592bbd068212))
+
 ## [4.8.0](https://github.com/googleapis/nodejs-pubsub/compare/v4.7.2...v4.8.0) (2024-10-15)
 
 

package/build/protos/google/pubsub/v1/pubsub.proto

@@ -323,6 +323,164 @@ message IngestionDataSourceSettings {
     string match_glob = 9 [(google.api.field_behavior) = OPTIONAL];
   }
 
+  // Ingestion settings for Azure Event Hubs.
+  message AzureEventHubs {
+    // Possible states for managed ingestion from Event Hubs.
+    enum State {
+      // Default value. This value is unused.
+      STATE_UNSPECIFIED = 0;
+
+      // Ingestion is active.
+      ACTIVE = 1;
+
+      // Permission denied encountered while consuming data from Event Hubs.
+      // This can happen when `client_id`, or `tenant_id` are invalid. Or the
+      // right permissions haven't been granted.
+      EVENT_HUBS_PERMISSION_DENIED = 2;
+
+      // Permission denied encountered while publishing to the topic.
+      PUBLISH_PERMISSION_DENIED = 3;
+
+      // The provided Event Hubs namespace couldn't be found.
+      NAMESPACE_NOT_FOUND = 4;
+
+      // The provided Event Hub couldn't be found.
+      EVENT_HUB_NOT_FOUND = 5;
+
+      // The provided Event Hubs subscription couldn't be found.
+      SUBSCRIPTION_NOT_FOUND = 6;
+
+      // The provided Event Hubs resource group couldn't be found.
+      RESOURCE_GROUP_NOT_FOUND = 7;
+    }
+
+    // Output only. An output-only field that indicates the state of the Event
+    // Hubs ingestion source.
+    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Optional. Name of the resource group within the azure subscription.
+    string resource_group = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the Event Hubs namespace.
+    string namespace = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the Event Hub.
+    string event_hub = 4 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The client id of the Azure application that is being used to
+    // authenticate Pub/Sub.
+    string client_id = 5 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The tenant id of the Azure application that is being used to
+    // authenticate Pub/Sub.
+    string tenant_id = 6 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The Azure subscription id.
+    string subscription_id = 7 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The GCP service account to be used for Federated Identity
+    // authentication.
+    string gcp_service_account = 8 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // Ingestion settings for Amazon MSK.
+  message AwsMsk {
+    // Possible states for managed ingestion from Amazon MSK.
+    enum State {
+      // Default value. This value is unused.
+      STATE_UNSPECIFIED = 0;
+
+      // Ingestion is active.
+      ACTIVE = 1;
+
+      // Permission denied encountered while consuming data from Amazon MSK.
+      MSK_PERMISSION_DENIED = 2;
+
+      // Permission denied encountered while publishing to the topic.
+      PUBLISH_PERMISSION_DENIED = 3;
+
+      // The provided MSK cluster wasn't found.
+      CLUSTER_NOT_FOUND = 4;
+
+      // The provided topic wasn't found.
+      TOPIC_NOT_FOUND = 5;
+    }
+
+    // Output only. An output-only field that indicates the state of the Amazon
+    // MSK ingestion source.
+    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Required. The Amazon Resource Name (ARN) that uniquely identifies the
+    // cluster.
+    string cluster_arn = 2 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The name of the topic in the Amazon MSK cluster that Pub/Sub
+    // will import from.
+    string topic = 3 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. AWS role ARN to be used for Federated Identity authentication
+    // with Amazon MSK. Check the Pub/Sub docs for how to set up this role and
+    // the required permissions that need to be attached to it.
+    string aws_role_arn = 4 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The GCP service account to be used for Federated Identity
+    // authentication with Amazon MSK (via a `AssumeRoleWithWebIdentity` call
+    // for the provided role). The `aws_role_arn` must be set up with
+    // `accounts.google.com:sub` equals to this service account number.
+    string gcp_service_account = 5 [(google.api.field_behavior) = REQUIRED];
+  }
+
+  // Ingestion settings for Confluent Cloud.
+  message ConfluentCloud {
+    // Possible states for managed ingestion from Confluent Cloud.
+    enum State {
+      // Default value. This value is unused.
+      STATE_UNSPECIFIED = 0;
+
+      // Ingestion is active.
+      ACTIVE = 1;
+
+      // Permission denied encountered while consuming data from Confluent
+      // Cloud.
+      CONFLUENT_CLOUD_PERMISSION_DENIED = 2;
+
+      // Permission denied encountered while publishing to the topic.
+      PUBLISH_PERMISSION_DENIED = 3;
+
+      // The provided bootstrap server address is unreachable.
+      UNREACHABLE_BOOTSTRAP_SERVER = 4;
+
+      // The provided cluster wasn't found.
+      CLUSTER_NOT_FOUND = 5;
+
+      // The provided topic wasn't found.
+      TOPIC_NOT_FOUND = 6;
+    }
+
+    // Output only. An output-only field that indicates the state of the
+    // Confluent Cloud ingestion source.
+    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Required. The address of the bootstrap server. The format is url:port.
+    string bootstrap_server = 2 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The id of the cluster.
+    string cluster_id = 3 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The name of the topic in the Confluent Cloud cluster that
+    // Pub/Sub will import from.
+    string topic = 4 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The id of the identity pool to be used for Federated Identity
+    // authentication with Confluent Cloud. See
+    // https://docs.confluent.io/cloud/current/security/authenticate/workload-identities/identity-providers/oauth/identity-pools.html#add-oauth-identity-pools.
+    string identity_pool_id = 5 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The GCP service account to be used for Federated Identity
+    // authentication with `identity_pool_id`.
+    string gcp_service_account = 6 [(google.api.field_behavior) = REQUIRED];
+  }
+
   // Only one source type can have settings set.
   oneof source {
     // Optional. Amazon Kinesis Data Streams.
@@ -330,6 +488,16 @@ message IngestionDataSourceSettings {
 
     // Optional. Cloud Storage.
     CloudStorage cloud_storage = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Azure Event Hubs.
+    AzureEventHubs azure_event_hubs = 3
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Amazon MSK.
+    AwsMsk aws_msk = 5 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Confluent Cloud.
+    ConfluentCloud confluent_cloud = 6 [(google.api.field_behavior) = OPTIONAL];
   }
 
   // Optional. Platform Logs settings. If unset, no Platform Logs will be
@@ -365,6 +533,147 @@ message PlatformLogsSettings {
   Severity severity = 1 [(google.api.field_behavior) = OPTIONAL];
 }
 
+// Payload of the Platform Log entry sent when a failure is encountered while
+// ingesting.
+message IngestionFailureEvent {
+  // Specifies the reason why some data may have been left out of
+  // the desired Pub/Sub message due to the API message limits
+  // (https://cloud.google.com/pubsub/quotas#resource_limits). For example,
+  // when the number of attributes is larger than 100, the number of
+  // attributes is truncated to 100 to respect the limit on the attribute count.
+  // Other attribute limits are treated similarly. When the size of the desired
+  // message would've been larger than 10MB, the message won't be published at
+  // all, and ingestion of the subsequent messages will proceed as normal.
+  message ApiViolationReason {}
+
+  // Set when an Avro file is unsupported or its format is not valid. When this
+  // occurs, one or more Avro objects won't be ingested.
+  message AvroFailureReason {}
+
+  // Failure when ingesting from a Cloud Storage source.
+  message CloudStorageFailure {
+    // Optional. Name of the Cloud Storage bucket used for ingestion.
+    string bucket = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Name of the Cloud Storage object which contained the section
+    // that couldn't be ingested.
+    string object_name = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Generation of the Cloud Storage object which contained the
+    // section that couldn't be ingested.
+    int64 object_generation = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Reason why ingestion failed for the specified object.
+    oneof reason {
+      // Optional. Failure encountered when parsing an Avro file.
+      AvroFailureReason avro_failure_reason = 5
+          [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. The Pub/Sub API limits prevented the desired message from
+      // being published.
+      ApiViolationReason api_violation_reason = 6
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+  }
+
+  // Failure when ingesting from an Amazon MSK source.
+  message AwsMskFailureReason {
+    // Optional. The ARN of the cluster of the topic being ingested from.
+    string cluster_arn = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the Kafka topic being ingested from.
+    string kafka_topic = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The partition ID of the message that failed to be ingested.
+    int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The offset within the partition of the message that failed to
+    // be ingested.
+    int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
+
+    // Reason why ingestion failed for the specified message.
+    oneof reason {
+      // Optional. The Pub/Sub API limits prevented the desired message from
+      // being published.
+      ApiViolationReason api_violation_reason = 5
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+  }
+
+  // Failure when ingesting from an Azure Event Hubs source.
+  message AzureEventHubsFailureReason {
+    // Optional. The namespace containing the event hub being ingested from.
+    string namespace = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the event hub being ingested from.
+    string event_hub = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The partition ID of the message that failed to be ingested.
+    int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The offset within the partition of the message that failed to
+    // be ingested.
+    int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
+
+    // Reason why ingestion failed for the specified message.
+    oneof reason {
+      // Optional. The Pub/Sub API limits prevented the desired message from
+      // being published.
+      ApiViolationReason api_violation_reason = 5
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+  }
+
+  // Failure when ingesting from a Confluent Cloud source.
+  message ConfluentCloudFailureReason {
+    // Optional. The cluster ID containing the topic being ingested from.
+    string cluster_id = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the Kafka topic being ingested from.
+    string kafka_topic = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The partition ID of the message that failed to be ingested.
+    int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The offset within the partition of the message that failed to
+    // be ingested.
+    int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
+
+    // Reason why ingestion failed for the specified message.
+    oneof reason {
+      // Optional. The Pub/Sub API limits prevented the desired message from
+      // being published.
+      ApiViolationReason api_violation_reason = 5
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+  }
+
+  // Required. Name of the import topic. Format is:
+  // projects/{project_name}/topics/{topic_name}.
+  string topic = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. Error details explaining why ingestion to Pub/Sub has failed.
+  string error_message = 2 [(google.api.field_behavior) = REQUIRED];
+
+  oneof failure {
+    // Optional. Failure when ingesting from Cloud Storage.
+    CloudStorageFailure cloud_storage_failure = 3
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Failure when ingesting from Amazon MSK.
+    AwsMskFailureReason aws_msk_failure = 4
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Failure when ingesting from Azure Event Hubs.
+    AzureEventHubsFailureReason azure_event_hubs_failure = 5
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Failure when ingesting from Confluent Cloud.
+    ConfluentCloudFailureReason confluent_cloud_failure = 6
+        [(google.api.field_behavior) = OPTIONAL];
+  }
+}
+
 // A topic resource.
 message Topic {
   option (google.api.resource) = {