@backstage/plugin-events-backend-module-kafka 0.1.6-next.1 → 0.2.0

package/config.d.ts

@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 import { HumanDuration } from '@backstage/types';
 
 export interface Config {
@@ -25,179 +24,548 @@ export interface Config {
       kafka?: {
         /**
          * Configuration for KafkaConsumingEventPublisher
+         *
+         * Supports either:
+         * 1. Single configuration object (legacy format)
+         * 2. Multiple named instances as a record where each key is a unique name for the Kafka instance
          */
-        kafkaConsumingEventPublisher?: {
-          /**
-           * (Required) Client ID used by Backstage to identify when connecting to the Kafka cluster.
-           */
-          clientId: string;
-          /**
-           * (Required) List of brokers in the Kafka cluster to connect to.
-           */
-          brokers: string[];
-          /**
-           * Optional SSL connection parameters to connect to the cluster. Passed directly to Node tls.connect.
-           * See https://nodejs.org/dist/latest-v8.x/docs/api/tls.html#tls_tls_createsecurecontext_options
-           */
-          ssl?:
-            | {
-                ca?: string[];
+        kafkaConsumingEventPublisher?:
+          | {
+              /**
+               * (Required) Client ID used by Backstage to identify when connecting to the Kafka cluster.
+               */
+              clientId: string;
+              /**
+               * (Required) List of brokers in the Kafka cluster to connect to.
+               */
+              brokers: string[];
+              /**
+               * Optional SSL connection parameters to connect to the cluster. Passed directly to Node tls.connect.
+               * See https://nodejs.org/dist/latest-v8.x/docs/api/tls.html#tls_tls_createsecurecontext_options
+               */
+              ssl?:
+                | {
+                    ca?: string[];
+                    /** @visibility secret */
+                    key?: string;
+                    cert?: string;
+                    rejectUnauthorized?: boolean;
+                  }
+                | boolean;
+              /**
+               * Optional SASL connection parameters.
+               */
+              sasl?: {
+                mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
+                username: string;
                 /** @visibility secret */
-                key?: string;
-                cert?: string;
-                rejectUnauthorized?: boolean;
-              }
-            | boolean;
-          /**
-           * Optional SASL connection parameters.
-           */
-          sasl?: {
-            mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
-            username: string;
-            /** @visibility secret */
-            password: string;
-          };
+                password: string;
+              };
 
-          /**
-           * Optional retry connection parameters.
-           */
-          retry: {
-            /**
-             * (Optional) Maximum wait time for a retry
-             * Default: 30000 ms.
-             */
-            maxRetryTime: HumanDuration | string;
+              /**
+               * Optional retry connection parameters.
+               */
+              retry?: {
+                /**
+                 * (Optional) Maximum wait time for a retry
+                 * Default: 30000 ms.
+                 */
+                maxRetryTime?: HumanDuration | string;
 
-            /**
-             * (Optional) Initial value used to calculate the retry (This is still randomized following the randomization factor)
-             * Default: 300 ms.
-             */
-            initialRetryTime: HumanDuration | string;
+                /**
+                 * (Optional) Initial value used to calculate the retry (This is still randomized following the randomization factor)
+                 * Default: 300 ms.
+                 */
+                initialRetryTime?: HumanDuration | string;
 
-            /**
-             * (Optional) Randomization factor
-             * Default: 0.2.
-             */
-            factor: number;
+                /**
+                 * (Optional) Randomization factor
+                 * Default: 0.2.
+                 */
+                factor?: number;
 
-            /**
-             * (Optional) Exponential factor
-             * Default: 2.
-             */
-            multiplier: number;
+                /**
+                 * (Optional) Exponential factor
+                 * Default: 2.
+                 */
+                multiplier?: number;
 
-            /**
-             * (Optional) Max number of retries per call
-             * Default: 5.
-             */
-            retries: number;
-          };
+                /**
+                 * (Optional) Max number of retries per call
+                 * Default: 5.
+                 */
+                retries?: number;
+              };
 
-          /**
-           * (Optional) Timeout for authentication requests.
-           * Default: 10000 ms.
-           */
-          authenticationTimeout: HumanDuration | string;
-
-          /**
-           * (Optional) Time to wait for a successful connection.
-           * Default: 1000 ms.
-           */
-          connectionTimeout: HumanDuration | string;
-
-          /**
-           * (Optional) Time to wait for a successful request.
-           * Default: 30000 ms.
-           */
-          requestTimeout: HumanDuration | string;
-
-          /**
-           * (Optional) The request timeout can be disabled by setting enforceRequestTimeout to false.
-           * Default: true
-           */
-          enforceRequestTimeout: boolean;
-
-          /**
-           * Contains a object per topic for which an Kafka queue
-           * should be used as source of events.
-           */
-          topics: Array<{
-            /**
-             * (Required) The Backstage topic to publish to
-             */
-            topic: string;
-            /**
-             * (Required) KafkaConsumer-related configuration.
-             */
-            kafka: {
               /**
-               * (Required) The Kafka topics to subscribe to
+               * (Optional) Timeout for authentication requests.
+               * Default: 10000 ms.
                */
-              topics: string[];
+              authenticationTimeout?: HumanDuration | string;
+
               /**
-               * (Required) The GroupId to be used by the topic consumers
+               * (Optional) Time to wait for a successful connection.
+               * Default: 1000 ms.
                */
-              groupId: string;
+              connectionTimeout?: HumanDuration | string;
 
               /**
-               * (Optional) Timeout used to detect failures.
-               * The consumer sends periodic heartbeats to indicate its liveness to the broker.
-               * If no heartbeats are received by the broker before the expiration of this session timeout,
-               * then the broker will remove this consumer from the group and initiate a rebalance
+               * (Optional) Time to wait for a successful request.
                * Default: 30000 ms.
                */
-              sessionTimeout: HumanDuration | string;
+              requestTimeout?: HumanDuration | string;
 
               /**
-               * (Optional) The maximum time that the coordinator will wait for each member to rejoin when rebalancing the group
-               * Default: 60000 ms.
+               * (Optional) The request timeout can be disabled by setting enforceRequestTimeout to false.
+               * Default: true
                */
-              rebalanceTimeout: HumanDuration | string;
+              enforceRequestTimeout?: boolean;
 
               /**
-               * (Optional) The expected time between heartbeats to the consumer coordinator.
-               * Heartbeats are used to ensure that the consumer's session stays active.
-               * The value must be set lower than session timeout
-               * Default: 3000 ms.
+               * Contains an object per topic for which a Kafka queue
+               * should be used as source of events.
                */
-              heartbeatInterval: HumanDuration | string;
+              topics: Array<{
+                /**
+                 * (Required) The Backstage topic to publish to
+                 */
+                topic: string;
+                /**
+                 * (Required) KafkaConsumer-related configuration.
+                 */
+                kafka: {
+                  /**
+                   * (Required) The Kafka topics to subscribe to
+                   */
+                  topics: string[];
+                  /**
+                   * (Required) The GroupId to be used by the topic consumers
+                   */
+                  groupId: string;
+
+                  /**
+                   * (Optional) Timeout used to detect failures.
+                   * The consumer sends periodic heartbeats to indicate its liveness to the broker.
+                   * If no heartbeats are received by the broker before the expiration of this session timeout,
+                   * then the broker will remove this consumer from the group and initiate a rebalance
+                   * Default: 30000 ms.
+                   */
+                  sessionTimeout?: HumanDuration | string;
+
+                  /**
+                   * (Optional) The maximum time that the coordinator will wait for each member to rejoin when rebalancing the group
+                   * Default: 60000 ms.
+                   */
+                  rebalanceTimeout?: HumanDuration | string;
+
+                  /**
+                   * (Optional) The expected time between heartbeats to the consumer coordinator.
+                   * Heartbeats are used to ensure that the consumer's session stays active.
+                   * The value must be set lower than session timeout
+                   * Default: 3000 ms.
+                   */
+                  heartbeatInterval?: HumanDuration | string;
+
+                  /**
+                   * (Optional) The period of time after which we force a refresh of metadata
+                   * even if we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
+                   * Default: 300000 ms (5 minutes).
+                   */
+                  metadataMaxAge?: HumanDuration | string;
+
+                  /**
+                   * (Optional) The maximum amount of data per-partition the server will return.
+                   * This size must be at least as large as the maximum message size the server allows
+                   * or else it is possible for the producer to send messages larger than the consumer can fetch.
+                   * If that happens, the consumer can get stuck trying to fetch a large message on a certain partition
+                   * Default: 1048576 (1MB)
+                   */
+                  maxBytesPerPartition?: number;
+
+                  /**
+                   * (Optional) Minimum amount of data the server should return for a fetch request, otherwise wait up to maxWaitTime for more data to accumulate.
+                   * Default: 1
+                   */
+                  minBytes?: number;
+
+                  /**
+                   * (Optional) Maximum amount of bytes to accumulate in the response. Supported by Kafka >= 0.10.1.0
+                   * Default: 10485760 (10MB)
+                   */
+                  maxBytes?: number;
+
+                  /**
+                   * (Optional) The maximum amount of time the server will block before answering the fetch request
+                   * if there isn't sufficient data to immediately satisfy the requirement given by minBytes
+                   * Default: 5000
+                   */
+                  maxWaitTime?: HumanDuration | string;
+                };
+              }>;
+            }
+          | {
+              [name: string]: {
+                /**
+                 * (Required) Client ID used by Backstage to identify when connecting to the Kafka cluster.
+                 */
+                clientId: string;
+                /**
+                 * (Required) List of brokers in the Kafka cluster to connect to.
+                 */
+                brokers: string[];
+                /**
+                 * Optional SSL connection parameters to connect to the cluster. Passed directly to Node tls.connect.
+                 * See https://nodejs.org/dist/latest-v8.x/docs/api/tls.html#tls_tls_createsecurecontext_options
+                 */
+                ssl?:
+                  | {
+                      ca?: string[];
+                      /** @visibility secret */
+                      key?: string;
+                      cert?: string;
+                      rejectUnauthorized?: boolean;
+                    }
+                  | boolean;
+                /**
+                 * Optional SASL connection parameters.
+                 */
+                sasl?: {
+                  mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
+                  username: string;
+                  /** @visibility secret */
+                  password: string;
+                };
+
+                /**
+                 * Optional retry connection parameters.
+                 */
+                retry?: {
+                  /**
+                   * (Optional) Maximum wait time for a retry
+                   * Default: 30000 ms.
+                   */
+                  maxRetryTime?: HumanDuration | string;
+
+                  /**
+                   * (Optional) Initial value used to calculate the retry (This is still randomized following the randomization factor)
+                   * Default: 300 ms.
+                   */
+                  initialRetryTime?: HumanDuration | string;
+
+                  /**
+                   * (Optional) Randomization factor
+                   * Default: 0.2.
+                   */
+                  factor?: number;
+
+                  /**
+                   * (Optional) Exponential factor
+                   * Default: 2.
+                   */
+                  multiplier?: number;
+
+                  /**
+                   * (Optional) Max number of retries per call
+                   * Default: 5.
+                   */
+                  retries?: number;
+                };
+
+                /**
+                 * (Optional) Timeout for authentication requests.
+                 * Default: 10000 ms.
+                 */
+                authenticationTimeout?: HumanDuration | string;
+
+                /**
+                 * (Optional) Time to wait for a successful connection.
+                 * Default: 1000 ms.
+                 */
+                connectionTimeout?: HumanDuration | string;
+
+                /**
+                 * (Optional) Time to wait for a successful request.
+                 * Default: 30000 ms.
+                 */
+                requestTimeout?: HumanDuration | string;
+
+                /**
+                 * (Optional) The request timeout can be disabled by setting enforceRequestTimeout to false.
+                 * Default: true
+                 */
+                enforceRequestTimeout?: boolean;
+
+                /**
+                 * Contains an object per topic for which a Kafka queue
+                 * should be used as source of events.
+                 */
+                topics: Array<{
+                  /**
+                   * (Required) The Backstage topic to publish to
+                   */
+                  topic: string;
+                  /**
+                   * (Required) KafkaConsumer-related configuration.
+                   */
+                  kafka: {
+                    /**
+                     * (Required) The Kafka topics to subscribe to
+                     */
+                    topics: string[];
+                    /**
+                     * (Required) The GroupId to be used by the topic consumers
+                     */
+                    groupId: string;
+
+                    /**
+                     * (Optional) Timeout used to detect failures.
+                     * The consumer sends periodic heartbeats to indicate its liveness to the broker.
+                     * If no heartbeats are received by the broker before the expiration of this session timeout,
+                     * then the broker will remove this consumer from the group and initiate a rebalance
+                     * Default: 30000 ms.
+                     */
+                    sessionTimeout?: HumanDuration | string;
+
+                    /**
+                     * (Optional) The maximum time that the coordinator will wait for each member to rejoin when rebalancing the group
+                     * Default: 60000 ms.
+                     */
+                    rebalanceTimeout?: HumanDuration | string;
+
+                    /**
+                     * (Optional) The expected time between heartbeats to the consumer coordinator.
+                     * Heartbeats are used to ensure that the consumer's session stays active.
+                     * The value must be set lower than session timeout
+                     * Default: 3000 ms.
+                     */
+                    heartbeatInterval?: HumanDuration | string;
+
+                    /**
+                     * (Optional) The period of time after which we force a refresh of metadata
+                     * even if we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
+                     * Default: 300000 ms (5 minutes).
+                     */
+                    metadataMaxAge?: HumanDuration | string;
 
+                    /**
+                     * (Optional) The maximum amount of data per-partition the server will return.
+                     * This size must be at least as large as the maximum message size the server allows
+                     * or else it is possible for the producer to send messages larger than the consumer can fetch.
+                     * If that happens, the consumer can get stuck trying to fetch a large message on a certain partition
+                     * Default: 1048576 (1MB)
+                     */
+                    maxBytesPerPartition?: number;
+
+                    /**
+                     * (Optional) Minimum amount of data the server should return for a fetch request, otherwise wait up to maxWaitTime for more data to accumulate.
+                     * Default: 1
+                     */
+                    minBytes?: number;
+
+                    /**
+                     * (Optional) Maximum amount of bytes to accumulate in the response. Supported by Kafka >= 0.10.1.0
+                     * Default: 10485760 (10MB)
+                     */
+                    maxBytes?: number;
+
+                    /**
+                     * (Optional) The maximum amount of time the server will block before answering the fetch request
+                     * if there isn't sufficient data to immediately satisfy the requirement given by minBytes
+                     * Default: 5000
+                     */
+                    maxWaitTime?: HumanDuration | string;
+                  };
+                }>;
+              };
+            };
+
+        /**
+         * Configuration for KafkaPublishingEventConsumer
+         *
+         * Supports multiple named instances as a record where each key is a unique name
+         * for the Kafka producer configuration.
+         */
+        kafkaPublishingEventConsumer?: {
+          [name: string]: {
+            /**
+             * (Required) Client ID used by Backstage to identify when connecting to the Kafka cluster.
+             */
+            clientId: string;
+            /**
+             * (Required) List of brokers in the Kafka cluster to connect to.
+             */
+            brokers: string[];
+            /**
+             * Optional SSL connection parameters to connect to the cluster. Passed directly to Node tls.connect.
+             * See https://nodejs.org/dist/latest-v8.x/docs/api/tls.html#tls_tls_createsecurecontext_options
+             */
+            ssl?:
+              | {
+                  ca?: string[];
+                  /** @visibility secret */
+                  key?: string;
+                  cert?: string;
+                  rejectUnauthorized?: boolean;
+                }
+              | boolean;
+            /**
+             * Optional SASL connection parameters.
+             */
+            sasl?: {
+              mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
+              username: string;
+              /** @visibility secret */
+              password: string;
+            };
+
+            /**
+             * Optional retry connection parameters.
+             */
+            retry?: {
               /**
-               * (Optional) The period of time after which we force a refresh of metadata
-               * even if we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
-               * Default: 300000 ms (5 minutes).
+               * (Optional) Maximum wait time for a retry
+               * Default: 30000 ms.
                */
-              metadataMaxAge: HumanDuration | string;
+              maxRetryTime?: HumanDuration | string;
 
               /**
-               * (Optional) The maximum amount of data per-partition the server will return.
-               * This size must be at least as large as the maximum message size the server allows
-               * or else it is possible for the producer to send messages larger than the consumer can fetch.
-               * If that happens, the consumer can get stuck trying to fetch a large message on a certain partition
-               * Default: 1048576 (1MB)
+               * (Optional) Initial value used to calculate the retry (This is still randomized following the randomization factor)
+               * Default: 300 ms.
                */
-              maxBytesPerPartition: number;
+              initialRetryTime?: HumanDuration | string;
 
               /**
-               * (Optional) Minimum amount of data the server should return for a fetch request, otherwise wait up to maxWaitTime for more data to accumulate.
-               * Default: 1
+               * (Optional) Randomization factor
+               * Default: 0.2.
                */
-              minBytes: number;
+              factor?: number;
 
               /**
-               * (Optional) Maximum amount of bytes to accumulate in the response. Supported by Kafka >= 0.10.1.0
-               * Default: 10485760 (10MB)
+               * (Optional) Exponential factor
+               * Default: 2.
                */
-              maxBytes: number;
+              multiplier?: number;
 
               /**
-               * (Optional) The maximum amount of time the server will block before answering the fetch request
-               * if there isn’t sufficient data to immediately satisfy the requirement given by minBytes
-               * Default: 5000
+               * (Optional) Max number of retries per call
+               * Default: 5.
                */
-              maxWaitTime: HumanDuration | string;
+              retries?: number;
             };
-          }>;
+
+            /**
+             * (Optional) Timeout for authentication requests.
+             * Default: 10000 ms.
+             */
+            authenticationTimeout?: HumanDuration | string;
+
+            /**
+             * (Optional) Time to wait for a successful connection.
+             * Default: 1000 ms.
+             */
+            connectionTimeout?: HumanDuration | string;
+
+            /**
+             * (Optional) Time to wait for a successful request.
+             * Default: 30000 ms.
+             */
+            requestTimeout?: HumanDuration | string;
+
+            /**
+             * (Optional) The request timeout can be disabled by setting enforceRequestTimeout to false.
+             * Default: true
+             */
+            enforceRequestTimeout?: boolean;
+
+            /**
+             * Contains an object per topic for which a Kafka queue
+             * should be used as destination for events.
+             */
+            topics: Array<{
+              /**
+               * (Required) The Backstage topic to consume from
+               */
+              topic: string;
+              /**
+               * (Required) KafkaProducer-related configuration.
+               */
+              kafka: {
+                /**
+                 * (Required) The Kafka topic to publish to
+                 */
+                topic: string;
+
+                /**
+                 * (Optional) Allow topic creation when querying metadata for non-existent topics.
+                 * Default: true
+                 */
+                allowAutoTopicCreation?: boolean;
+
+                /**
+                 * (Optional) The period of time after which we force a refresh of metadata
+                 * even if we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
+                 * Default: 300000 ms (5 minutes).
+                 */
+                metadataMaxAge?: HumanDuration | string;
+
+                /**
+                 * (Optional) The maximum amount of time in ms that the transaction coordinator will wait for a transaction status update
+                 * from the producer before proactively aborting the ongoing transaction.
+                 * If this value is larger than the `transaction.max.timeout.ms`` setting in the broker, the request will fail with a `InvalidTransactionTimeout` error
+                 * Default: 60000 ms.
+                 */
+                transactionTimeout?: HumanDuration | string;
+
+                /**
+                 * (Optional) Experimental. If enabled producer will ensure each message is written exactly once. Acks must be set to -1 ("all").
+                 * Retries will default to MAX_SAFE_INTEGER.
+                 * Default: false.
+                 */
+                idempotent?: boolean;
+
+                /**
+                 * (Optional) Max number of requests that may be in progress at any time. If falsey then no limit.
+                 * Default: null.
+                 */
+                maxInFlightRequests?: number;
+
+                /**
+                 * Optional retry connection parameters.
+                 */
+                retry?: {
+                  /**
+                   * (Optional) Maximum wait time for a retry
+                   * Default: 30000 ms.
+                   */
+                  maxRetryTime?: HumanDuration | string;
+
+                  /**
+                   * (Optional) Initial value used to calculate the retry (This is still randomized following the randomization factor)
+                   * Default: 300 ms.
+                   */
+                  initialRetryTime?: HumanDuration | string;
+
+                  /**
+                   * (Optional) Randomization factor
+                   * Default: 0.2.
+                   */
+                  factor?: number;
+
+                  /**
+                   * (Optional) Exponential factor
+                   * Default: 2.
+                   */
+                  multiplier?: number;
+
+                  /**
+                   * (Optional) Max number of retries per call
+                   * Default: 5.
+                   */
+                  retries?: number;
+                };
+              };
+            }>;
+          };
         };
       };
     };