@backstage/plugin-events-backend-module-kafka 0.0.0-nightly-20251216024726 → 0.0.0-nightly-20260108025012

package/CHANGELOG.md

@@ -1,6 +1,20 @@
 # @backstage/plugin-events-backend-module-kafka
 
-## 0.0.0-nightly-20251216024726
+## 0.0.0-nightly-20260108025012
+
+### Minor Changes
+
+- ef5bbd8: Add support for Kafka offset configuration (`fromBeginning`) and `autoCommit`
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/backend-plugin-api@1.6.0
+  - @backstage/config@1.3.6
+  - @backstage/types@1.2.2
+  - @backstage/plugin-events-node@0.4.18
+
+## 0.2.0
 
 ### Minor Changes
 
@@ -12,10 +26,8 @@
 ### Patch Changes
 
 - Updated dependencies
-  - @backstage/plugin-events-node@0.0.0-nightly-20251216024726
-  - @backstage/backend-plugin-api@0.0.0-nightly-20251216024726
-  - @backstage/config@1.3.6
-  - @backstage/types@1.2.2
+  - @backstage/plugin-events-node@0.4.18
+  - @backstage/backend-plugin-api@1.6.0
 
 ## 0.1.6-next.1
 

package/README.md

@@ -33,6 +33,10 @@ events:
                 topics: # (Required) The Kafka topics to subscribe to.
                   - topic1
                 groupId: your-group-id # (Required) The GroupId to be used by the topic consumers.
+                # Optional offset management settings (these can be omitted to use defaults):
+                # fromBeginning: false # Start from earliest offset when no committed offset exists. Default: not set (latest)
+                # autoCommit: true # Enable auto-commit. Default: true (for backward compatibility)
+                # pauseOnError: false # Pause consumer on error. Default: false (for backward compatibility)
 ```
 
 ### KafkaPublishingEventConsumer Configuration
@@ -57,6 +61,77 @@ events:
 
 For a complete list of all available fields that can be configured, refer to the [config.d.ts file](./config.d.ts).
 
+### Offset Management
+
+The plugin supports configurable offset management to control message delivery semantics:
+
+#### Auto Commit (Default - Backward Compatible)
+
+By default (`autoCommit: true` or not specified), Kafka automatically commits offsets at regular intervals. This is the original behavior and ensures backward compatibility.
+
+#### Manual Commit (Opt-in for Reliability)
+
+When you explicitly set `autoCommit: false`, the plugin will:
+
+1. Start consuming from the last committed offset for the consumer group
+2. Process each message by publishing it to the Backstage events system
+3. Only commit the offset after successful processing
+4. If processing fails, pause the consumer and do not commit the offset
+
+**Example configuration for manual commit:**
+
+```yaml
+kafka:
+  topics:
+    - topic1
+  groupId: my-group
+  autoCommit: false # Enable manual commit
+```
+
+#### Error Handling
+
+The `pauseOnError` option controls how the consumer behaves when message processing fails:
+
+**Skip Failed Messages (Default - Backward Compatible)**
+
+By default (`pauseOnError: false` or not specified), the consumer will skip failed messages and continue processing:
+
+- The consumer logs the error but continues processing subsequent messages
+- If `autoCommit: false`, the offset is still committed to skip the failed message
+- If `autoCommit: true`, Kafka's auto-commit handles the offset
+- Recommended when occasional message failures are acceptable and should not block processing
+
+**Pause on Error (Opt-in)**
+
+When you explicitly set `pauseOnError: true`, the consumer will pause when an error occurs during message processing:
+
+- The consumer pauses and stops processing new messages
+- The failed message offset is not committed
+- The error is re-thrown and logged
+- Recommended when you want to investigate and fix issues before continuing
+
+**Example configuration to pause on error:**
+
+```yaml
+kafka:
+  topics:
+    - topic1
+  groupId: my-group
+  autoCommit: false
+  pauseOnError: true # Pause consumer when a message fails
+```
+
+**Note:** When using the default behavior (`pauseOnError: false`) with `autoCommit: false`, failed messages will have their offsets committed, meaning they will be skipped and not reprocessed. Use this configuration carefully based on your application's requirements.
+
+#### Starting Position
+
+The `fromBeginning` option controls where the consumer starts when no committed offset exists:
+
+- `fromBeginning: true` - Start from the earliest available message
+- `fromBeginning: false` (default) - Start from the latest message (only new messages)
+
+Once the consumer group has committed an offset, it will always resume from that position, regardless of the `fromBeginning` setting.
+
 ### Optional SSL Configuration
 
 If your Kafka cluster requires SSL, you can configure it for both `kafkaConsumingEventPublisher` and `kafkaPublishingEventConsumer` instances:

package/config.d.ts

@@ -200,6 +200,29 @@ export interface Config {
                    * Default: 5000
                    */
                   maxWaitTime?: HumanDuration | string;
+
+                  /**
+                   * (Optional) If true, the consumer group will start from the earliest offset when no committed offset is found.
+                   * If false or not specified, it will start from the latest offset.
+                   * Default: undefined (start from latest)
+                   */
+                  fromBeginning?: boolean;
+
+                  /**
+                   * (Optional) Enable auto-commit of offsets.
+                   * When true (default), offsets are automatically committed at regular intervals (at-most-once delivery).
+                   * When false, offsets are only committed after successful message processing (at-least-once delivery).
+                   * Default: true (auto-commit enabled for backward compatibility)
+                   */
+                  autoCommit?: boolean;
+
+                  /**
+                   * (Optional) When true, the consumer will pause on error and stop processing messages.
+                   * When false (default), the consumer will skip failed messages and continue processing.
+                   * Note: When pauseOnError is false and autoCommit is also false, failed messages will still have their offsets committed.
+                   * Default: false (skip errors for backward compatibility)
+                   */
+                  pauseOnError?: boolean;
                 };
               }>;
             }
@@ -374,6 +397,29 @@ export interface Config {
                      * Default: 5000
                      */
                     maxWaitTime?: HumanDuration | string;
+
+                    /**
+                     * (Optional) If true, the consumer group will start from the earliest offset when no committed offset is found.
+                     * If false or not specified, it will start from the latest offset.
+                     * Default: undefined (start from latest)
+                     */
+                    fromBeginning?: boolean;
+
+                    /**
+                     * (Optional) Enable auto-commit of offsets.
+                     * When true (default), offsets are automatically committed at regular intervals (at-most-once delivery).
+                     * When false, offsets are only committed after successful message processing (at-least-once delivery).
+                     * Default: true (auto-commit enabled for backward compatibility)
+                     */
+                    autoCommit?: boolean;
+
+                    /**
+                     * (Optional) When true, the consumer will pause on error and stop processing messages.
+                     * When false (default), the consumer will skip failed messages and continue processing.
+                     * Note: When pauseOnError is false and autoCommit is also false, failed messages will still have their offsets committed.
+                     * Default: false (skip errors for backward compatibility)
+                     */
+                    pauseOnError?: boolean;
                   };
                 }>;
               };

package/dist/KafkaConsumingEventPublisher/KafkaConsumingEventPublisher.cjs.js

@@ -42,12 +42,49 @@ class KafkaConsumingEventPublisher {
           await consumer.connect();
           await consumer.subscribe(config.consumerSubscribeTopics);
           await consumer.run({
-            eachMessage: async ({ message }) => {
-              this.events.publish({
-                topic: config.backstageTopic,
-                eventPayload: JSON.parse(message.value?.toString()),
-                metadata: kafkaTransformers.convertHeadersToMetadata(message.headers)
-              });
+            autoCommit: config.autoCommit,
+            eachMessage: async ({
+              topic,
+              partition,
+              message,
+              heartbeat,
+              pause
+            }) => {
+              try {
+                await this.events.publish({
+                  topic: config.backstageTopic,
+                  eventPayload: JSON.parse(message.value?.toString()),
+                  metadata: kafkaTransformers.convertHeadersToMetadata(message.headers)
+                });
+                if (!config.autoCommit) {
+                  await consumer.commitOffsets([
+                    {
+                      topic,
+                      partition,
+                      offset: (parseInt(message.offset, 10) + 1).toString()
+                    }
+                  ]);
+                }
+                await heartbeat();
+              } catch (error) {
+                consumerLogger.error(
+                  `Failed to process message at offset ${message.offset} on partition ${partition} of topic ${topic}`,
+                  error
+                );
+                if (config.pauseOnError) {
+                  pause();
+                  throw error;
+                }
+                if (!config.autoCommit) {
+                  await consumer.commitOffsets([
+                    {
+                      topic,
+                      partition,
+                      offset: (parseInt(message.offset, 10) + 1).toString()
+                    }
+                  ]);
+                }
+              }
             }
           });
         } catch (error) {

package/dist/KafkaConsumingEventPublisher/KafkaConsumingEventPublisher.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"KafkaConsumingEventPublisher.cjs.js","sources":["../../src/KafkaConsumingEventPublisher/KafkaConsumingEventPublisher.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { Consumer, Kafka } from 'kafkajs';\nimport {\n  KafkaConsumerConfig,\n  KafkaConsumingEventPublisherConfig,\n  readConsumerConfig,\n} from './config';\nimport { Config } from '@backstage/config';\nimport { loggerServiceAdapter } from '../utils/LoggerServiceAdapter';\nimport { convertHeadersToMetadata } from '../utils/kafkaTransformers';\n\ntype KafkaConsumer = {\n  consumer: Consumer;\n  config: KafkaConsumerConfig;\n};\n\n/**\n * This class subscribes to Kafka topics and publishes events received to the registered subscriber.\n * The message payload will be used as the event payload and passed to the subscribers.\n */\nexport class KafkaConsumingEventPublisher {\n  private readonly kafkaConsumers: KafkaConsumer[];\n  private readonly logger: LoggerService;\n\n  static fromConfig(env: {\n    config: Config;\n    events: EventsService;\n    logger: LoggerService;\n  }): KafkaConsumingEventPublisher[] {\n    const configs = readConsumerConfig(env.config, env.logger);\n\n    return configs.map(\n      kafkaConfig =>\n        new KafkaConsumingEventPublisher(env.logger, env.events, kafkaConfig),\n    );\n  }\n\n  private constructor(\n    logger: LoggerService,\n    private readonly events: EventsService,\n    config: KafkaConsumingEventPublisherConfig,\n  ) {\n    this.logger = logger.child({\n      class: KafkaConsumingEventPublisher.prototype.constructor.name,\n      instance: config.instance,\n    });\n\n    const kafka = new Kafka({\n      ...config.kafkaConfig,\n      logCreator: loggerServiceAdapter(this.logger),\n    });\n\n    this.kafkaConsumers = config.kafkaConsumerConfigs.map(consumerConfig => ({\n      consumer: kafka.consumer(consumerConfig.consumerConfig),\n      config: consumerConfig,\n    }));\n  }\n\n  async start(): Promise<void> {\n    await Promise.all(\n      this.kafkaConsumers.map(async ({ consumer, config }) => {\n        const consumerLogger = this.logger.child({\n          id: `events.kafka.publisher:${config.backstageTopic}`,\n          groupId: config.consumerConfig.groupId,\n          kafkaTopics: config.consumerSubscribeTopics.topics.toString(),\n          backstageTopic: config.backstageTopic,\n        });\n        try {\n          await consumer.connect();\n          await consumer.subscribe(config.consumerSubscribeTopics);\n\n          await consumer.run({\n            eachMessage: async ({ message }) => {\n              this.events.publish({\n                topic: config.backstageTopic,\n                eventPayload: JSON.parse(message.value?.toString()!),\n                metadata: convertHeadersToMetadata(message.headers),\n              });\n            },\n          });\n        } catch (error: any) {\n          consumerLogger.error('Kafka consumer connection failed', error);\n        }\n      }),\n    );\n  }\n\n  async shutdown(): Promise<void> {\n    await Promise.all(\n      this.kafkaConsumers.map(({ consumer }) => consumer.disconnect()),\n    );\n  }\n}\n"],"names":["Kafka","loggerServiceAdapter","readConsumerConfig","convertHeadersToMetadata"],"mappings":";;;;;;;AAoCO,MAAM,4BAAA,CAA6B;AAAA,EAiBhC,WAAA,CACN,MAAA,EACiB,MAAA,EACjB,MAAA,EACA;AAFiB,IAAA,IAAA,CAAA,MAAA,GAAA,MAAA;AAGjB,IAAA,IAAA,CAAK,MAAA,GAAS,OAAO,KAAA,CAAM;AAAA,MACzB,KAAA,EAAO,4BAAA,CAA6B,SAAA,CAAU,WAAA,CAAY,IAAA;AAAA,MAC1D,UAAU,MAAA,CAAO;AAAA,KAClB,CAAA;AAED,IAAA,MAAM,KAAA,GAAQ,IAAIA,aAAA,CAAM;AAAA,MACtB,GAAG,MAAA,CAAO,WAAA;AAAA,MACV,UAAA,EAAYC,yCAAA,CAAqB,IAAA,CAAK,MAAM;AAAA,KAC7C,CAAA;AAED,IAAA,IAAA,CAAK,cAAA,GAAiB,MAAA,CAAO,oBAAA,CAAqB,GAAA,CAAI,CAAA,cAAA,MAAmB;AAAA,MACvE,QAAA,EAAU,KAAA,CAAM,QAAA,CAAS,cAAA,CAAe,cAAc,CAAA;AAAA,MACtD,MAAA,EAAQ;AAAA,KACV,CAAE,CAAA;AAAA,EACJ;AAAA,EAnCiB,cAAA;AAAA,EACA,MAAA;AAAA,EAEjB,OAAO,WAAW,GAAA,EAIiB;AACjC,IAAA,MAAM,OAAA,GAAUC,yBAAA,CAAmB,GAAA,CAAI,MAAA,EAAQ,IAAI,MAAM,CAAA;AAEzD,IAAA,OAAO,OAAA,CAAQ,GAAA;AAAA,MACb,iBACE,IAAI,4BAAA,CAA6B,IAAI,MAAA,EAAQ,GAAA,CAAI,QAAQ,WAAW;AAAA,KACxE;AAAA,EACF;AAAA,EAuBA,MAAM,KAAA,GAAuB;AAC3B,IAAA,MAAM,OAAA,CAAQ,GAAA;AAAA,MACZ,KAAK,cAAA,CAAe,GAAA,CAAI,OAAO,EAAE,QAAA,EAAU,QAAO,KAAM;AACtD,QAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM;AAAA,UACvC,EAAA,EAAI,CAAA,uBAAA,EAA0B,MAAA,CAAO,cAAc,CAAA,CAAA;AAAA,UACnD,OAAA,EAAS,OAAO,cAAA,CAAe,OAAA;AAAA,UAC/B,WAAA,EAAa,MAAA,CAAO,uBAAA,CAAwB,MAAA,CAAO,QAAA,EAAS;AAAA,UAC5D,gBAAgB,MAAA,CAAO;AAAA,SACxB,CAAA;AACD,QAAA,IAAI;AACF,UAAA,MAAM,SAAS,OAAA,EAAQ;AACvB,UAAA,MAAM,QAAA,CAAS,SAAA,CAAU,MAAA,CAAO,uBAAuB,CAAA;AAEvD,UAAA,MAAM,SAAS,GAAA,CAAI;AAAA,YACjB,WAAA,EAAa,OAAO,EAAE,OAAA,EAAQ,KAAM;AAClC,cAAA,IAAA,CAAK,OAAO,OAAA,CAAQ;AAAA,gBAClB,OAAO,MAAA,CAAO,cAAA;AAAA,gBACd,cAAc,IAAA,CAAK,KAAA,CAAM,OAAA,CAAQ,KAAA,EAAO,UAAW,CAAA;AAAA,gBACnD,QAAA,EAAUC,0CAAA,CAAyB,OAAA,CAAQ,OAAO;AAAA,eACnD,CAAA;AAAA,YACH;AAAA,WACD,CAAA;AAAA,QACH,SAAS,KAAA,EAAY;AACnB,UAAA,cAAA,CAAe,KAAA,CAAM,oCAAoC,KAAK,CAAA;AAAA,QAChE;AAAA,MACF,CAAC;AAAA,KACH;AAAA,EACF;AAAA,EAEA,MAAM,QAAA,GAA0B;AAC9B,IAAA,MAAM,OAAA,CAAQ,GAAA;AAAA,MACZ,IAAA,CAAK,eAAe,GAAA,CAAI,CAAC,EAAE,QAAA,EAAS,KAAM,QAAA,CAAS,UAAA,EAAY;AAAA,KACjE;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"KafkaConsumingEventPublisher.cjs.js","sources":["../../src/KafkaConsumingEventPublisher/KafkaConsumingEventPublisher.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { Consumer, Kafka } from 'kafkajs';\nimport {\n  KafkaConsumerConfig,\n  KafkaConsumingEventPublisherConfig,\n  readConsumerConfig,\n} from './config';\nimport { Config } from '@backstage/config';\nimport { loggerServiceAdapter } from '../utils/LoggerServiceAdapter';\nimport { convertHeadersToMetadata } from '../utils/kafkaTransformers';\n\ntype KafkaConsumer = {\n  consumer: Consumer;\n  config: KafkaConsumerConfig;\n};\n\n/**\n * This class subscribes to Kafka topics and publishes events received to the registered subscriber.\n * The message payload will be used as the event payload and passed to the subscribers.\n */\nexport class KafkaConsumingEventPublisher {\n  private readonly kafkaConsumers: KafkaConsumer[];\n  private readonly logger: LoggerService;\n\n  static fromConfig(env: {\n    config: Config;\n    events: EventsService;\n    logger: LoggerService;\n  }): KafkaConsumingEventPublisher[] {\n    const configs = readConsumerConfig(env.config, env.logger);\n\n    return configs.map(\n      kafkaConfig =>\n        new KafkaConsumingEventPublisher(env.logger, env.events, kafkaConfig),\n    );\n  }\n\n  private constructor(\n    logger: LoggerService,\n    private readonly events: EventsService,\n    config: KafkaConsumingEventPublisherConfig,\n  ) {\n    this.logger = logger.child({\n      class: KafkaConsumingEventPublisher.prototype.constructor.name,\n      instance: config.instance,\n    });\n\n    const kafka = new Kafka({\n      ...config.kafkaConfig,\n      logCreator: loggerServiceAdapter(this.logger),\n    });\n\n    this.kafkaConsumers = config.kafkaConsumerConfigs.map(consumerConfig => ({\n      consumer: kafka.consumer(consumerConfig.consumerConfig),\n      config: consumerConfig,\n    }));\n  }\n\n  async start(): Promise<void> {\n    await Promise.all(\n      this.kafkaConsumers.map(async ({ consumer, config }) => {\n        const consumerLogger = this.logger.child({\n          id: `events.kafka.publisher:${config.backstageTopic}`,\n          groupId: config.consumerConfig.groupId,\n          kafkaTopics: config.consumerSubscribeTopics.topics.toString(),\n          backstageTopic: config.backstageTopic,\n        });\n        try {\n          await consumer.connect();\n          await consumer.subscribe(config.consumerSubscribeTopics);\n\n          await consumer.run({\n            autoCommit: config.autoCommit,\n            eachMessage: async ({\n              topic,\n              partition,\n              message,\n              heartbeat,\n              pause,\n            }) => {\n              try {\n                await this.events.publish({\n                  topic: config.backstageTopic,\n                  eventPayload: JSON.parse(message.value?.toString()!),\n                  metadata: convertHeadersToMetadata(message.headers),\n                });\n\n                // Only commit offset manually if autoCommit is disabled\n                if (!config.autoCommit) {\n                  await consumer.commitOffsets([\n                    {\n                      topic,\n                      partition,\n                      offset: (parseInt(message.offset, 10) + 1).toString(),\n                    },\n                  ]);\n                }\n\n                await heartbeat();\n              } catch (error: any) {\n                consumerLogger.error(\n                  `Failed to process message at offset ${message.offset} on partition ${partition} of topic ${topic}`,\n                  error,\n                );\n\n                if (config.pauseOnError) {\n                  pause();\n                  throw error;\n                }\n\n                // Skip the failed message by committing its offset if autoCommit is disabled\n                if (!config.autoCommit) {\n                  await consumer.commitOffsets([\n                    {\n                      topic,\n                      partition,\n                      offset: (parseInt(message.offset, 10) + 1).toString(),\n                    },\n                  ]);\n                }\n              }\n            },\n          });\n        } catch (error: any) {\n          consumerLogger.error('Kafka consumer connection failed', error);\n        }\n      }),\n    );\n  }\n\n  async shutdown(): Promise<void> {\n    await Promise.all(\n      this.kafkaConsumers.map(({ consumer }) => consumer.disconnect()),\n    );\n  }\n}\n"],"names":["Kafka","loggerServiceAdapter","readConsumerConfig","convertHeadersToMetadata"],"mappings":";;;;;;;AAoCO,MAAM,4BAAA,CAA6B;AAAA,EAiBhC,WAAA,CACN,MAAA,EACiB,MAAA,EACjB,MAAA,EACA;AAFiB,IAAA,IAAA,CAAA,MAAA,GAAA,MAAA;AAGjB,IAAA,IAAA,CAAK,MAAA,GAAS,OAAO,KAAA,CAAM;AAAA,MACzB,KAAA,EAAO,4BAAA,CAA6B,SAAA,CAAU,WAAA,CAAY,IAAA;AAAA,MAC1D,UAAU,MAAA,CAAO;AAAA,KAClB,CAAA;AAED,IAAA,MAAM,KAAA,GAAQ,IAAIA,aAAA,CAAM;AAAA,MACtB,GAAG,MAAA,CAAO,WAAA;AAAA,MACV,UAAA,EAAYC,yCAAA,CAAqB,IAAA,CAAK,MAAM;AAAA,KAC7C,CAAA;AAED,IAAA,IAAA,CAAK,cAAA,GAAiB,MAAA,CAAO,oBAAA,CAAqB,GAAA,CAAI,CAAA,cAAA,MAAmB;AAAA,MACvE,QAAA,EAAU,KAAA,CAAM,QAAA,CAAS,cAAA,CAAe,cAAc,CAAA;AAAA,MACtD,MAAA,EAAQ;AAAA,KACV,CAAE,CAAA;AAAA,EACJ;AAAA,EAnCiB,cAAA;AAAA,EACA,MAAA;AAAA,EAEjB,OAAO,WAAW,GAAA,EAIiB;AACjC,IAAA,MAAM,OAAA,GAAUC,yBAAA,CAAmB,GAAA,CAAI,MAAA,EAAQ,IAAI,MAAM,CAAA;AAEzD,IAAA,OAAO,OAAA,CAAQ,GAAA;AAAA,MACb,iBACE,IAAI,4BAAA,CAA6B,IAAI,MAAA,EAAQ,GAAA,CAAI,QAAQ,WAAW;AAAA,KACxE;AAAA,EACF;AAAA,EAuBA,MAAM,KAAA,GAAuB;AAC3B,IAAA,MAAM,OAAA,CAAQ,GAAA;AAAA,MACZ,KAAK,cAAA,CAAe,GAAA,CAAI,OAAO,EAAE,QAAA,EAAU,QAAO,KAAM;AACtD,QAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM;AAAA,UACvC,EAAA,EAAI,CAAA,uBAAA,EAA0B,MAAA,CAAO,cAAc,CAAA,CAAA;AAAA,UACnD,OAAA,EAAS,OAAO,cAAA,CAAe,OAAA;AAAA,UAC/B,WAAA,EAAa,MAAA,CAAO,uBAAA,CAAwB,MAAA,CAAO,QAAA,EAAS;AAAA,UAC5D,gBAAgB,MAAA,CAAO;AAAA,SACxB,CAAA;AACD,QAAA,IAAI;AACF,UAAA,MAAM,SAAS,OAAA,EAAQ;AACvB,UAAA,MAAM,QAAA,CAAS,SAAA,CAAU,MAAA,CAAO,uBAAuB,CAAA;AAEvD,UAAA,MAAM,SAAS,GAAA,CAAI;AAAA,YACjB,YAAY,MAAA,CAAO,UAAA;AAAA,YACnB,aAAa,OAAO;AAAA,cAClB,KAAA;AAAA,cACA,SAAA;AAAA,cACA,OAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA,aACF,KAAM;AACJ,cAAA,IAAI;AACF,gBAAA,MAAM,IAAA,CAAK,OAAO,OAAA,CAAQ;AAAA,kBACxB,OAAO,MAAA,CAAO,cAAA;AAAA,kBACd,cAAc,IAAA,CAAK,KAAA,CAAM,OAAA,CAAQ,KAAA,EAAO,UAAW,CAAA;AAAA,kBACnD,QAAA,EAAUC,0CAAA,CAAyB,OAAA,CAAQ,OAAO;AAAA,iBACnD,CAAA;AAGD,gBAAA,IAAI,CAAC,OAAO,UAAA,EAAY;AACtB,kBAAA,MAAM,SAAS,aAAA,CAAc;AAAA,oBAC3B;AAAA,sBACE,KAAA;AAAA,sBACA,SAAA;AAAA,sBACA,SAAS,QAAA,CAAS,OAAA,CAAQ,QAAQ,EAAE,CAAA,GAAI,GAAG,QAAA;AAAS;AACtD,mBACD,CAAA;AAAA,gBACH;AAEA,gBAAA,MAAM,SAAA,EAAU;AAAA,cAClB,SAAS,KAAA,EAAY;AACnB,gBAAA,cAAA,CAAe,KAAA;AAAA,kBACb,uCAAuC,OAAA,CAAQ,MAAM,CAAA,cAAA,EAAiB,SAAS,aAAa,KAAK,CAAA,CAAA;AAAA,kBACjG;AAAA,iBACF;AAEA,gBAAA,IAAI,OAAO,YAAA,EAAc;AACvB,kBAAA,KAAA,EAAM;AACN,kBAAA,MAAM,KAAA;AAAA,gBACR;AAGA,gBAAA,IAAI,CAAC,OAAO,UAAA,EAAY;AACtB,kBAAA,MAAM,SAAS,aAAA,CAAc;AAAA,oBAC3B;AAAA,sBACE,KAAA;AAAA,sBACA,SAAA;AAAA,sBACA,SAAS,QAAA,CAAS,OAAA,CAAQ,QAAQ,EAAE,CAAA,GAAI,GAAG,QAAA;AAAS;AACtD,mBACD,CAAA;AAAA,gBACH;AAAA,cACF;AAAA,YACF;AAAA,WACD,CAAA;AAAA,QACH,SAAS,KAAA,EAAY;AACnB,UAAA,cAAA,CAAe,KAAA,CAAM,oCAAoC,KAAK,CAAA;AAAA,QAChE;AAAA,MACF,CAAC;AAAA,KACH;AAAA,EACF;AAAA,EAEA,MAAM,QAAA,GAA0B;AAC9B,IAAA,MAAM,OAAA,CAAQ,GAAA;AAAA,MACZ,IAAA,CAAK,eAAe,GAAA,CAAI,CAAC,EAAE,QAAA,EAAS,KAAM,QAAA,CAAS,UAAA,EAAY;AAAA,KACjE;AAAA,EACF;AACF;;;;"}

package/dist/KafkaConsumingEventPublisher/config.cjs.js

@@ -39,8 +39,15 @@ const processSinglePublisher = (instanceName, publisherConfig) => {
           )
         },
         consumerSubscribeTopics: {
-          topics: topicConfig.getStringArray("kafka.topics")
-        }
+          topics: topicConfig.getStringArray("kafka.topics"),
+          fromBeginning: topicConfig.getOptionalBoolean(
+            "kafka.fromBeginning"
+          )
+        },
+        // Default autoCommit to true to match KafkaJS default and ensure consistency
+        // between KafkaJS's auto-commit behavior and our manual commit logic
+        autoCommit: topicConfig.getOptionalBoolean("kafka.autoCommit") ?? true,
+        pauseOnError: topicConfig.getOptionalBoolean("kafka.pauseOnError") ?? false
       };
     })
   };

package/dist/KafkaConsumingEventPublisher/config.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.cjs.js","sources":["../../src/KafkaConsumingEventPublisher/config.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { Config } from '@backstage/config';\nimport { ConsumerConfig, ConsumerSubscribeTopics, KafkaConfig } from 'kafkajs';\nimport {\n  readKafkaConfig,\n  readOptionalHumanDurationInMs,\n} from '../utils/config';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nexport interface KafkaConsumerConfig {\n  backstageTopic: string;\n  consumerConfig: ConsumerConfig;\n  consumerSubscribeTopics: ConsumerSubscribeTopics;\n}\n\nexport interface KafkaConsumingEventPublisherConfig {\n  instance: string;\n  kafkaConfig: KafkaConfig;\n  kafkaConsumerConfigs: KafkaConsumerConfig[];\n}\n\nconst CONFIG_PREFIX_PUBLISHER =\n  'events.modules.kafka.kafkaConsumingEventPublisher';\n\nconst processSinglePublisher = (\n  instanceName: string,\n  publisherConfig: Config,\n): KafkaConsumingEventPublisherConfig => {\n  return {\n    instance: instanceName,\n    kafkaConfig: readKafkaConfig(publisherConfig),\n    kafkaConsumerConfigs: publisherConfig\n      .getConfigArray('topics')\n      .map(topicConfig => {\n        return {\n          backstageTopic: topicConfig.getString('topic'),\n          consumerConfig: {\n            groupId: topicConfig.getString('kafka.groupId'),\n            sessionTimeout: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.sessionTimeout',\n            ),\n            rebalanceTimeout: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.rebalanceTimeout',\n            ),\n            heartbeatInterval: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.heartbeatInterval',\n            ),\n            metadataMaxAge: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.metadataMaxAge',\n            ),\n            maxBytesPerPartition: topicConfig.getOptionalNumber(\n              'kafka.maxBytesPerPartition',\n            ),\n            minBytes: topicConfig.getOptionalNumber('kafka.minBytes'),\n            maxBytes: topicConfig.getOptionalNumber('kafka.maxBytes'),\n            maxWaitTimeInMs: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.maxWaitTime',\n            ),\n          },\n          consumerSubscribeTopics: {\n            topics: topicConfig.getStringArray('kafka.topics'),\n          },\n        };\n      }),\n  };\n};\n\nexport const readConsumerConfig = (\n  config: Config,\n  logger: LoggerService,\n): KafkaConsumingEventPublisherConfig[] => {\n  const publishersConfig = config.getOptionalConfig(CONFIG_PREFIX_PUBLISHER);\n\n  // Check for legacy single publisher format\n  if (publishersConfig?.getOptionalString('clientId')) {\n    logger.warn(\n      'Legacy single config format detected at events.modules.kafka.kafkaConsumingEventPublisher.',\n    );\n    return [\n      processSinglePublisher(\n        'default', // use `default` as instance name for legacy single config\n        publishersConfig,\n      ),\n    ];\n  }\n\n  return (\n    publishersConfig\n      ?.keys()\n      ?.map(publisherKey =>\n        processSinglePublisher(\n          publisherKey,\n          publishersConfig.getConfig(publisherKey),\n        ),\n      ) ?? []\n  );\n};\n"],"names":["readKafkaConfig","readOptionalHumanDurationInMs"],"mappings":";;;;AAmCA,MAAM,uBAAA,GACJ,mDAAA;AAEF,MAAM,sBAAA,GAAyB,CAC7B,YAAA,EACA,eAAA,KACuC;AACvC,EAAA,OAAO;AAAA,IACL,QAAA,EAAU,YAAA;AAAA,IACV,WAAA,EAAaA,uBAAgB,eAAe,CAAA;AAAA,IAC5C,sBAAsB,eAAA,CACnB,cAAA,CAAe,QAAQ,CAAA,CACvB,IAAI,CAAA,WAAA,KAAe;AAClB,MAAA,OAAO;AAAA,QACL,cAAA,EAAgB,WAAA,CAAY,SAAA,CAAU,OAAO,CAAA;AAAA,QAC7C,cAAA,EAAgB;AAAA,UACd,OAAA,EAAS,WAAA,CAAY,SAAA,CAAU,eAAe,CAAA;AAAA,UAC9C,cAAA,EAAgBC,oCAAA;AAAA,YACd,WAAA;AAAA,YACA;AAAA,WACF;AAAA,UACA,gBAAA,EAAkBA,oCAAA;AAAA,YAChB,WAAA;AAAA,YACA;AAAA,WACF;AAAA,UACA,iBAAA,EAAmBA,oCAAA;AAAA,YACjB,WAAA;AAAA,YACA;AAAA,WACF;AAAA,UACA,cAAA,EAAgBA,oCAAA;AAAA,YACd,WAAA;AAAA,YACA;AAAA,WACF;AAAA,UACA,sBAAsB,WAAA,CAAY,iBAAA;AAAA,YAChC;AAAA,WACF;AAAA,UACA,QAAA,EAAU,WAAA,CAAY,iBAAA,CAAkB,gBAAgB,CAAA;AAAA,UACxD,QAAA,EAAU,WAAA,CAAY,iBAAA,CAAkB,gBAAgB,CAAA;AAAA,UACxD,eAAA,EAAiBA,oCAAA;AAAA,YACf,WAAA;AAAA,YACA;AAAA;AACF,SACF;AAAA,QACA,uBAAA,EAAyB;AAAA,UACvB,MAAA,EAAQ,WAAA,CAAY,cAAA,CAAe,cAAc;AAAA;AACnD,OACF;AAAA,IACF,CAAC;AAAA,GACL;AACF,CAAA;AAEO,MAAM,kBAAA,GAAqB,CAChC,MAAA,EACA,MAAA,KACyC;AACzC,EAAA,MAAM,gBAAA,GAAmB,MAAA,CAAO,iBAAA,CAAkB,uBAAuB,CAAA;AAGzE,EAAA,IAAI,gBAAA,EAAkB,iBAAA,CAAkB,UAAU,CAAA,EAAG;AACnD,IAAA,MAAA,CAAO,IAAA;AAAA,MACL;AAAA,KACF;AACA,IAAA,OAAO;AAAA,MACL,sBAAA;AAAA,QACE,SAAA;AAAA;AAAA,QACA;AAAA;AACF,KACF;AAAA,EACF;AAEA,EAAA,OACE,gBAAA,EACI,MAAK,EACL,GAAA;AAAA,IAAI,CAAA,YAAA,KACJ,sBAAA;AAAA,MACE,YAAA;AAAA,MACA,gBAAA,CAAiB,UAAU,YAAY;AAAA;AACzC,OACG,EAAC;AAEZ;;;;"}
+{"version":3,"file":"config.cjs.js","sources":["../../src/KafkaConsumingEventPublisher/config.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { Config } from '@backstage/config';\nimport { ConsumerConfig, ConsumerSubscribeTopics, KafkaConfig } from 'kafkajs';\nimport {\n  readKafkaConfig,\n  readOptionalHumanDurationInMs,\n} from '../utils/config';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nexport interface KafkaConsumerConfig {\n  backstageTopic: string;\n  consumerConfig: ConsumerConfig;\n  consumerSubscribeTopics: ConsumerSubscribeTopics;\n  autoCommit: boolean;\n  pauseOnError: boolean;\n}\n\nexport interface KafkaConsumingEventPublisherConfig {\n  instance: string;\n  kafkaConfig: KafkaConfig;\n  kafkaConsumerConfigs: KafkaConsumerConfig[];\n}\n\nconst CONFIG_PREFIX_PUBLISHER =\n  'events.modules.kafka.kafkaConsumingEventPublisher';\n\nconst processSinglePublisher = (\n  instanceName: string,\n  publisherConfig: Config,\n): KafkaConsumingEventPublisherConfig => {\n  return {\n    instance: instanceName,\n    kafkaConfig: readKafkaConfig(publisherConfig),\n    kafkaConsumerConfigs: publisherConfig\n      .getConfigArray('topics')\n      .map(topicConfig => {\n        return {\n          backstageTopic: topicConfig.getString('topic'),\n          consumerConfig: {\n            groupId: topicConfig.getString('kafka.groupId'),\n            sessionTimeout: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.sessionTimeout',\n            ),\n            rebalanceTimeout: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.rebalanceTimeout',\n            ),\n            heartbeatInterval: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.heartbeatInterval',\n            ),\n            metadataMaxAge: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.metadataMaxAge',\n            ),\n            maxBytesPerPartition: topicConfig.getOptionalNumber(\n              'kafka.maxBytesPerPartition',\n            ),\n            minBytes: topicConfig.getOptionalNumber('kafka.minBytes'),\n            maxBytes: topicConfig.getOptionalNumber('kafka.maxBytes'),\n            maxWaitTimeInMs: readOptionalHumanDurationInMs(\n              topicConfig,\n              'kafka.maxWaitTime',\n            ),\n          },\n          consumerSubscribeTopics: {\n            topics: topicConfig.getStringArray('kafka.topics'),\n            fromBeginning: topicConfig.getOptionalBoolean(\n              'kafka.fromBeginning',\n            ),\n          },\n          // Default autoCommit to true to match KafkaJS default and ensure consistency\n          // between KafkaJS's auto-commit behavior and our manual commit logic\n          autoCommit:\n            topicConfig.getOptionalBoolean('kafka.autoCommit') ?? true,\n          pauseOnError:\n            topicConfig.getOptionalBoolean('kafka.pauseOnError') ?? false,\n        };\n      }),\n  };\n};\n\nexport const readConsumerConfig = (\n  config: Config,\n  logger: LoggerService,\n): KafkaConsumingEventPublisherConfig[] => {\n  const publishersConfig = config.getOptionalConfig(CONFIG_PREFIX_PUBLISHER);\n\n  // Check for legacy single publisher format\n  if (publishersConfig?.getOptionalString('clientId')) {\n    logger.warn(\n      'Legacy single config format detected at events.modules.kafka.kafkaConsumingEventPublisher.',\n    );\n    return [\n      processSinglePublisher(\n        'default', // use `default` as instance name for legacy single config\n        publishersConfig,\n      ),\n    ];\n  }\n\n  return (\n    publishersConfig\n      ?.keys()\n      ?.map(publisherKey =>\n        processSinglePublisher(\n          publisherKey,\n          publishersConfig.getConfig(publisherKey),\n        ),\n      ) ?? []\n  );\n};\n"],"names":["readKafkaConfig","readOptionalHumanDurationInMs"],"mappings":";;;;AAqCA,MAAM,uBAAA,GACJ,mDAAA;AAEF,MAAM,sBAAA,GAAyB,CAC7B,YAAA,EACA,eAAA,KACuC;AACvC,EAAA,OAAO;AAAA,IACL,QAAA,EAAU,YAAA;AAAA,IACV,WAAA,EAAaA,uBAAgB,eAAe,CAAA;AAAA,IAC5C,sBAAsB,eAAA,CACnB,cAAA,CAAe,QAAQ,CAAA,CACvB,IAAI,CAAA,WAAA,KAAe;AAClB,MAAA,OAAO;AAAA,QACL,cAAA,EAAgB,WAAA,CAAY,SAAA,CAAU,OAAO,CAAA;AAAA,QAC7C,cAAA,EAAgB;AAAA,UACd,OAAA,EAAS,WAAA,CAAY,SAAA,CAAU,eAAe,CAAA;AAAA,UAC9C,cAAA,EAAgBC,oCAAA;AAAA,YACd,WAAA;AAAA,YACA;AAAA,WACF;AAAA,UACA,gBAAA,EAAkBA,oCAAA;AAAA,YAChB,WAAA;AAAA,YACA;AAAA,WACF;AAAA,UACA,iBAAA,EAAmBA,oCAAA;AAAA,YACjB,WAAA;AAAA,YACA;AAAA,WACF;AAAA,UACA,cAAA,EAAgBA,oCAAA;AAAA,YACd,WAAA;AAAA,YACA;AAAA,WACF;AAAA,UACA,sBAAsB,WAAA,CAAY,iBAAA;AAAA,YAChC;AAAA,WACF;AAAA,UACA,QAAA,EAAU,WAAA,CAAY,iBAAA,CAAkB,gBAAgB,CAAA;AAAA,UACxD,QAAA,EAAU,WAAA,CAAY,iBAAA,CAAkB,gBAAgB,CAAA;AAAA,UACxD,eAAA,EAAiBA,oCAAA;AAAA,YACf,WAAA;AAAA,YACA;AAAA;AACF,SACF;AAAA,QACA,uBAAA,EAAyB;AAAA,UACvB,MAAA,EAAQ,WAAA,CAAY,cAAA,CAAe,cAAc,CAAA;AAAA,UACjD,eAAe,WAAA,CAAY,kBAAA;AAAA,YACzB;AAAA;AACF,SACF;AAAA;AAAA;AAAA,QAGA,UAAA,EACE,WAAA,CAAY,kBAAA,CAAmB,kBAAkB,CAAA,IAAK,IAAA;AAAA,QACxD,YAAA,EACE,WAAA,CAAY,kBAAA,CAAmB,oBAAoB,CAAA,IAAK;AAAA,OAC5D;AAAA,IACF,CAAC;AAAA,GACL;AACF,CAAA;AAEO,MAAM,kBAAA,GAAqB,CAChC,MAAA,EACA,MAAA,KACyC;AACzC,EAAA,MAAM,gBAAA,GAAmB,MAAA,CAAO,iBAAA,CAAkB,uBAAuB,CAAA;AAGzE,EAAA,IAAI,gBAAA,EAAkB,iBAAA,CAAkB,UAAU,CAAA,EAAG;AACnD,IAAA,MAAA,CAAO,IAAA;AAAA,MACL;AAAA,KACF;AACA,IAAA,OAAO;AAAA,MACL,sBAAA;AAAA,QACE,SAAA;AAAA;AAAA,QACA;AAAA;AACF,KACF;AAAA,EACF;AAEA,EAAA,OACE,gBAAA,EACI,MAAK,EACL,GAAA;AAAA,IAAI,CAAA,YAAA,KACJ,sBAAA;AAAA,MACE,YAAA;AAAA,MACA,gBAAA,CAAiB,UAAU,YAAY;AAAA;AACzC,OACG,EAAC;AAEZ;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-events-backend-module-kafka",
-  "version": "0.0.0-nightly-20251216024726",
+  "version": "0.0.0-nightly-20260108025012",
   "description": "The kafka backend module for the events plugin.",
   "backstage": {
     "role": "backend-plugin-module",
@@ -38,16 +38,16 @@
     "test": "backstage-cli package test"
   },
   "dependencies": {
-    "@backstage/backend-plugin-api": "0.0.0-nightly-20251216024726",
+    "@backstage/backend-plugin-api": "1.6.0",
     "@backstage/config": "1.3.6",
-    "@backstage/plugin-events-node": "0.0.0-nightly-20251216024726",
+    "@backstage/plugin-events-node": "0.4.18",
     "@backstage/types": "1.2.2",
     "kafkajs": "^2.2.4"
   },
   "devDependencies": {
-    "@backstage/backend-test-utils": "0.0.0-nightly-20251216024726",
-    "@backstage/cli": "0.0.0-nightly-20251216024726",
-    "@backstage/plugin-events-backend-test-utils": "0.0.0-nightly-20251216024726"
+    "@backstage/backend-test-utils": "0.0.0-nightly-20260108025012",
+    "@backstage/cli": "0.0.0-nightly-20260108025012",
+    "@backstage/plugin-events-backend-test-utils": "0.1.51"
   },
   "configSchema": "config.d.ts",
   "typesVersions": {