@relaybus/kafka 0.3.1 → 0.4.1

package/README.md

@@ -7,28 +7,28 @@ outbox events to Kafka topics and feeds events consumed from Kafka into the
 bus inbox, with at-least-once delivery and idempotent storage.
 
 - Wire format: [CloudEvents Kafka protocol binding](https://github.com/cloudevents/spec/blob/main/cloudevents/bindings/kafka-protocol-binding.md), binary content mode, so any CloudEvents-aware consumer or tool can read the topic.
-- Built for and tested against [`kafkajs`](https://kafka.js.org) 2.2.x on a real broker. The KafkaJS-compatible API of `@confluentinc/kafka-javascript` is **experimental and untested**, see [Confluent](#confluent-experimental-untested). Neither client is a dependency; install the one you use.
+- Tested on a real broker with both clients: [`@confluentinc/kafka-javascript`](https://github.com/confluentinc/confluent-kafka-javascript) 1.11.0 (its KafkaJS-compatible API, **recommended**: maintained, librdkafka based) and [`kafkajs`](https://kafka.js.org) 2.2.4 (unmaintained). See [Choosing a client](#choosing-a-client). Neither client is a dependency; install the one you use.
 - Never loses an event: the offset is committed only after the bus stored the event; a failing store blocks the partition instead of skipping.
 
 ## Install
 
 ```sh
-pnpm add @relaybus/core @relaybus/kafka kafkajs
+pnpm add @relaybus/core @relaybus/kafka @confluentinc/kafka-javascript
 ```
 
 ## Usage
 
 ```ts
-import { Kafka, Partitioners } from 'kafkajs';
+import { KafkaJS } from '@confluentinc/kafka-javascript';
 import { RelayBus } from '@relaybus/core';
 import { KafkaTransport } from '@relaybus/kafka';
 
-const kafka = new Kafka({ clientId: 'orders', brokers: ['kafka:9092'] });
+const kafka = new KafkaJS.Kafka({ kafkaJS: { clientId: 'orders', brokers: ['kafka:9092'] } });
 
 const transport = new KafkaTransport({
   kafka,
+  client: 'confluent', // the default is 'kafkajs'
   groupId: 'billing', // required to subscribe
-  producerConfig: { createPartitioner: Partitioners.DefaultPartitioner },
 });
 
 const bus = new RelayBus({
@@ -55,13 +55,15 @@ const consuming = transport.isConsuming();
 
 | Option                         | Default             | Meaning                                                                                                                                                                                                                                                                                                                           |
 | ------------------------------ | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `kafka`                        | required            | A `kafkajs` `Kafka` or Confluent `KafkaJS.Kafka` instance. It is the producer/consumer factory.                                                                                                                                                                                                                                   |
+| `kafka`                        | required            | A `kafkajs` `Kafka` or Confluent `KafkaJS.Kafka` instance: the factory of the transport's consumer and producer. Optional with `producer`, but then the transport cannot subscribe.                                                                                                                                               |
+| `producer`                     | none                | An already-connected producer the application owns (kafkajs `Producer`, Confluent `KafkaJS.Producer`), used to publish and dead-letter instead of one of `kafka`. Never connected or disconnected by the transport. See [Reusing the application's producer](#reusing-the-applications-producer).                                 |
+| `client`                       | `'kafkajs'`         | Which client `kafka` is: `'kafkajs'` or `'confluent'`. Set `'confluent'` for a Confluent `KafkaJS.Kafka`; see [Choosing a client](#choosing-a-client).                                                                                                                                                                            |
 | `name`                         | `'kafka'`           | Transport name used by routes and subscriptions.                                                                                                                                                                                                                                                                                  |
 | `groupId`                      | none                | Consumer group. Required as soon as `subscribe` is used; publish-only services can omit it.                                                                                                                                                                                                                                       |
 | `fromBeginning`                | `true`              | Where a group without committed offsets starts. `true` so events published before the first start are not skipped.                                                                                                                                                                                                                |
 | `maxInFlightRequests`          | `1`                 | Producer in-flight requests, 1 to 5. `1` keeps retries from reordering events of one key.                                                                                                                                                                                                                                         |
-| `producerConfig`               | `{}`                | Extra producer settings (partitioner, compression, ...). `idempotent` and `maxInFlightRequests` always win.                                                                                                                                                                                                                       |
-| `consumerConfig`               | `{}`                | Extra consumer settings (`sessionTimeout`, `retry`, `heartbeatInterval`, ...). `groupId` always wins.                                                                                                                                                                                                                             |
+| `producerConfig`               | `{}`                | Extra producer settings (partitioner, compression, ...). `idempotent` and `maxInFlightRequests` always win. Confluent: KafkaJS-style settings in a `kafkaJS` block, librdkafka properties at the top level; `acks: -1` also wins, as do `enable.idempotence`, `max.in.flight` and `acks` there.                                   |
+| `consumerConfig`               | `{}`                | Extra consumer settings (`sessionTimeout`, `retry`, `heartbeatInterval`, ...). `groupId` always wins. Confluent: KafkaJS-style settings in a `kafkaJS` block, librdkafka properties at the top level; `fromBeginning` also wins, as do `group.id` and `auto.offset.reset` there. Your `rebalance_cb` is still called.             |
 | `processing.maxAttempts`       | `5`                 | In-process attempts per delivery of a record.                                                                                                                                                                                                                                                                                     |
 | `processing.initialBackoff`    | `'200ms'`           | Delay before attempt 2, doubling per attempt.                                                                                                                                                                                                                                                                                     |
 | `processing.maxBackoff`        | `'5s'`              | Upper bound of the delay between attempts.                                                                                                                                                                                                                                                                                        |
@@ -77,7 +79,8 @@ const consuming = transport.isConsuming();
 
 ## Wire format
 
-Value: the JSON payload. Key: the envelope key (`null` means no key).
+Value: the JSON payload. Key: the event key (`null` means no key); `jsonEnvelope` can write
+another key or none (`recordKey`).
 
 | Header                               | Content                                                   |
 | ------------------------------------ | --------------------------------------------------------- |
@@ -171,8 +174,10 @@ indexes an array) or a header (`{ header: 'event-type' }`) for `type`, `id`, `pa
 `'whole-body'`), `key`, `source`, `occurredAt`, `correlationId`, `causationId` and `headers`, plus a
 `contentType`. `type`, `id` and `payload` are required by the types; `id: 'record-position'` uses the
 record position (records relaybus writes carry the event id in a `relaybus-event-id` header, which
-wins on read, so outbox retries stay deduplicated). The record key is always the event key, and when `occurredAt` is not mapped the
-record timestamp carries it. Event headers that are record headers keep their names, so a name the
+wins on read, so outbox retries stay deduplicated); `{ body: 'id', whenMissing: 'record-position' }`
+uses the body id when a record has one. The record key is the event key unless `recordKey` says
+otherwise (`'none'` or `(event) => string | null`), `contentTypeHeader: false` writes no
+`content-type` header, and when `occurredAt` is not mapped the record timestamp carries it. Event headers that are record headers keep their names, so a name the
 mapping uses, `content-type` or `relaybus-event-id`, is refused with `UnencodableEventError`. The
 [interoperability recipe](https://berkcansavur.github.io/relaybus/recipes/interoperating-with-other-services)
 has a full table and a MassTransit example.
@@ -225,14 +230,23 @@ without a key (`key: null`) are spread over partitions and have no ordering. Use
 as the key where order matters. Use the same partitioner in every producer of a topic, otherwise
 one key can map to different partitions depending on who wrote it.
 
+The record key is the event key by default. A `jsonEnvelope` codec with `recordKey: 'none'`
+writes records without a key, as many existing producers do, and `recordKey: (event) => ...`
+derives another one. The event key, and the outbox's per-key order, stay as they are: relaybus
+still publishes events of one key one after the other. But a record without a key is placed on a
+partition by the producer (round-robin or sticky), so events of one key can land on different
+partitions and consumers may see them out of order. Keep the event key on the wire wherever
+consumers rely on per-key order.
+
 ## Guarantees
 
 - **Publish**: idempotent producer, `acks=-1`; `publish` resolves after the broker ack and rejects
   with `PublishTimeoutError` after `publishTimeout` (kafkajs would otherwise retry an idempotent
   send for as long as the broker is unreachable). A rejected publish makes the outbox retry; a
   timed-out send may still land, so a message can be duplicated, and the idempotent producer plus
-  the consumer's dedup on the event id absorb it. Pass `producerConfig: { retry: { retries } }` to
-  bound client-side retries further.
+  the consumer's dedup on the event id absorb it. Pass `producerConfig: { retry: { retries } }`
+  (Confluent: `producerConfig: { kafkaJS: { retry: { retries } } }`) to bound client-side retries
+  further.
 - **Consume**: `eachMessage` awaits the bus (`RelayBus.receive`, one idempotent insert) and the
   client commits the offset only afterwards. A duplicate message is stored once.
 - **Handler failure**: retried in-process `processing.maxAttempts` times with backoff and
@@ -243,7 +257,8 @@ one key can map to different partitions depending on who wrote it.
   same record. The record is never skipped, so a persistent outage blocks the partition until the
   store recovers. That is deliberate: `receive` is a cheap insert, and skipping would lose the
   event. `isConsuming()` is `false` while any partition is stuck (tracked per partition).
-- **Dead consumer**: the transport does not rely on the client to restart itself. When the client
+- **Dead consumer** (kafkajs; Confluent never gives up on a consumer): the transport does not rely
+  on the client to restart itself. When the client
   reports a crash with `restart=false`, the transport logs it and reconnects, re-subscribes and
   re-runs the consumer with bounded backoff (`restart.*`) until it joined again or the transport
   closed. `isReady()` is `false` meanwhile.
@@ -275,31 +290,70 @@ Only these members are called (`KafkaClientLike` and friends in `src/kafka-clien
 | `kafka`                    | `producer(config)`, `consumer(config)`                                                                   |
 | producer                   | `connect()`, `send({ topic, acks, messages: [{ key, value, headers }] })`, `disconnect()`                |
 | consumer                   | `connect()`, `subscribe({ topics, fromBeginning })`, `run({ eachMessage })`, `disconnect()`              |
+| Confluent consumer config  | `rebalance_cb(error, assignment)` reports joins (`-175`, assigned) and rebalances (`-174`, revoked)      |
 | `eachMessage` payload      | `topic`, `partition`, `message.{ key, value, headers, offset }`, `heartbeat()`                           |
 | consumer events (optional) | `consumer.on(name, listener)` and `consumer.events.{ GROUP_JOIN, CRASH, REBALANCING, STOP, DISCONNECT }` |
 
-`on`/`events` are optional: without `GROUP_JOIN` the transport cannot wait for the join or observe
-crashes, so `subscribe` resolves when `run` returns and `isConsuming()` only reflects processing
-stalls and `close()`. kafkajs 2.2.4 is verified by the integration tests.
+With Confluent, `send` gets no `acks` and `subscribe` no `fromBeginning` (it rejects both; they are
+set on the producer and consumer instead), and `on`/`events` are not used.
+
+`on`/`events` are optional for other kafkajs-like clients: without `GROUP_JOIN` the transport cannot
+wait for the join or observe crashes, so `subscribe` resolves when `run` returns and
+`isConsuming()` only reflects processing stalls and `close()`. kafkajs 2.2.4 and
+`@confluentinc/kafka-javascript` 1.11.0 are verified by the integration tests.
 
-## Confluent (experimental, untested)
+## Reusing the application's producer
 
-`@confluentinc/kafka-javascript` exposes a KafkaJS-compatible API, and the structural client types
-here are satisfied by it, but nothing in this repository runs against it. Its documented
-differences from kafkajs mean the options below do **not** work as written for Confluent:
+An application that already has a connected producer, with its SSL and SASL settings, can hand it
+to the transport instead of letting it open a second connection:
 
-- `idempotent` and `maxInFlightRequests` (producer) and `groupId` (consumer) must be nested in the
-  client's `kafkaJS: { ... }` config block. The transport passes them at the top level, so with
-  Confluent the producer is not guaranteed to be idempotent and the group id may be ignored.
-- `fromBeginning` is not supported per `subscribe()`; it is a consumer-level `kafkaJS` setting.
-  The transport's `fromBeginning` option is therefore not applied.
-- `heartbeat()` is a no-op there and the consumer `events` (`GROUP_JOIN`, `CRASH`, ...) are not
-  documented, so start-up falls back to "resolved when `run` returns" and crash recovery cannot
-  trigger.
-- Failed `eachMessage` handling (re-seek) and the error classification of a rethrown error differ.
+```ts
+const transport = new KafkaTransport({
+  producer: ordersService.producer, // already connected, owned by the application
+  client: 'confluent', // when it is a Confluent producer
+});
+```
 
-Use it only after verifying these against your version; until a Confluent path is tested, treat
-kafkajs as the supported client.
+Without `kafka` the transport only publishes; give `kafka` and `groupId` as well to subscribe (the
+consumer stays the transport's own). The transport calls only `send` on the injected producer: it
+never connects or disconnects it, so it is usable before `bus.start()` and stays open after
+`bus.stop()`; closing it is the application's job, after the bus stopped. The guarantees hold
+only if that producer is configured like the transport's own: idempotent, `acks` -1 (all in-sync
+replicas) and at most 5 in-flight requests (1 keeps retries from reordering a key). For kafkajs
+that is `kafka.producer({ idempotent: true, maxInFlightRequests: 1 })`, for Confluent
+`kafka.producer({ kafkaJS: { idempotent: true, acks: -1, maxInFlightRequests: 1 } })`. Because the
+transport cannot apply `producerConfig` or `maxInFlightRequests` to it, giving them together with
+`producer` is a configuration error.
+
+## Choosing a client
+
+**Use `@confluentinc/kafka-javascript`** (`client: 'confluent'`): it is maintained and built on
+librdkafka, while `kafkajs` is no longer maintained. Every integration scenario runs against a real
+broker with both clients (publish and subscribe, retries and redelivery, dead letters, codecs,
+per-key order across partitions, start-up failure, close, interoperability and two services
+through the bus); Confluent is tested with 1.11.0, the lowest version the peer range allows.
+
+What the transport does differently for Confluent, because its KafkaJS-compatible API differs:
+
+- `idempotent`, `maxInFlightRequests` and `acks: -1` go in the producer's `kafkaJS` block, and
+  `groupId` and `fromBeginning` in the consumer's: Confluent rejects them at the top level, in
+  `send` and in `subscribe`. They are also set as librdkafka properties (`enable.idempotence`,
+  `max.in.flight`, `acks`, `group.id`, `auto.offset.reset`), which Confluent applies after the
+  block. Your own `kafkaJS` block and properties are merged below them.
+- The group join and rebalances are followed through `rebalance_cb` (Confluent has no
+  instrumentation events), so `subscribe` still waits for the join and `isConsuming()` still turns
+  `false` during a rebalance.
+- Confluent never gives up on a consumer: a failed `eachMessage` makes it seek back and redeliver
+  the record, and it reports no crash. The restart options therefore never apply.
+- `heartbeat()` is a no-op there; librdkafka heartbeats in the background. Keep the time a record
+  can spend in retries (`maxAttempts * maxBackoff`) well below `max.poll.interval.ms` (5 minutes by
+  default) instead of `sessionTimeout`.
+- Its producer partitions keys with librdkafka's `murmur2_random`, which librdkafka documents as
+  Java compatible, like kafkajs' `DefaultPartitioner`. `createPartitioner` does not exist there.
+
+Moving from kafkajs: create the client with `new KafkaJS.Kafka({ kafkaJS: { brokers, clientId } })`,
+add `client: 'confluent'`, and move KafkaJS-style settings in `producerConfig` and `consumerConfig`
+into a `kafkaJS` block (drop `createPartitioner`). The wire format is unchanged.
 
 ## Operations
 
@@ -313,31 +367,31 @@ kafkajs as the supported client.
   consumer group.
 - **Session timeout**: the retry loop heartbeats between attempts. Keep
   `maxAttempts * maxBackoff` well below the consumer `sessionTimeout` (30 s default).
-- **Concurrency**: one record at a time per partition (kafkajs default); scale with partitions and
+- **Concurrency**: one record at a time per partition (the default of both clients); scale with partitions and
   service replicas in the same group. With `partitionsConsumedConcurrently > 1` in `consumerConfig`
   the stall state is tracked per partition.
 - **Retention vs. inbox**: Kafka may redeliver old records after rebalances or offset resets. The
   bus deduplicates on the event id for as long as it keeps completed deliveries
   (`retention.completed`, default 7 days), so keep that longer than your redelivery window.
-- **Partitioner warning** (kafkajs 2.x): pass `createPartitioner: Partitioners.DefaultPartitioner`
+- **Partitioner warning** (kafkajs 2.x only): pass `createPartitioner: Partitioners.DefaultPartitioner`
   in `producerConfig` to silence the legacy partitioner warning.
 
 ## Troubleshooting
 
 - `did not join group ... within 30000ms` at start: broker unreachable, wrong credentials, or the
-  group is rebalancing for longer than `startupTimeout`. Check the kafkajs connection settings.
+  group is rebalancing for longer than `startupTimeout`. Check the client's connection settings.
 - `isConsuming()` stays `false` with `Giving up for now` logs: `RelayBus.receive` keeps failing
   (database down?). Fix the store; consumption resumes by itself.
 - `Dead-lettered malformed message` logs: a producer on that topic is not writing CloudEvents binary
   mode with `content-type: application/json`. Inspect `<topic>.dead` and its `relaybus-invalid-reason`.
 - `Consumer crashed ... restart=false` followed by `Consumer restarted`: the client gave up and the
-  transport recovered it. Repeated `Restart failed` logs mean the broker or credentials are the problem.
+  transport recovered it (kafkajs only). Repeated `Restart failed` logs mean the broker or credentials are the problem.
 - `PublishTimeoutError`: no broker ack within `publishTimeout`; the outbox retries.
 - Events appear twice in logs but handlers run once: expected, the inbox deduplicates on the id.
 
 ## When to use @relaybus/kafka, and when not to
 
-Use it to send events between services through Kafka without a dual write between your database and the broker. It is a transport, not a store: it needs `@relaybus/core` and a store, and `@confluentinc/kafka-javascript` support is experimental and untested.
+Use it to send events between services through Kafka without a dual write between your database and the broker. It is a transport, not a store: it needs `@relaybus/core`, a store and one of the two clients, `@confluentinc/kafka-javascript` (recommended) or `kafkajs`.
 
 Documentation: <https://berkcansavur.github.io/relaybus/packages/kafka>
 

package/dist/index.cjs

@@ -437,15 +437,30 @@ function jsonEnvelope(mapping) {
     transportIds: [RECORD_POSITION],
     failures: { invalid, unencodable: (reason) => new UnencodableEventError(reason) }
   });
-  const contentType = mapping.contentType ?? DEFAULT_CONTENT_TYPE;
+  const contentType = mapping.contentTypeHeader === false ? {} : { [CONTENT_TYPE]: mapping.contentType ?? DEFAULT_CONTENT_TYPE };
   const stampsTime = mapping.occurredAt === void 0;
-  const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { [CONTENT_TYPE]: contentType, [EVENT_ID_HEADER]: envelope.id } : { [CONTENT_TYPE]: contentType };
+  const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { ...contentType, [EVENT_ID_HEADER]: envelope.id } : contentType;
+  const recordKey = resolveRecordKey(mapping.recordKey);
   return {
-    encode: (envelope) => encodeJson(envelope, { format, stampsTime, ownHeaders }),
+    encode: (envelope) => encodeJson(envelope, { format, stampsTime, ownHeaders, recordKey }),
     decode: (record) => decodeJson(record, format)
   };
 }
-function encodeJson(envelope, { format, stampsTime, ownHeaders }) {
+function resolveRecordKey(recordKey) {
+  if (recordKey === void 0 || recordKey === "event-key") {
+    return (envelope) => envelope.key;
+  }
+  if (recordKey === "none") {
+    return () => null;
+  }
+  if (typeof recordKey === "function") {
+    return recordKey;
+  }
+  throw new core.ConfigurationError(
+    `jsonEnvelope recordKey must be 'event-key', 'none' or a function, received '${String(recordKey)}'`
+  );
+}
+function encodeJson(envelope, { format, stampsTime, ownHeaders, recordKey }) {
   assertRepresentable(envelope);
   const { body, headers } = format.write(envelope);
   const taken = [CONTENT_TYPE, EVENT_ID_HEADER].find((name) => Object.hasOwn(headers, name));
@@ -455,7 +470,7 @@ function encodeJson(envelope, { format, stampsTime, ownHeaders }) {
     );
   }
   return {
-    key: envelope.key,
+    key: recordKey(envelope),
     value: serializeJson(envelope, body),
     headers: { ...headers, ...ownHeaders(envelope) },
     ...stampsTime ? { timestamp: recordTimestamp(envelope) } : {}
@@ -486,9 +501,6 @@ function recordTimestamp(envelope) {
   formatTime(envelope);
   return String(envelope.occurredAt.getTime());
 }
-function tracksMembership(consumer) {
-  return consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0;
-}
 var GroupJoinWatch = class {
   #settings;
   #waiter;
@@ -514,7 +526,7 @@ var GroupJoinWatch = class {
     eachMessage
   }) {
     await consumer.connect();
-    await consumer.subscribe({ topics, fromBeginning });
+    await consumer.subscribe(this.#settings.dialect.subscription(topics, fromBeginning));
     const joined = this.#wait(consumer);
     try {
       await consumer.run({ eachMessage });
@@ -524,7 +536,7 @@ var GroupJoinWatch = class {
     }
   }
   #wait(consumer) {
-    if (!tracksMembership(consumer)) {
+    if (!this.#settings.dialect.tracksMembership(consumer)) {
       return { promise: Promise.resolve(), dispose: () => void 0 };
     }
     const { name, groupId, timings } = this.#settings;
@@ -598,7 +610,7 @@ var KafkaProducerSession = class {
         delayMs: timings.publishTimeoutMs
       });
     });
-    const sending = producer.send(record);
+    const sending = producer.send(this.#settings.dialect.record(record));
     sending.catch(() => void 0);
     try {
       await Promise.race([sending, timeout]);
@@ -607,8 +619,12 @@ var KafkaProducerSession = class {
     }
   }
   async #open() {
-    const { kafka, producerConfig, maxInFlightRequests } = this.#settings;
-    const producer = kafka.producer({ ...producerConfig, idempotent: true, maxInFlightRequests });
+    const { kafka, dialect, injectedProducer } = this.#settings;
+    if (injectedProducer !== void 0) {
+      this.#connected = true;
+      return withoutLifecycle(injectedProducer);
+    }
+    const producer = kafka.producer(dialect.producerConfig(this.#settings));
     try {
       await producer.connect();
     } catch (error) {
@@ -619,6 +635,88 @@ var KafkaProducerSession = class {
     return producer;
   }
 };
+function withoutLifecycle(producer) {
+  return {
+    send: (record) => producer.send(record),
+    disconnect: () => Promise.resolve()
+  };
+}
+var KAFKAJS_DIALECT = {
+  producerConfig: ({ producerConfig, maxInFlightRequests }) => ({
+    ...producerConfig,
+    idempotent: true,
+    maxInFlightRequests
+  }),
+  consumerConfig: ({ consumerConfig, groupId }) => ({ ...consumerConfig, groupId }),
+  subscription: (topics, fromBeginning) => ({ topics, fromBeginning }),
+  record: (record) => record,
+  tracksMembership: (consumer) => consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0,
+  observe: (consumer, membership) => {
+    if (!KAFKAJS_DIALECT.tracksMembership(consumer)) {
+      return;
+    }
+    const names = consumer.events;
+    const listen = (name, listener) => {
+      if (name !== void 0) {
+        consumer.on(name, listener);
+      }
+    };
+    listen(names.GROUP_JOIN, () => membership.joined());
+    listen(names.REBALANCING, () => membership.rebalancing());
+    listen(names.CRASH, (event) => membership.crashed(event));
+    listen(names.STOP, () => membership.left());
+    listen(names.DISCONNECT, () => membership.left());
+  }
+};
+var ASSIGN_PARTITIONS = -175;
+var REVOKE_PARTITIONS = -174;
+var CONFLUENT_DIALECT = {
+  producerConfig: ({ producerConfig, maxInFlightRequests }) => {
+    const user = producerConfig;
+    return {
+      ...user,
+      kafkaJS: { ...user.kafkaJS, idempotent: true, maxInFlightRequests, acks: -1 },
+      "enable.idempotence": true,
+      "max.in.flight": maxInFlightRequests,
+      acks: -1
+    };
+  },
+  consumerConfig: ({ consumerConfig, groupId, fromBeginning }, membership) => {
+    const user = consumerConfig;
+    const userCallback = typeof user.rebalance_cb === "function" ? user.rebalance_cb : void 0;
+    return {
+      ...user,
+      kafkaJS: { ...user.kafkaJS, groupId, fromBeginning },
+      "group.id": groupId,
+      "auto.offset.reset": fromBeginning ? "earliest" : "latest",
+      rebalance_cb: (...args) => {
+        if (args[0].code === ASSIGN_PARTITIONS) {
+          membership.joined();
+        } else if (args[0].code === REVOKE_PARTITIONS) {
+          membership.rebalancing();
+        }
+        return userCallback?.(...args);
+      }
+    };
+  },
+  subscription: (topics) => ({ topics }),
+  record: ({ acks: _acks, ...record }) => record,
+  tracksMembership: () => true,
+  observe: () => void 0
+};
+function clientDialect(client) {
+  if (client === void 0 || client === "kafkajs") {
+    return KAFKAJS_DIALECT;
+  }
+  if (client === "confluent") {
+    return CONFLUENT_DIALECT;
+  }
+  throw new core.ConfigurationError(
+    `Unknown Kafka client '${String(client)}'; use 'kafkajs' or 'confluent'`
+  );
+}
+
+// src/kafka-settings.ts
 var DEFAULTS = {
   name: "kafka",
   maxInFlightRequests: 1,
@@ -636,6 +734,8 @@ function resolveKafkaSettings(options) {
   return {
     name: options.name ?? DEFAULTS.name,
     kafka: options.kafka,
+    injectedProducer: resolveInjectedProducer(options),
+    dialect: clientDialect(options.client),
     groupId: options.groupId,
     fromBeginning: options.fromBeginning ?? true,
     maxInFlightRequests: internal.BoundedInteger.of(
@@ -650,6 +750,23 @@ function resolveKafkaSettings(options) {
     timings: resolveTimings(options)
   };
 }
+function resolveInjectedProducer(options) {
+  const { producer, kafka, producerConfig, maxInFlightRequests } = options;
+  if (producer === void 0) {
+    if (kafka === void 0) {
+      throw new core.ConfigurationError(
+        "Give 'kafka', or an already-connected 'producer' to publish with"
+      );
+    }
+    return void 0;
+  }
+  if (producerConfig !== void 0 || maxInFlightRequests !== void 0) {
+    throw new core.ConfigurationError(
+      "Give 'producerConfig' and 'maxInFlightRequests' to a producer you inject yourself"
+    );
+  }
+  return producer;
+}
 function resolveCodec({ codec, decode }) {
   if (decode === void 0) {
     return codec ?? cloudEventsBinary;
@@ -719,7 +836,7 @@ function assertCanSubscribe(state, {
   settings,
   destinations
 }) {
-  const { name, groupId, invalidPolicy } = settings;
+  const { name, groupId, invalidPolicy, kafka } = settings;
   if (state === "closed") {
     throw new TransportClosedError(`Transport '${name}' is closed`);
   }
@@ -728,6 +845,9 @@ function assertCanSubscribe(state, {
       `Transport '${name}' already subscribed; list every topic in one subscription or use one KafkaTransport per consumer group`
     );
   }
+  if (kafka === void 0) {
+    throw new core.ConfigurationError(`Transport '${name}' needs 'kafka' to create its consumer`);
+  }
   if (groupId === void 0 || groupId.length === 0) {
     throw new core.ConfigurationError(`Transport '${name}' needs a groupId to subscribe`);
   }
@@ -746,10 +866,7 @@ function assertCanSubscribe(state, {
 
 // src/kafka-records.ts
 function decodeRecord(codec, { topic, partition, message }) {
-  if (codec === cloudEventsBinary) {
-    return decodeEnvelope(message);
-  }
-  const record = {
+  return decodeKafkaRecord(codec, {
     topic,
     partition,
     offset: message.offset,
@@ -757,7 +874,12 @@ function decodeRecord(codec, { topic, partition, message }) {
     value: message.value,
     headers: message.headers,
     timestamp: message.timestamp
-  };
+  });
+}
+function decodeKafkaRecord(codec, record) {
+  if (codec === cloudEventsBinary) {
+    return decodeEnvelope(record);
+  }
   return completeDecodedEvent(codec.decode(record), record);
 }
 function deadLetterRecord({ topic, partition, message }, {
@@ -895,10 +1017,12 @@ var KafkaTransport = class extends internal.SubscribingTransport {
   async #subscribe({ destinations, onMessage, hooks }) {
     assertCanSubscribe(this.#state, { settings: this.#settings, destinations });
     this.#state = "subscribed";
-    const { kafka, consumerConfig, groupId } = this.#settings;
-    const consumer = kafka.consumer({ ...consumerConfig, groupId });
+    const { kafka, dialect, groupId } = this.#settings;
+    const membership = this.#observeConsumer();
+    const consumer = kafka.consumer(dialect.consumerConfig(this.#settings, membership));
     this.#consumer = consumer;
-    this.#observeConsumer(consumer);
+    this.#groupJoined = !dialect.tracksMembership(consumer);
+    dialect.observe(consumer, membership);
     const start = () => this.#groupJoin.start(consumer, {
       topics: [...destinations],
       fromBeginning: this.#settings.fromBeginning,
@@ -1039,34 +1163,25 @@ var KafkaTransport = class extends internal.SubscribingTransport {
       }
     }
   }
-  #observeConsumer(consumer) {
-    if (!tracksMembership(consumer)) {
-      this.#groupJoined = true;
-      return;
-    }
+  /** What the transport does when the consumer joins, rebalances, crashes or leaves its group. */
+  #observeConsumer() {
     const { logger, groupId } = this.#settings;
-    const names = consumer.events;
-    const listen = (name, listener) => {
-      if (name !== void 0) {
-        consumer.on(name, listener);
+    return {
+      joined: () => {
+        this.#groupJoined = true;
+        this.#groupJoin.joined();
+        logger.log(`[KafkaTransport](observeConsumer) Joined group | group=${groupId}`);
+      },
+      rebalancing: () => {
+        this.#groupJoined = false;
+        this.#stalledPartitions.clear();
+        logger.warn(`[KafkaTransport](observeConsumer) Rebalancing | group=${groupId}`);
+      },
+      crashed: (event) => this.#handleCrash(event),
+      left: () => {
+        this.#groupJoined = false;
       }
     };
-    listen(names.GROUP_JOIN, () => {
-      this.#groupJoined = true;
-      this.#groupJoin.joined();
-      logger.log(`[KafkaTransport](observeConsumer) Joined group | group=${groupId}`);
-    });
-    listen(names.REBALANCING, () => {
-      this.#groupJoined = false;
-      this.#stalledPartitions.clear();
-      logger.warn(`[KafkaTransport](observeConsumer) Rebalancing | group=${groupId}`);
-    });
-    listen(names.CRASH, (event) => this.#handleCrash(event));
-    const leftGroup = () => {
-      this.#groupJoined = false;
-    };
-    listen(names.STOP, leftGroup);
-    listen(names.DISCONNECT, leftGroup);
   }
   #handleCrash(event) {
     const { error, restart } = event.payload ?? {};
@@ -1128,6 +1243,7 @@ exports.cloudEventsBinary = cloudEventsBinary;
 exports.cloudEventsStructured = cloudEventsStructured;
 exports.decodeCloudEvent = decodeCloudEvent;
 exports.decodeEnvelope = decodeEnvelope;
+exports.decodeKafkaRecord = decodeKafkaRecord;
 exports.encodeEnvelope = encodeEnvelope;
 exports.jsonEnvelope = jsonEnvelope;
 exports.plainJsonDecoder = plainJsonDecoder;