@venizia/ignis-docs 0.0.7 → 0.0.8-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/mcp-server/common/paths.d.ts +4 -2
- package/dist/mcp-server/common/paths.d.ts.map +1 -1
- package/dist/mcp-server/common/paths.js +8 -6
- package/dist/mcp-server/common/paths.js.map +1 -1
- package/dist/mcp-server/helpers/docs.helper.d.ts.map +1 -1
- package/dist/mcp-server/helpers/docs.helper.js +1 -1
- package/dist/mcp-server/helpers/docs.helper.js.map +1 -1
- package/dist/mcp-server/tools/base.tool.d.ts +1 -1
- package/dist/mcp-server/tools/docs/get-document-content.tool.d.ts +1 -1
- package/dist/mcp-server/tools/docs/get-document-content.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/docs/get-document-content.tool.js +7 -7
- package/dist/mcp-server/tools/docs/get-document-metadata.tool.js +3 -3
- package/dist/mcp-server/tools/docs/get-package-overview.tool.d.ts +1 -1
- package/dist/mcp-server/tools/docs/get-package-overview.tool.js +1 -1
- package/dist/mcp-server/tools/docs/search-documents.tool.d.ts +1 -1
- package/dist/mcp-server/tools/docs/search-documents.tool.js +1 -1
- package/dist/mcp-server/tools/docs/search-documents.tool.js.map +1 -1
- package/dist/mcp-server/tools/github/list-project-files.tool.d.ts +1 -1
- package/dist/mcp-server/tools/github/list-project-files.tool.js +1 -1
- package/dist/mcp-server/tools/github/list-project-files.tool.js.map +1 -1
- package/dist/mcp-server/tools/github/search-code.tool.d.ts +1 -1
- package/dist/mcp-server/tools/github/search-code.tool.js +1 -1
- package/dist/mcp-server/tools/github/search-code.tool.js.map +1 -1
- package/package.json +9 -9
- package/wiki/best-practices/api-usage-examples.md +9 -9
- package/wiki/best-practices/architectural-patterns.md +19 -3
- package/wiki/best-practices/architecture-decisions.md +6 -6
- package/wiki/best-practices/code-style-standards/advanced-patterns.md +1 -1
- package/wiki/best-practices/code-style-standards/control-flow.md +1 -1
- package/wiki/best-practices/code-style-standards/function-patterns.md +2 -2
- package/wiki/best-practices/code-style-standards/index.md +2 -2
- package/wiki/best-practices/code-style-standards/naming-conventions.md +1 -1
- package/wiki/best-practices/code-style-standards/route-definitions.md +4 -4
- package/wiki/best-practices/data-modeling.md +1 -1
- package/wiki/best-practices/deployment-strategies.md +1 -1
- package/wiki/best-practices/error-handling.md +2 -2
- package/wiki/best-practices/performance-optimization.md +3 -3
- package/wiki/best-practices/security-guidelines.md +2 -2
- package/wiki/best-practices/troubleshooting-tips.md +1 -1
- package/wiki/{references → extensions}/components/authentication/api.md +12 -20
- package/wiki/{references → extensions}/components/authentication/errors.md +1 -1
- package/wiki/{references → extensions}/components/authentication/index.md +5 -8
- package/wiki/{references → extensions}/components/authentication/usage.md +20 -36
- package/wiki/{references → extensions}/components/authorization/api.md +62 -13
- package/wiki/{references → extensions}/components/authorization/errors.md +12 -7
- package/wiki/{references → extensions}/components/authorization/index.md +93 -6
- package/wiki/{references → extensions}/components/authorization/usage.md +42 -4
- package/wiki/{references → extensions}/components/health-check.md +5 -4
- package/wiki/{references → extensions}/components/index.md +2 -0
- package/wiki/{references → extensions}/components/mail/index.md +1 -1
- package/wiki/{references → extensions}/components/request-tracker.md +1 -1
- package/wiki/{references → extensions}/components/socket-io/api.md +2 -2
- package/wiki/{references → extensions}/components/socket-io/errors.md +2 -0
- package/wiki/{references → extensions}/components/socket-io/index.md +24 -20
- package/wiki/{references → extensions}/components/socket-io/usage.md +2 -2
- package/wiki/{references → extensions}/components/static-asset/api.md +14 -15
- package/wiki/{references → extensions}/components/static-asset/errors.md +3 -1
- package/wiki/{references → extensions}/components/static-asset/index.md +158 -89
- package/wiki/{references → extensions}/components/static-asset/usage.md +8 -5
- package/wiki/{references → extensions}/components/swagger.md +3 -3
- package/wiki/{references → extensions}/components/template/index.md +4 -4
- package/wiki/{references → extensions}/components/template/setup-page.md +1 -1
- package/wiki/{references → extensions}/components/template/single-page.md +1 -1
- package/wiki/{references → extensions}/components/websocket/api.md +7 -6
- package/wiki/{references → extensions}/components/websocket/errors.md +17 -3
- package/wiki/{references → extensions}/components/websocket/index.md +17 -11
- package/wiki/{references → extensions}/components/websocket/usage.md +2 -2
- package/wiki/{references → extensions}/helpers/crypto/index.md +1 -1
- package/wiki/{references → extensions}/helpers/env/index.md +9 -5
- package/wiki/{references → extensions}/helpers/error/index.md +2 -7
- package/wiki/{references → extensions}/helpers/index.md +18 -6
- package/wiki/{references → extensions}/helpers/kafka/admin.md +13 -1
- package/wiki/{references → extensions}/helpers/kafka/consumer.md +32 -31
- package/wiki/{references → extensions}/helpers/kafka/examples.md +20 -20
- package/wiki/{references → extensions}/helpers/kafka/index.md +61 -54
- package/wiki/{references → extensions}/helpers/kafka/producer.md +21 -20
- package/wiki/{references → extensions}/helpers/kafka/schema-registry.md +25 -25
- package/wiki/{references → extensions}/helpers/logger/index.md +2 -2
- package/wiki/{references → extensions}/helpers/queue/index.md +400 -4
- package/wiki/{references → extensions}/helpers/storage/api.md +170 -10
- package/wiki/{references → extensions}/helpers/storage/index.md +44 -8
- package/wiki/{references → extensions}/helpers/template/index.md +1 -1
- package/wiki/{references → extensions}/helpers/testing/index.md +4 -4
- package/wiki/{references → extensions}/helpers/types/index.md +63 -16
- package/wiki/{references → extensions}/helpers/websocket/index.md +1 -1
- package/wiki/extensions/index.md +48 -0
- package/wiki/guides/core-concepts/application/bootstrapping.md +55 -37
- package/wiki/guides/core-concepts/application/index.md +95 -35
- package/wiki/guides/core-concepts/components-guide.md +23 -19
- package/wiki/guides/core-concepts/components.md +34 -10
- package/wiki/guides/core-concepts/dependency-injection.md +99 -34
- package/wiki/guides/core-concepts/grpc-controllers.md +295 -0
- package/wiki/guides/core-concepts/persistent/datasources.md +37 -19
- package/wiki/guides/core-concepts/persistent/index.md +6 -6
- package/wiki/guides/core-concepts/persistent/models.md +50 -6
- package/wiki/guides/core-concepts/persistent/repositories.md +83 -8
- package/wiki/guides/core-concepts/persistent/transactions.md +39 -8
- package/wiki/guides/core-concepts/{controllers.md → rest-controllers.md} +32 -35
- package/wiki/guides/core-concepts/services.md +19 -6
- package/wiki/guides/get-started/5-minute-quickstart.md +17 -17
- package/wiki/guides/get-started/philosophy.md +1 -1
- package/wiki/guides/index.md +2 -2
- package/wiki/guides/reference/glossary.md +7 -7
- package/wiki/guides/reference/mcp-docs-server.md +1 -1
- package/wiki/guides/tutorials/building-a-crud-api.md +45 -39
- package/wiki/guides/tutorials/complete-installation.md +74 -51
- package/wiki/guides/tutorials/ecommerce-api.md +39 -30
- package/wiki/guides/tutorials/realtime-chat.md +12 -13
- package/wiki/guides/tutorials/testing.md +2 -2
- package/wiki/index.md +4 -3
- package/wiki/references/base/application.md +341 -21
- package/wiki/references/base/bootstrapping.md +43 -13
- package/wiki/references/base/components.md +259 -8
- package/wiki/references/base/controllers.md +556 -253
- package/wiki/references/base/datasources.md +159 -79
- package/wiki/references/base/dependency-injection.md +299 -48
- package/wiki/references/base/filter-system/application-usage.md +18 -2
- package/wiki/references/base/filter-system/array-operators.md +14 -6
- package/wiki/references/base/filter-system/comparison-operators.md +9 -3
- package/wiki/references/base/filter-system/default-filter.md +28 -3
- package/wiki/references/base/filter-system/fields-order-pagination.md +17 -13
- package/wiki/references/base/filter-system/index.md +169 -11
- package/wiki/references/base/filter-system/json-filtering.md +51 -18
- package/wiki/references/base/filter-system/list-operators.md +4 -3
- package/wiki/references/base/filter-system/logical-operators.md +7 -2
- package/wiki/references/base/filter-system/null-operators.md +50 -0
- package/wiki/references/base/filter-system/quick-reference.md +82 -243
- package/wiki/references/base/filter-system/range-operators.md +7 -1
- package/wiki/references/base/filter-system/tips.md +34 -7
- package/wiki/references/base/filter-system/use-cases.md +6 -5
- package/wiki/references/base/grpc-controllers.md +984 -0
- package/wiki/references/base/index.md +32 -24
- package/wiki/references/base/middleware.md +347 -0
- package/wiki/references/base/models.md +390 -46
- package/wiki/references/base/providers.md +14 -14
- package/wiki/references/base/repositories/advanced.md +195 -69
- package/wiki/references/base/repositories/index.md +447 -12
- package/wiki/references/base/repositories/mixins.md +103 -98
- package/wiki/references/base/repositories/relations.md +129 -45
- package/wiki/references/base/repositories/soft-deletable.md +104 -23
- package/wiki/references/base/services.md +94 -14
- package/wiki/references/index.md +12 -10
- package/wiki/references/quick-reference.md +98 -65
- package/wiki/references/utilities/crypto.md +21 -4
- package/wiki/references/utilities/date.md +25 -7
- package/wiki/references/utilities/index.md +26 -24
- package/wiki/references/utilities/jsx.md +54 -54
- package/wiki/references/utilities/module.md +8 -6
- package/wiki/references/utilities/parse.md +16 -9
- package/wiki/references/utilities/performance.md +22 -7
- package/wiki/references/utilities/promise.md +19 -16
- package/wiki/references/utilities/request.md +48 -26
- package/wiki/references/utilities/schema.md +69 -6
- package/wiki/references/utilities/statuses.md +131 -140
- /package/wiki/{references → extensions}/components/mail/api.md +0 -0
- /package/wiki/{references → extensions}/components/mail/errors.md +0 -0
- /package/wiki/{references → extensions}/components/mail/usage.md +0 -0
- /package/wiki/{references → extensions}/components/template/api-page.md +0 -0
- /package/wiki/{references → extensions}/components/template/errors-page.md +0 -0
- /package/wiki/{references → extensions}/components/template/usage-page.md +0 -0
- /package/wiki/{references → extensions}/helpers/cron/index.md +0 -0
- /package/wiki/{references → extensions}/helpers/inversion/index.md +0 -0
- /package/wiki/{references → extensions}/helpers/network/api.md +0 -0
- /package/wiki/{references → extensions}/helpers/network/index.md +0 -0
- /package/wiki/{references → extensions}/helpers/redis/index.md +0 -0
- /package/wiki/{references → extensions}/helpers/socket-io/api.md +0 -0
- /package/wiki/{references → extensions}/helpers/socket-io/index.md +0 -0
- /package/wiki/{references → extensions}/helpers/template/single-page.md +0 -0
- /package/wiki/{references → extensions}/helpers/uid/index.md +0 -0
- /package/wiki/{references → extensions}/helpers/websocket/api.md +0 -0
- /package/wiki/{references → extensions}/helpers/worker-thread/index.md +0 -0
- /package/wiki/{references → extensions}/src-details/mcp-server.md +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Kafka
|
|
2
2
|
|
|
3
|
-
Apache Kafka event streaming with producer, consumer, admin, and schema registry helpers. Built on [`@platformatic/kafka`](https://github.com/platformatic/kafka) v1.30.0
|
|
3
|
+
Apache Kafka event streaming with producer, consumer, admin, and schema registry helpers. Built on [`@platformatic/kafka`](https://github.com/platformatic/kafka) v1.30.0 -- a pure TypeScript Kafka client with zero native dependencies.
|
|
4
4
|
|
|
5
5
|
## Overview
|
|
6
6
|
|
|
@@ -16,9 +16,10 @@ The Kafka module provides four helper classes built on a shared `BaseKafkaHelper
|
|
|
16
16
|
All helpers (except schema registry) extend `BaseKafkaHelper` which provides:
|
|
17
17
|
|
|
18
18
|
- **Scoped logging** via `BaseHelper` (Winston with daily rotation)
|
|
19
|
-
- **Health tracking**
|
|
20
|
-
- **Broker event callbacks**
|
|
21
|
-
- **
|
|
19
|
+
- **Health tracking** -- per-broker connection tracking via `isHealthy()`, `isReady()`, `getHealthStatus()`, `getConnectedBrokerCount()`
|
|
20
|
+
- **Broker event callbacks** -- `onBrokerConnect`, `onBrokerDisconnect`
|
|
21
|
+
- **Broker failure tracking** -- automatic `configureBrokerFailed()` sets status to `'disconnected'` only when all brokers are gone
|
|
22
|
+
- **Graceful shutdown** -- timeout-based with force fallback
|
|
22
23
|
- **Sensible defaults** via `KafkaDefaults` constants
|
|
23
24
|
- **Factory pattern** via `newInstance()` static method
|
|
24
25
|
|
|
@@ -27,7 +28,7 @@ Use `getProducer()`, `getConsumer()`, or `getAdmin()` to access the full underly
|
|
|
27
28
|
### Import Path
|
|
28
29
|
|
|
29
30
|
```typescript
|
|
30
|
-
// Helpers & constants
|
|
31
|
+
// Helpers & constants (via subpath export)
|
|
31
32
|
import {
|
|
32
33
|
KafkaProducerHelper,
|
|
33
34
|
KafkaConsumerHelper,
|
|
@@ -84,6 +85,9 @@ import type {
|
|
|
84
85
|
} from '@platformatic/kafka';
|
|
85
86
|
```
|
|
86
87
|
|
|
88
|
+
> [!NOTE]
|
|
89
|
+
> Kafka helpers are **not** re-exported from the main `@venizia/ignis-helpers` entry point. You must use the `@venizia/ignis-helpers/kafka` subpath import. This keeps the optional `@platformatic/kafka` peer dependency isolated for tree-shaking.
|
|
90
|
+
|
|
87
91
|
### Installation
|
|
88
92
|
|
|
89
93
|
```bash
|
|
@@ -96,13 +100,12 @@ bun add @platformatic/kafka
|
|
|
96
100
|
|
|
97
101
|
```
|
|
98
102
|
BaseHelper (scoped logging, identifier)
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
└── KafkaSchemaRegistryHelper<K,V,HK,HV> (no broker connection)
|
|
103
|
+
+-- BaseKafkaHelper<TClient> (health tracking, broker events, graceful shutdown)
|
|
104
|
+
| +-- KafkaProducerHelper<K,V,HK,HV>
|
|
105
|
+
| +-- KafkaConsumerHelper<K,V,HK,HV>
|
|
106
|
+
| +-- KafkaAdminHelper
|
|
107
|
+
|
|
|
108
|
+
+-- KafkaSchemaRegistryHelper<K,V,HK,HV> (no broker connection)
|
|
106
109
|
```
|
|
107
110
|
|
|
108
111
|
### BaseKafkaHelper
|
|
@@ -112,21 +115,25 @@ All Kafka helpers (except schema registry) extend `BaseKafkaHelper<TClient>`, wh
|
|
|
112
115
|
```typescript
|
|
113
116
|
abstract class BaseKafkaHelper<TClient extends Base<BaseOptions>> extends BaseHelper {
|
|
114
117
|
// Health
|
|
115
|
-
isHealthy(): boolean;
|
|
116
|
-
isReady(): boolean;
|
|
117
|
-
getHealthStatus(): TKafkaHealthStatus;
|
|
118
|
+
isHealthy(): boolean; // true when at least one broker is connected
|
|
119
|
+
isReady(): boolean; // healthStatus === 'connected' (consumer overrides: + isActive())
|
|
120
|
+
getHealthStatus(): TKafkaHealthStatus; // 'connected' | 'disconnected' | 'unknown'
|
|
121
|
+
getConnectedBrokerCount(): number; // number of currently connected brokers
|
|
118
122
|
|
|
119
123
|
// Shutdown (used by subclasses)
|
|
120
124
|
protected closeClient(): Promise<void>;
|
|
121
125
|
protected gracefulCloseClient(): Promise<void>; // races closeClient vs shutdownTimeout
|
|
126
|
+
protected resetHealthState(): void; // clears broker tracking + sets 'disconnected'
|
|
122
127
|
}
|
|
123
128
|
```
|
|
124
129
|
|
|
130
|
+
Health tracking uses a **per-broker connection set** (`host:port` keys). A single idle broker disconnect does not make the client unhealthy -- only when **all** brokers are disconnected does `isHealthy()` return `false`.
|
|
131
|
+
|
|
125
132
|
Health status transitions automatically via broker events:
|
|
126
|
-
- `client:broker:connect`
|
|
127
|
-
- `client:broker:disconnect`
|
|
128
|
-
- `client:broker:failed`
|
|
129
|
-
- `close()`
|
|
133
|
+
- `client:broker:connect` -> adds broker, sets `healthStatus` to `'connected'`
|
|
134
|
+
- `client:broker:disconnect` -> removes broker, sets `healthStatus` to `'disconnected'` only when all brokers are gone
|
|
135
|
+
- `client:broker:failed` -> removes broker, sets `healthStatus` to `'disconnected'` only when all brokers are gone
|
|
136
|
+
- `close()` -> clears all brokers, sets `healthStatus` to `'disconnected'`
|
|
130
137
|
|
|
131
138
|
## Connection Options
|
|
132
139
|
|
|
@@ -145,15 +152,15 @@ interface IKafkaConnectionOptions extends ConnectionOptions {
|
|
|
145
152
|
|
|
146
153
|
| Option | Type | Default | Description |
|
|
147
154
|
|--------|------|---------|-------------|
|
|
148
|
-
| `bootstrapBrokers` | `string[]` |
|
|
149
|
-
| `clientId` | `string` |
|
|
155
|
+
| `bootstrapBrokers` | `string[]` | -- | Kafka broker addresses (`host:port`). **Required** |
|
|
156
|
+
| `clientId` | `string` | -- | Unique client identifier. **Required** |
|
|
150
157
|
| `retries` | `number` | `3` | Number of connection retries before failing |
|
|
151
158
|
| `retryDelay` | `number` | `1000` | Delay between retries in milliseconds |
|
|
152
|
-
| `sasl` | `SASLOptions` |
|
|
153
|
-
| `tls` | `TLSConnectionOptions` |
|
|
154
|
-
| `ssl` | `TLSConnectionOptions` |
|
|
155
|
-
| `connectTimeout` | `number` |
|
|
156
|
-
| `requestTimeout` | `number` |
|
|
159
|
+
| `sasl` | `SASLOptions` | -- | SASL authentication configuration |
|
|
160
|
+
| `tls` | `TLSConnectionOptions` | -- | TLS/SSL connection options |
|
|
161
|
+
| `ssl` | `TLSConnectionOptions` | -- | Alias for `tls` |
|
|
162
|
+
| `connectTimeout` | `number` | -- | TCP connection timeout in milliseconds |
|
|
163
|
+
| `requestTimeout` | `number` | -- | Kafka request timeout in milliseconds |
|
|
157
164
|
|
|
158
165
|
### Shared Helper Options
|
|
159
166
|
|
|
@@ -163,8 +170,8 @@ These options are available on all three helpers (`IKafkaProducerOptions`, `IKaf
|
|
|
163
170
|
|--------|------|---------|-------------|
|
|
164
171
|
| `identifier` | `string` | `'kafka-{type}'` | Scoped logging identifier |
|
|
165
172
|
| `shutdownTimeout` | `number` | `30000` | Graceful shutdown timeout in ms |
|
|
166
|
-
| `onBrokerConnect` | `TKafkaBrokerEventCallback` |
|
|
167
|
-
| `onBrokerDisconnect` | `TKafkaBrokerEventCallback` |
|
|
173
|
+
| `onBrokerConnect` | `TKafkaBrokerEventCallback` | -- | Called when broker connects |
|
|
174
|
+
| `onBrokerDisconnect` | `TKafkaBrokerEventCallback` | -- | Called when broker disconnects |
|
|
168
175
|
|
|
169
176
|
### SASL Authentication
|
|
170
177
|
|
|
@@ -247,10 +254,10 @@ const helper = KafkaProducerHelper.newInstance({
|
|
|
247
254
|
|
|
248
255
|
| Export | Type | Description |
|
|
249
256
|
|--------|------|-------------|
|
|
250
|
-
| `stringSerializer` | `Serializer<string>` | `string
|
|
251
|
-
| `stringDeserializer` | `Deserializer<string>` | `Buffer
|
|
252
|
-
| `jsonSerializer` | `Serializer<T>` | `object
|
|
253
|
-
| `jsonDeserializer` | `Deserializer<T>` | `Buffer
|
|
257
|
+
| `stringSerializer` | `Serializer<string>` | `string -> Buffer` (UTF-8) |
|
|
258
|
+
| `stringDeserializer` | `Deserializer<string>` | `Buffer -> string` (UTF-8) |
|
|
259
|
+
| `jsonSerializer` | `Serializer<T>` | `object -> Buffer` (JSON.stringify + UTF-8) |
|
|
260
|
+
| `jsonDeserializer` | `Deserializer<T>` | `Buffer -> object` (UTF-8 + JSON.parse) |
|
|
254
261
|
| `stringSerializers` | `Serializers<string, string, string, string>` | All four positions as string |
|
|
255
262
|
| `stringDeserializers` | `Deserializers<string, string, string, string>` | All four positions as string |
|
|
256
263
|
|
|
@@ -397,8 +404,8 @@ import { KafkaDefaults } from '@venizia/ignis-helpers/kafka';
|
|
|
397
404
|
| `STRICT` | `true` | Producer | Fail on unknown topics |
|
|
398
405
|
| `AUTOCREATE_TOPICS` | `false` | Producer | Auto-create topics on produce |
|
|
399
406
|
| `AUTOCOMMIT` | `false` | Consumer | Auto-commit offsets |
|
|
400
|
-
| `SESSION_TIMEOUT` | `
|
|
401
|
-
| `HEARTBEAT_INTERVAL` | `
|
|
407
|
+
| `SESSION_TIMEOUT` | `60000` | Consumer | Session timeout in ms |
|
|
408
|
+
| `HEARTBEAT_INTERVAL` | `10000` | Consumer | Heartbeat interval in ms |
|
|
402
409
|
| `HIGH_WATER_MARK` | `1024` | Consumer | Stream buffer size (messages) |
|
|
403
410
|
| `MIN_BYTES` | `1` | Consumer | Min bytes per fetch |
|
|
404
411
|
| `METADATA_MAX_AGE` | `300000` | Consumer | Metadata cache TTL in ms |
|
|
@@ -453,7 +460,7 @@ import { KafkaAcks } from '@venizia/ignis-helpers/kafka';
|
|
|
453
460
|
|
|
454
461
|
| Constant | Value | Description | Trade-off |
|
|
455
462
|
|----------|-------|-------------|-----------|
|
|
456
|
-
| `NONE` | `0` | No acknowledgment
|
|
463
|
+
| `NONE` | `0` | No acknowledgment -- fire-and-forget | Fastest, no durability guarantee |
|
|
457
464
|
| `LEADER` | `1` | Leader broker acknowledges | Fast, leader-durable |
|
|
458
465
|
| `ALL` | `-1` | All in-sync replicas acknowledge | Slowest, fully durable |
|
|
459
466
|
|
|
@@ -468,7 +475,7 @@ import { KafkaGroupProtocol } from '@venizia/ignis-helpers/kafka';
|
|
|
468
475
|
| Constant | Value | Description |
|
|
469
476
|
|----------|-------|-------------|
|
|
470
477
|
| `CLASSIC` | `'classic'` | Classic consumer group protocol (default, all Kafka versions) |
|
|
471
|
-
| `CONSUMER` | `'consumer'` | New consumer group protocol
|
|
478
|
+
| `CONSUMER` | `'consumer'` | New consumer group protocol -- KIP-848 (Kafka 3.7+) |
|
|
472
479
|
|
|
473
480
|
### Derived Types
|
|
474
481
|
|
|
@@ -558,16 +565,16 @@ const consumer = KafkaConsumerHelper.newInstance({
|
|
|
558
565
|
| Admin | Producer | Consumer |
|
|
559
566
|
|-------|----------|----------|
|
|
560
567
|
| `admin.getAdmin()` | `producer.getProducer()` | `consumer.getConsumer()` |
|
|
561
|
-
|
|
|
562
|
-
|
|
|
563
|
-
|
|
|
564
|
-
|
|
|
568
|
+
| -- | `producer.getProducer().send(...)` | `await consumer.start({ topics: ['t1'] })` |
|
|
569
|
+
| -- | `await producer.runInTransaction(async ({ send, addConsumer, addOffset }) => { ... })` | `consumer.startLagMonitoring({ topics: ['t1'], interval: 10_000 })` |
|
|
570
|
+
| -- | -- | `consumer.stopLagMonitoring()` |
|
|
571
|
+
| -- | -- | `consumer.getStream()` |
|
|
565
572
|
|
|
566
573
|
### Health Checks
|
|
567
574
|
|
|
568
575
|
```typescript
|
|
569
|
-
// All three
|
|
570
|
-
helper.isHealthy(); // true when broker connected
|
|
576
|
+
// All three -- identical API
|
|
577
|
+
helper.isHealthy(); // true when at least one broker connected
|
|
571
578
|
helper.isReady(); // Admin/Producer: same as isHealthy()
|
|
572
579
|
// Consumer: isHealthy() + consumer.isActive()
|
|
573
580
|
helper.getHealthStatus(); // 'connected' | 'disconnected' | 'unknown'
|
|
@@ -576,8 +583,8 @@ helper.getHealthStatus(); // 'connected' | 'disconnected' | 'unknown'
|
|
|
576
583
|
### Shutdown
|
|
577
584
|
|
|
578
585
|
```typescript
|
|
579
|
-
// All three
|
|
580
|
-
await helper.close(); // graceful (timeout
|
|
586
|
+
// All three -- identical API
|
|
587
|
+
await helper.close(); // graceful (timeout -> force fallback)
|
|
581
588
|
await helper.close({ isForce: true }); // immediate force close
|
|
582
589
|
```
|
|
583
590
|
|
|
@@ -618,19 +625,19 @@ const result = await producer.runInTransaction(async ({ send, addConsumer, addOf
|
|
|
618
625
|
|
|
619
626
|
## Pages
|
|
620
627
|
|
|
621
|
-
- **[Producer](./producer)**
|
|
622
|
-
- **[Consumer](./consumer)**
|
|
623
|
-
- **[Admin](./admin)**
|
|
624
|
-
- **[Schema Registry](./schema-registry)**
|
|
625
|
-
- **[Examples & Troubleshooting](./examples)**
|
|
628
|
+
- **[Producer](./producer)** -- Producer helper, transactions, and full `@platformatic/kafka` Producer API reference
|
|
629
|
+
- **[Consumer](./consumer)** -- Consumer helper, message callbacks, lag monitoring, and full Consumer API reference
|
|
630
|
+
- **[Admin](./admin)** -- Admin helper and full Admin API reference
|
|
631
|
+
- **[Schema Registry](./schema-registry)** -- Schema registry helper for Avro/Protobuf/JSON Schema validation
|
|
632
|
+
- **[Examples & Troubleshooting](./examples)** -- Complete examples, IoC integration, and troubleshooting guide
|
|
626
633
|
|
|
627
634
|
## See Also
|
|
628
635
|
|
|
629
636
|
- **Other Helpers:**
|
|
630
|
-
- [Queue Helper](../queue/)
|
|
631
|
-
- [Redis Helper](../redis/)
|
|
637
|
+
- [Queue Helper](../queue/) -- BullMQ, MQTT, and in-memory queues
|
|
638
|
+
- [Redis Helper](../redis/) -- Redis connection management
|
|
632
639
|
|
|
633
640
|
- **External Resources:**
|
|
634
|
-
- [@platformatic/kafka](https://github.com/platformatic/kafka)
|
|
635
|
-
- [Apache Kafka Documentation](https://kafka.apache.org/documentation/)
|
|
636
|
-
- [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848)
|
|
641
|
+
- [@platformatic/kafka](https://github.com/platformatic/kafka) -- Underlying Kafka client library
|
|
642
|
+
- [Apache Kafka Documentation](https://kafka.apache.org/documentation/) -- Official Kafka docs
|
|
643
|
+
- [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848) -- New consumer group protocol
|
|
@@ -18,9 +18,10 @@ class KafkaProducerHelper<
|
|
|
18
18
|
| `newInstance(opts)` | `static newInstance<K,V,HK,HV>(opts): KafkaProducerHelper<K,V,HK,HV>` | Factory method |
|
|
19
19
|
| `getProducer()` | `(): Producer<K,V,HK,HV>` | Access the underlying `Producer` |
|
|
20
20
|
| `runInTransaction(cb)` | `<R>(cb: TKafkaTransactionCallback<R,K,V,HK,HV>): Promise<R>` | Execute callback within a Kafka transaction |
|
|
21
|
-
| `isHealthy()` | `(): boolean` | `true` when broker connected |
|
|
21
|
+
| `isHealthy()` | `(): boolean` | `true` when at least one broker connected |
|
|
22
22
|
| `isReady()` | `(): boolean` | Same as `isHealthy()` |
|
|
23
23
|
| `getHealthStatus()` | `(): TKafkaHealthStatus` | `'connected'` \| `'disconnected'` \| `'unknown'` |
|
|
24
|
+
| `getConnectedBrokerCount()` | `(): number` | Number of currently connected brokers |
|
|
24
25
|
| `close(opts?)` | `(opts?: { isForce?: boolean }): Promise<void>` | Close the producer (default: graceful) |
|
|
25
26
|
|
|
26
27
|
## IKafkaProducerOptions
|
|
@@ -33,17 +34,17 @@ interface IKafkaProducerOptions<KeyType, ValueType, HeaderKeyType, HeaderValueTy
|
|
|
33
34
|
| Option | Type | Default | Description |
|
|
34
35
|
|--------|------|---------|-------------|
|
|
35
36
|
| `identifier` | `string` | `'kafka-producer'` | Scoped logging identifier |
|
|
36
|
-
| `serializers` | `Partial<Serializers<K,V,HK,HV>>` |
|
|
37
|
-
| `compression` | `CompressionAlgorithmValue` |
|
|
38
|
-
| `acks` | `TKafkaAcks` |
|
|
39
|
-
| `idempotent` | `boolean` |
|
|
40
|
-
| `transactionalId` | `string` |
|
|
41
|
-
| `strict` | `boolean` | `true` | Strict mode
|
|
37
|
+
| `serializers` | `Partial<Serializers<K,V,HK,HV>>` | -- | Key/value/header serializers |
|
|
38
|
+
| `compression` | `CompressionAlgorithmValue` | -- | `'none'`, `'gzip'`, `'snappy'`, `'lz4'`, `'zstd'` |
|
|
39
|
+
| `acks` | `TKafkaAcks` | -- | Acknowledgment level: `0`, `1`, or `-1` |
|
|
40
|
+
| `idempotent` | `boolean` | -- | Enable idempotent producer (exactly-once within partition) |
|
|
41
|
+
| `transactionalId` | `string` | -- | Transactional ID for exactly-once across partitions |
|
|
42
|
+
| `strict` | `boolean` | `true` | Strict mode -- fail on unknown topics |
|
|
42
43
|
| `autocreateTopics` | `boolean` | `false` | Auto-create topics on first produce |
|
|
43
44
|
| `shutdownTimeout` | `number` | `30000` | Graceful shutdown timeout in ms |
|
|
44
|
-
| `registry` | `SchemaRegistry` |
|
|
45
|
-
| `onBrokerConnect` | `TKafkaBrokerEventCallback` |
|
|
46
|
-
| `onBrokerDisconnect` | `TKafkaBrokerEventCallback` |
|
|
45
|
+
| `registry` | `SchemaRegistry` | -- | Schema registry for auto ser/deser |
|
|
46
|
+
| `onBrokerConnect` | `TKafkaBrokerEventCallback` | -- | Called when broker connects |
|
|
47
|
+
| `onBrokerDisconnect` | `TKafkaBrokerEventCallback` | -- | Called when broker disconnects |
|
|
47
48
|
|
|
48
49
|
Plus all [Connection Options](./#connection-options).
|
|
49
50
|
|
|
@@ -64,7 +65,7 @@ const helper = KafkaProducerHelper.newInstance({
|
|
|
64
65
|
});
|
|
65
66
|
|
|
66
67
|
// Health check
|
|
67
|
-
helper.isHealthy(); // true when connected
|
|
68
|
+
helper.isHealthy(); // true when at least one broker connected
|
|
68
69
|
helper.getHealthStatus(); // 'connected' | 'disconnected' | 'unknown'
|
|
69
70
|
|
|
70
71
|
// Send messages via the underlying producer
|
|
@@ -84,7 +85,7 @@ await producer.send({
|
|
|
84
85
|
],
|
|
85
86
|
});
|
|
86
87
|
|
|
87
|
-
// Graceful close (waits for in-flight, times out after shutdownTimeout
|
|
88
|
+
// Graceful close (waits for in-flight, times out after shutdownTimeout -> force)
|
|
88
89
|
await helper.close();
|
|
89
90
|
|
|
90
91
|
// Or force close immediately
|
|
@@ -93,7 +94,7 @@ await helper.close({ isForce: true });
|
|
|
93
94
|
|
|
94
95
|
## Transactions
|
|
95
96
|
|
|
96
|
-
`runInTransaction()` wraps `beginTransaction()`
|
|
97
|
+
`runInTransaction()` wraps `beginTransaction()` -> callback -> `commit()` / `abort()` with automatic logging.
|
|
97
98
|
|
|
98
99
|
> [!NOTE]
|
|
99
100
|
> Requires `transactionalId` and `idempotent: true` in producer options.
|
|
@@ -149,9 +150,11 @@ If the callback throws, the transaction is automatically aborted and the error r
|
|
|
149
150
|
|
|
150
151
|
`close()` implements a two-phase shutdown:
|
|
151
152
|
|
|
152
|
-
1. **Graceful** (default):
|
|
153
|
+
1. **Graceful** (default): Calls `close(true)` on the underlying producer (force-flush), with a timeout (`shutdownTimeout`, default 30s)
|
|
153
154
|
2. **Force fallback**: If graceful times out, automatically force-closes
|
|
154
|
-
3. **Force** (`{ isForce: true }`): Immediately
|
|
155
|
+
3. **Force** (`{ isForce: true }`): Immediately calls `close(true)` without timeout protection
|
|
156
|
+
|
|
157
|
+
After `close()`, all broker tracking is cleared and `healthStatus` is set to `'disconnected'`.
|
|
155
158
|
|
|
156
159
|
```typescript
|
|
157
160
|
// Graceful (recommended)
|
|
@@ -161,8 +164,6 @@ await helper.close();
|
|
|
161
164
|
await helper.close({ isForce: true });
|
|
162
165
|
```
|
|
163
166
|
|
|
164
|
-
After `close()`, `healthStatus` is set to `'disconnected'`.
|
|
165
|
-
|
|
166
167
|
## API Reference (`@platformatic/kafka`)
|
|
167
168
|
|
|
168
169
|
After calling `helper.getProducer()`, you have full access to the `Producer` class:
|
|
@@ -258,9 +259,9 @@ Close the producer connection.
|
|
|
258
259
|
|
|
259
260
|
By default, `@platformatic/kafka` uses **murmur2 hashing** on the message key to determine the target partition:
|
|
260
261
|
|
|
261
|
-
- Same key
|
|
262
|
-
- `undefined` key
|
|
263
|
-
- Explicit `partition` field
|
|
262
|
+
- Same key -> always same partition -> guaranteed ordering per key
|
|
263
|
+
- `undefined` key -> round-robin across partitions
|
|
264
|
+
- Explicit `partition` field -> overrides the partitioner
|
|
264
265
|
|
|
265
266
|
```typescript
|
|
266
267
|
// Custom partitioner
|
|
@@ -12,7 +12,7 @@ class KafkaSchemaRegistryHelper<
|
|
|
12
12
|
```
|
|
13
13
|
|
|
14
14
|
> [!NOTE]
|
|
15
|
-
> `KafkaSchemaRegistryHelper` extends `BaseHelper` directly (not `BaseKafkaHelper`)
|
|
15
|
+
> `KafkaSchemaRegistryHelper` extends `BaseHelper` directly (not `BaseKafkaHelper`) -- it has no broker connection or health tracking. It's a configuration wrapper, not a client.
|
|
16
16
|
|
|
17
17
|
## Helper API
|
|
18
18
|
|
|
@@ -33,10 +33,10 @@ interface IKafkaSchemaRegistryOptions extends ConfluentSchemaRegistryOptions {
|
|
|
33
33
|
|
|
34
34
|
| Option | Type | Default | Description |
|
|
35
35
|
|--------|------|---------|-------------|
|
|
36
|
-
| `url` | `string` |
|
|
37
|
-
| `auth` | `{ username: string; password: string }` |
|
|
38
|
-
| `protobufTypeMapper` | `ProtobufTypeMapper` |
|
|
39
|
-
| `jsonValidateSend` | `boolean` |
|
|
36
|
+
| `url` | `string` | -- | Schema registry URL. **Required** |
|
|
37
|
+
| `auth` | `{ username: string; password: string }` | -- | Basic auth credentials |
|
|
38
|
+
| `protobufTypeMapper` | `ProtobufTypeMapper` | -- | Custom Protobuf type mapper |
|
|
39
|
+
| `jsonValidateSend` | `boolean` | -- | Validate JSON schema on produce |
|
|
40
40
|
| `identifier` | `string` | `'kafka-schema-registry'` | Scoped logging identifier |
|
|
41
41
|
|
|
42
42
|
## What Schema Registry Solves
|
|
@@ -46,14 +46,14 @@ Without a schema registry, producers and consumers must agree on message format
|
|
|
46
46
|
**Schema Registry** is a centralized server (Confluent Schema Registry) that stores and validates schemas (Avro, Protobuf, JSON Schema). It enforces a contract:
|
|
47
47
|
|
|
48
48
|
```
|
|
49
|
-
Producer
|
|
50
|
-
Kafka
|
|
49
|
+
Producer -> "I want to send this shape" -> Schema Registry validates -> Kafka
|
|
50
|
+
Kafka -> Consumer -> "What shape is this?" -> Schema Registry tells -> Deserialize
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
### Without Schema Registry (raw strings)
|
|
54
54
|
|
|
55
55
|
```typescript
|
|
56
|
-
// Producer
|
|
56
|
+
// Producer -- manually serialize
|
|
57
57
|
const producer = KafkaProducerHelper.newInstance({
|
|
58
58
|
bootstrapBrokers: ['127.0.0.1:29092'],
|
|
59
59
|
clientId: 'order-producer',
|
|
@@ -63,17 +63,17 @@ await producer.getProducer().send({
|
|
|
63
63
|
messages: [{
|
|
64
64
|
topic: 'orders',
|
|
65
65
|
key: 'order-1',
|
|
66
|
-
value: JSON.stringify({ id: 1, total: 99.99 }), //
|
|
66
|
+
value: JSON.stringify({ id: 1, total: 99.99 }), // <- just a string, no validation
|
|
67
67
|
}],
|
|
68
68
|
});
|
|
69
69
|
|
|
70
|
-
// Consumer
|
|
70
|
+
// Consumer -- manually deserialize, hope the shape is correct
|
|
71
71
|
const consumer = KafkaConsumerHelper.newInstance({
|
|
72
72
|
bootstrapBrokers: ['127.0.0.1:29092'],
|
|
73
73
|
clientId: 'order-consumer',
|
|
74
74
|
groupId: 'order-group',
|
|
75
75
|
onMessage: async ({ message }) => {
|
|
76
|
-
const order = JSON.parse(message.value as string); //
|
|
76
|
+
const order = JSON.parse(message.value as string); // <- pray it matches
|
|
77
77
|
console.log(order.id, order.total);
|
|
78
78
|
},
|
|
79
79
|
});
|
|
@@ -84,34 +84,34 @@ Problem: if producer adds `{ id: 1, total: 99.99, currency: 'USD' }` or removes
|
|
|
84
84
|
### With Schema Registry (auto serialize/deserialize)
|
|
85
85
|
|
|
86
86
|
```typescript
|
|
87
|
-
// 1. Create registry
|
|
87
|
+
// 1. Create registry -- points to Confluent Schema Registry server
|
|
88
88
|
const registry = KafkaSchemaRegistryHelper.newInstance({
|
|
89
89
|
url: 'http://localhost:8081',
|
|
90
90
|
// auth: { username: 'user', password: 'pass' }, // optional
|
|
91
91
|
});
|
|
92
92
|
|
|
93
|
-
// 2. Producer
|
|
93
|
+
// 2. Producer -- pass registry, it auto-serializes values using registered schema
|
|
94
94
|
const producer = KafkaProducerHelper.newInstance({
|
|
95
95
|
bootstrapBrokers: ['127.0.0.1:29092'],
|
|
96
96
|
clientId: 'order-producer',
|
|
97
|
-
registry: registry.getRegistry(), //
|
|
97
|
+
registry: registry.getRegistry(), // <- registry handles serialization
|
|
98
98
|
});
|
|
99
99
|
|
|
100
100
|
await producer.getProducer().send({
|
|
101
101
|
messages: [{
|
|
102
102
|
topic: 'orders',
|
|
103
103
|
key: 'order-1',
|
|
104
|
-
value: { id: 1, total: 99.99 }, //
|
|
104
|
+
value: { id: 1, total: 99.99 }, // <- object, not string! Registry serializes it
|
|
105
105
|
}],
|
|
106
106
|
});
|
|
107
|
-
// If the value doesn't match the registered schema
|
|
107
|
+
// If the value doesn't match the registered schema -> error BEFORE sending to Kafka
|
|
108
108
|
|
|
109
|
-
// 3. Consumer
|
|
109
|
+
// 3. Consumer -- pass same registry, it auto-deserializes
|
|
110
110
|
const consumer = KafkaConsumerHelper.newInstance({
|
|
111
111
|
bootstrapBrokers: ['127.0.0.1:29092'],
|
|
112
112
|
clientId: 'order-consumer',
|
|
113
113
|
groupId: 'order-group',
|
|
114
|
-
registry: registry.getRegistry(), //
|
|
114
|
+
registry: registry.getRegistry(), // <- registry handles deserialization
|
|
115
115
|
onMessage: async ({ message }) => {
|
|
116
116
|
// message.value is already a typed object, not a raw string
|
|
117
117
|
console.log(message.value.id, message.value.total);
|
|
@@ -124,7 +124,7 @@ const consumer = KafkaConsumerHelper.newInstance({
|
|
|
124
124
|
| | Without Registry | With Registry |
|
|
125
125
|
|---|---|---|
|
|
126
126
|
| **Message format** | Raw string, manual `JSON.stringify/parse` | Typed object, auto ser/deser |
|
|
127
|
-
| **Validation** | None
|
|
127
|
+
| **Validation** | None -- runtime crashes | Schema validated before send |
|
|
128
128
|
| **Schema evolution** | Break consumers silently | Backward/forward compatibility enforced |
|
|
129
129
|
| **Where schemas live** | Nowhere (tribal knowledge) | Centralized server `http://registry:8081` |
|
|
130
130
|
|
|
@@ -135,12 +135,12 @@ You only need it when you want **schema enforcement** across producers/consumers
|
|
|
135
135
|
```typescript
|
|
136
136
|
import { KafkaSchemaRegistryHelper, KafkaProducerHelper, KafkaConsumerHelper } from '@venizia/ignis-helpers/kafka';
|
|
137
137
|
|
|
138
|
-
// 1. Create registry
|
|
138
|
+
// 1. Create registry -- points to Confluent Schema Registry server
|
|
139
139
|
const registry = KafkaSchemaRegistryHelper.newInstance({
|
|
140
140
|
url: 'http://localhost:8081',
|
|
141
141
|
});
|
|
142
142
|
|
|
143
|
-
// 2. Producer
|
|
143
|
+
// 2. Producer -- registry auto-serializes values using registered schema
|
|
144
144
|
const producer = KafkaProducerHelper.newInstance({
|
|
145
145
|
bootstrapBrokers: ['localhost:9092'],
|
|
146
146
|
clientId: 'order-producer',
|
|
@@ -151,12 +151,12 @@ await producer.getProducer().send({
|
|
|
151
151
|
messages: [{
|
|
152
152
|
topic: 'orders',
|
|
153
153
|
key: 'order-1',
|
|
154
|
-
value: { id: 1, total: 99.99 }, // object, not string
|
|
154
|
+
value: { id: 1, total: 99.99 }, // object, not string -- registry serializes
|
|
155
155
|
}],
|
|
156
156
|
});
|
|
157
|
-
// If value doesn't match the registered schema
|
|
157
|
+
// If value doesn't match the registered schema -> error BEFORE sending to Kafka
|
|
158
158
|
|
|
159
|
-
// 3. Consumer
|
|
159
|
+
// 3. Consumer -- registry auto-deserializes
|
|
160
160
|
const consumer = KafkaConsumerHelper.newInstance({
|
|
161
161
|
bootstrapBrokers: ['localhost:9092'],
|
|
162
162
|
clientId: 'order-consumer',
|
|
@@ -210,5 +210,5 @@ const consumer = KafkaConsumerHelper.newInstance({
|
|
|
210
210
|
|
|
211
211
|
## When to Use
|
|
212
212
|
|
|
213
|
-
- **Use schema registry** when you need schema enforcement, validation, and compatibility checks across producers/consumers
|
|
213
|
+
- **Use schema registry** when you need schema enforcement, validation, and compatibility checks across producers/consumers -- especially in multi-team environments
|
|
214
214
|
- **Skip schema registry** for simple string/JSON messages where both sides are controlled by the same team and format changes are coordinated
|
|
@@ -587,13 +587,13 @@ APP_ENV_APPLICATION_NAME=my-service
|
|
|
587
587
|
|
|
588
588
|
- **Related Concepts:**
|
|
589
589
|
- [Services](/guides/core-concepts/services) -- Logging in services
|
|
590
|
-
- [Controllers](/guides/core-concepts/controllers) -- Logging in controllers
|
|
590
|
+
- [Controllers](/guides/core-concepts/rest-controllers) -- Logging in controllers
|
|
591
591
|
|
|
592
592
|
- **Other Helpers:**
|
|
593
593
|
- [Helpers Index](../index) -- All available helpers
|
|
594
594
|
|
|
595
595
|
- **References:**
|
|
596
|
-
- [Request Tracker Component](/
|
|
596
|
+
- [Request Tracker Component](/extensions/components/request-tracker) -- Request logging
|
|
597
597
|
|
|
598
598
|
- **External Resources:**
|
|
599
599
|
- [Winston Documentation](https://github.com/winstonjs/winston) -- Winston logging library
|