@veridot/kafka 2.2.0

package/README.md

@@ -0,0 +1,61 @@
+# @veridot/kafka
+
+Kafka implementation for Veridot metadata broker.
+
+## Installation
+
+```bash
+npm install @veridot/kafka @veridot/core
+```
+
+## Usage
+
+```typescript
+import { KafkaMetadataBroker } from '@veridot/kafka';
+import { GenericSignerVerifier, BasicConfigurer, TokenMode } from '@veridot/core';
+
+// Create Kafka broker
+const broker = await KafkaMetadataBroker.of({
+  clientId: 'my-service',
+  brokers: ['localhost:9092'],
+  topic: 'veridot_keys',
+  dbPath: './data/veridot-keys',
+});
+
+// Create signer/verifier
+const signer = new GenericSignerVerifier(broker, process.env.APP_SALT!);
+
+// Sign a token
+const config = new BasicConfigurer()
+  .mode(TokenMode.JWT)
+  .validity(30, 'minutes');
+
+const token = await signer.sign({ userId: 123, role: 'admin' }, config);
+
+// Verify token (in another service)
+const payload = await signer.verify(token, (data) => JSON.parse(data));
+console.log('User:', payload.userId); // 123
+
+// Cleanup
+await signer.shutdown();
+await broker.disconnect();
+```
+
+## Configuration
+
+Environment variables:
+
+- `VDOT_KAFKA_BOOTSTRAP_SERVERS`: Kafka broker addresses (default: `localhost:9092`)
+- `VDOT_TOKEN_VERIFIER_TOPIC`: Kafka topic name (default: `veridot_keys`)
+- `VDOT_EMBEDDED_DATABASE_PATH`: LMDB database path (default: `./veridot-keys`)
+
+## Features
+
+- **Distributed**: Public keys propagated via Kafka to all services
+- **Cached**: Keys stored locally in LMDB for fast retrieval
+- **Persistent**: LMDB ensures keys survive service restarts
+- **Scalable**: Kafka handles high-throughput key distribution
+
+## License
+
+MIT © Darlin TEUMA

package/dist/KafkaMetadataBroker.d.ts

@@ -0,0 +1,212 @@
+import { MetadataBroker } from '@veridot/core';
+/**
+ * Configuration options for {@link KafkaMetadataBroker}.
+ *
+ * @public
+ */
+export interface KafkaMetadataBrokerOptions {
+    /**
+     * Kafka client ID for this service instance.
+     */
+    clientId: string;
+    /**
+     * Kafka broker addresses (comma-separated or array).
+     *
+     * @example ['localhost:9092', 'kafka2:9092']
+     */
+    brokers: string | string[];
+    /**
+     * Kafka topic name for key distribution.
+     *
+     * @default 'veridot_keys' (from Config.TOKEN_VERIFIER_TOPIC)
+     */
+    topic?: string;
+    /**
+     * Path to the embedded LMDB database for local caching.
+     *
+     * @default './veridot-keys' (from Config.EMBEDDED_DATABASE_PATH)
+     */
+    dbPath?: string;
+    /**
+     * Consumer group ID for Kafka.
+     *
+     * @default clientId + '-consumer'
+     */
+    groupId?: string;
+}
+/**
+ * Kafka-based implementation of {@link MetadataBroker}.
+ *
+ * This broker uses:
+ * - **Kafka**: For distributed message propagation between services
+ * - **LMDB**: For local persistent caching of public keys
+ *
+ * ## Architecture
+ *
+ * ```
+ * ┌──────────────┐      ┌──────────────┐      ┌──────────────┐
+ * │  Service A   │      │    Kafka     │      │  Service B   │
+ * │              │─────>│    Topic     │─────>│              │
+ * │ [Publisher]  │      │ veridot_keys │      │ [Consumer]   │
+ * └──────────────┘      └──────────────┘      └──────────────┘
+ *        │                                            │
+ *        v                                            v
+ * ┌──────────────┐                          ┌──────────────┐
+ * │ LMDB (local) │                          │ LMDB (local) │
+ * │  Key: keyId  │                          │  Key: keyId  │
+ * │  Val: pubKey │                          │  Val: pubKey │
+ * └──────────────┘                          └──────────────┘
+ * ```
+ *
+ * ## Key Features
+ *
+ * - **Distributed**: Public keys are propagated via Kafka to all services
+ * - **Cached**: Keys are stored locally in LMDB for fast retrieval
+ * - **Persistent**: LMDB ensures keys survive service restarts
+ * - **Scalable**: Kafka handles high-throughput key distribution
+ *
+ * ## Usage Example
+ *
+ * ```typescript
+ * import { KafkaMetadataBroker } from '@veridot/kafka';
+ * import { GenericSignerVerifier } from '@veridot/core';
+ *
+ * // Create broker
+ * const broker = await KafkaMetadataBroker.of({
+ *   clientId: 'my-service',
+ *   brokers: ['localhost:9092'],
+ *   topic: 'veridot_keys',
+ *   dbPath: './data/veridot-keys',
+ * });
+ *
+ * // Use with GenericSignerVerifier
+ * const signer = new GenericSignerVerifier(broker, process.env.APP_SALT!);
+ *
+ * // Sign and verify tokens
+ * const token = await signer.sign({ userId: 123 }, config);
+ * const payload = await signer.verify(token, JSON.parse);
+ *
+ * // Cleanup
+ * await broker.disconnect();
+ * ```
+ *
+ * ## Environment Variables
+ *
+ * - `VDOT_KAFKA_BOOTSTRAP_SERVERS`: Kafka broker addresses
+ * - `VDOT_TOKEN_VERIFIER_TOPIC`: Kafka topic name
+ * - `VDOT_EMBEDDED_DATABASE_PATH`: LMDB database path
+ *
+ * @public
+ * @since 2.2.0
+ * @author Darlin TEUMA
+ */
+export declare class KafkaMetadataBroker implements MetadataBroker {
+    private readonly producer;
+    private readonly consumer;
+    private readonly db;
+    private readonly topic;
+    private isConnected;
+    /**
+     * Private constructor. Use {@link KafkaMetadataBroker.of} to create instances.
+     *
+     * @internal
+     */
+    private constructor();
+    /**
+     * Factory method to create and initialize a KafkaMetadataBroker.
+     *
+     * This method:
+     * 1. Creates Kafka producer and consumer
+     * 2. Opens LMDB database
+     * 3. Connects to Kafka
+     * 4. Subscribes to the topic
+     * 5. Starts consuming messages
+     *
+     * @param options - Configuration options
+     * @returns A fully initialized KafkaMetadataBroker
+     *
+     * @throws {Error} If initialization fails
+     *
+     * @example
+     * ```typescript
+     * const broker = await KafkaMetadataBroker.of({
+     *   clientId: 'my-service',
+     *   brokers: ['localhost:9092'],
+     * });
+     * ```
+     */
+    static of(options: KafkaMetadataBrokerOptions): Promise<KafkaMetadataBroker>;
+    /**
+     * Publishes a metadata message associated with a given key.
+     *
+     * The message is sent to Kafka and also stored locally in LMDB.
+     *
+     * @param key - The key (typically keyId or tracking ID)
+     * @param message - The metadata message
+     *
+     * @throws {BrokerTransportException} If sending fails
+     *
+     * @example
+     * ```typescript
+     * await broker.send('abc123xyz', 'jwt:MIIBIjAN...:1731331200000');
+     * ```
+     */
+    send(key: string, message: string): Promise<void>;
+    /**
+     * Retrieves the metadata message associated with the given key.
+     *
+     * First checks the local LMDB cache. If not found, the method throws
+     * an exception (keys should have been received via Kafka consumer).
+     *
+     * @param key - The key to retrieve
+     * @returns The metadata message
+     *
+     * @throws {BrokerExtractionException} If the key is not found
+     *
+     * @example
+     * ```typescript
+     * const metadata = await broker.get('abc123xyz');
+     * console.log(metadata); // 'jwt:MIIBIjAN...:1731331200000'
+     * ```
+     */
+    get(key: string): Promise<string>;
+    /**
+     * Connects to Kafka and starts consuming messages.
+     *
+     * @internal
+     */
+    private connect;
+    /**
+     * Handles incoming Kafka messages by storing them in LMDB.
+     *
+     * @internal
+     */
+    private handleMessage;
+    /**
+     * Disconnects from Kafka and closes the LMDB database.
+     *
+     * Call this method when shutting down the application.
+     *
+     * @example
+     * ```typescript
+     * process.on('SIGTERM', async () => {
+     *   await broker.disconnect();
+     *   process.exit(0);
+     * });
+     * ```
+     */
+    disconnect(): Promise<void>;
+    /**
+     * Checks if the broker is connected.
+     *
+     * @returns True if connected
+     */
+    isReady(): boolean;
+    /**
+     * Gets the topic name.
+     *
+     * @returns The Kafka topic name
+     */
+    getTopic(): string;
+}
+//# sourceMappingURL=KafkaMetadataBroker.d.ts.map

package/dist/KafkaMetadataBroker.js

@@ -0,0 +1,325 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaMetadataBroker = void 0;
+const kafkajs_1 = require("kafkajs");
+const lmdb_1 = require("lmdb");
+const core_1 = require("@veridot/core");
+const core_2 = require("@veridot/core");
+/**
+ * Kafka-based implementation of {@link MetadataBroker}.
+ *
+ * This broker uses:
+ * - **Kafka**: For distributed message propagation between services
+ * - **LMDB**: For local persistent caching of public keys
+ *
+ * ## Architecture
+ *
+ * ```
+ * ┌──────────────┐      ┌──────────────┐      ┌──────────────┐
+ * │  Service A   │      │    Kafka     │      │  Service B   │
+ * │              │─────>│    Topic     │─────>│              │
+ * │ [Publisher]  │      │ veridot_keys │      │ [Consumer]   │
+ * └──────────────┘      └──────────────┘      └──────────────┘
+ *        │                                            │
+ *        v                                            v
+ * ┌──────────────┐                          ┌──────────────┐
+ * │ LMDB (local) │                          │ LMDB (local) │
+ * │  Key: keyId  │                          │  Key: keyId  │
+ * │  Val: pubKey │                          │  Val: pubKey │
+ * └──────────────┘                          └──────────────┘
+ * ```
+ *
+ * ## Key Features
+ *
+ * - **Distributed**: Public keys are propagated via Kafka to all services
+ * - **Cached**: Keys are stored locally in LMDB for fast retrieval
+ * - **Persistent**: LMDB ensures keys survive service restarts
+ * - **Scalable**: Kafka handles high-throughput key distribution
+ *
+ * ## Usage Example
+ *
+ * ```typescript
+ * import { KafkaMetadataBroker } from '@veridot/kafka';
+ * import { GenericSignerVerifier } from '@veridot/core';
+ *
+ * // Create broker
+ * const broker = await KafkaMetadataBroker.of({
+ *   clientId: 'my-service',
+ *   brokers: ['localhost:9092'],
+ *   topic: 'veridot_keys',
+ *   dbPath: './data/veridot-keys',
+ * });
+ *
+ * // Use with GenericSignerVerifier
+ * const signer = new GenericSignerVerifier(broker, process.env.APP_SALT!);
+ *
+ * // Sign and verify tokens
+ * const token = await signer.sign({ userId: 123 }, config);
+ * const payload = await signer.verify(token, JSON.parse);
+ *
+ * // Cleanup
+ * await broker.disconnect();
+ * ```
+ *
+ * ## Environment Variables
+ *
+ * - `VDOT_KAFKA_BOOTSTRAP_SERVERS`: Kafka broker addresses
+ * - `VDOT_TOKEN_VERIFIER_TOPIC`: Kafka topic name
+ * - `VDOT_EMBEDDED_DATABASE_PATH`: LMDB database path
+ *
+ * @public
+ * @since 2.2.0
+ * @author Darlin TEUMA
+ */
+class KafkaMetadataBroker {
+    /**
+     * Private constructor. Use {@link KafkaMetadataBroker.of} to create instances.
+     *
+     * @internal
+     */
+    constructor(producer, consumer, db, topic) {
+        this.isConnected = false;
+        this.producer = producer;
+        this.consumer = consumer;
+        this.db = db;
+        this.topic = topic;
+    }
+    /**
+     * Factory method to create and initialize a KafkaMetadataBroker.
+     *
+     * This method:
+     * 1. Creates Kafka producer and consumer
+     * 2. Opens LMDB database
+     * 3. Connects to Kafka
+     * 4. Subscribes to the topic
+     * 5. Starts consuming messages
+     *
+     * @param options - Configuration options
+     * @returns A fully initialized KafkaMetadataBroker
+     *
+     * @throws {Error} If initialization fails
+     *
+     * @example
+     * ```typescript
+     * const broker = await KafkaMetadataBroker.of({
+     *   clientId: 'my-service',
+     *   brokers: ['localhost:9092'],
+     * });
+     * ```
+     */
+    static async of(options) {
+        console.log('[KafkaMetadataBroker] Initializing...');
+        // Parse brokers
+        const brokers = Array.isArray(options.brokers)
+            ? options.brokers
+            : options.brokers.split(',').map((b) => b.trim());
+        // Get topic name
+        const topic = options.topic || core_2.Config.TOKEN_VERIFIER_TOPIC;
+        // Get database path
+        const dbPath = options.dbPath || core_2.Config.EMBEDDED_DATABASE_PATH;
+        // Get group ID
+        const groupId = options.groupId || `${options.clientId}-consumer`;
+        console.log(`[KafkaMetadataBroker] Config: brokers=${brokers.join(',')}, topic=${topic}, dbPath=${dbPath}`);
+        // Create Kafka client
+        const kafka = new kafkajs_1.Kafka({
+            clientId: options.clientId,
+            brokers,
+        });
+        // Create producer
+        const producer = kafka.producer();
+        // Create consumer
+        const consumer = kafka.consumer({ groupId });
+        // Open LMDB database
+        const db = (0, lmdb_1.open)({
+            path: dbPath,
+            compression: true,
+        });
+        // Create broker instance
+        const broker = new KafkaMetadataBroker(producer, consumer, db, topic);
+        // Connect and initialize
+        await broker.connect();
+        console.log('[KafkaMetadataBroker] Initialized successfully');
+        return broker;
+    }
+    /**
+     * Publishes a metadata message associated with a given key.
+     *
+     * The message is sent to Kafka and also stored locally in LMDB.
+     *
+     * @param key - The key (typically keyId or tracking ID)
+     * @param message - The metadata message
+     *
+     * @throws {BrokerTransportException} If sending fails
+     *
+     * @example
+     * ```typescript
+     * await broker.send('abc123xyz', 'jwt:MIIBIjAN...:1731331200000');
+     * ```
+     */
+    async send(key, message) {
+        if (!this.isConnected) {
+            throw new core_1.BrokerTransportException('Broker is not connected');
+        }
+        try {
+            // Store in local LMDB database
+            await this.db.put(key, message);
+            // Send to Kafka
+            await this.producer.send({
+                topic: this.topic,
+                messages: [
+                    {
+                        key,
+                        value: message,
+                    },
+                ],
+            });
+            console.log(`[KafkaMetadataBroker] Sent metadata for key: ${key}`);
+        }
+        catch (error) {
+            throw new core_1.BrokerTransportException(`Failed to send metadata for key ${key}: ${error.message}`, error);
+        }
+    }
+    /**
+     * Retrieves the metadata message associated with the given key.
+     *
+     * First checks the local LMDB cache. If not found, the method throws
+     * an exception (keys should have been received via Kafka consumer).
+     *
+     * @param key - The key to retrieve
+     * @returns The metadata message
+     *
+     * @throws {BrokerExtractionException} If the key is not found
+     *
+     * @example
+     * ```typescript
+     * const metadata = await broker.get('abc123xyz');
+     * console.log(metadata); // 'jwt:MIIBIjAN...:1731331200000'
+     * ```
+     */
+    async get(key) {
+        if (!this.isConnected) {
+            throw new core_1.BrokerExtractionException('Broker is not connected');
+        }
+        try {
+            // Retrieve from local LMDB database
+            const message = await this.db.get(key);
+            if (!message) {
+                throw new core_1.BrokerExtractionException(`Metadata not found for key: ${key}. ` +
+                    'The key may have expired, been revoked, or never existed.');
+            }
+            return message;
+        }
+        catch (error) {
+            if (error instanceof core_1.BrokerExtractionException) {
+                throw error;
+            }
+            throw new core_1.BrokerExtractionException(`Failed to retrieve metadata for key ${key}: ${error.message}`);
+        }
+    }
+    /**
+     * Connects to Kafka and starts consuming messages.
+     *
+     * @internal
+     */
+    async connect() {
+        try {
+            console.log('[KafkaMetadataBroker] Connecting to Kafka...');
+            // Connect producer
+            await this.producer.connect();
+            console.log('[KafkaMetadataBroker] Producer connected');
+            // Connect consumer
+            await this.consumer.connect();
+            console.log('[KafkaMetadataBroker] Consumer connected');
+            // Subscribe to topic
+            await this.consumer.subscribe({ topic: this.topic, fromBeginning: true });
+            console.log(`[KafkaMetadataBroker] Subscribed to topic: ${this.topic}`);
+            // Start consuming
+            await this.consumer.run({
+                eachMessage: async (payload) => {
+                    await this.handleMessage(payload);
+                },
+            });
+            this.isConnected = true;
+            console.log('[KafkaMetadataBroker] Connected successfully');
+        }
+        catch (error) {
+            throw new Error(`Failed to connect to Kafka: ${error.message}`);
+        }
+    }
+    /**
+     * Handles incoming Kafka messages by storing them in LMDB.
+     *
+     * @internal
+     */
+    async handleMessage(payload) {
+        const { message } = payload;
+        if (!message.key || !message.value) {
+            console.warn('[KafkaMetadataBroker] Received message without key or value');
+            return;
+        }
+        const key = message.key.toString();
+        const value = message.value.toString();
+        try {
+            // Store in local LMDB database
+            await this.db.put(key, value);
+            console.log(`[KafkaMetadataBroker] Stored metadata for key: ${key}`);
+        }
+        catch (error) {
+            console.error(`[KafkaMetadataBroker] Failed to store metadata for key ${key}:`, error);
+        }
+    }
+    /**
+     * Disconnects from Kafka and closes the LMDB database.
+     *
+     * Call this method when shutting down the application.
+     *
+     * @example
+     * ```typescript
+     * process.on('SIGTERM', async () => {
+     *   await broker.disconnect();
+     *   process.exit(0);
+     * });
+     * ```
+     */
+    async disconnect() {
+        if (!this.isConnected) {
+            return;
+        }
+        console.log('[KafkaMetadataBroker] Disconnecting...');
+        try {
+            // Disconnect consumer
+            await this.consumer.disconnect();
+            console.log('[KafkaMetadataBroker] Consumer disconnected');
+            // Disconnect producer
+            await this.producer.disconnect();
+            console.log('[KafkaMetadataBroker] Producer disconnected');
+            // Close LMDB database
+            this.db.close();
+            console.log('[KafkaMetadataBroker] Database closed');
+            this.isConnected = false;
+            console.log('[KafkaMetadataBroker] Disconnected successfully');
+        }
+        catch (error) {
+            console.error('[KafkaMetadataBroker] Error during disconnect:', error);
+            throw error;
+        }
+    }
+    /**
+     * Checks if the broker is connected.
+     *
+     * @returns True if connected
+     */
+    isReady() {
+        return this.isConnected;
+    }
+    /**
+     * Gets the topic name.
+     *
+     * @returns The Kafka topic name
+     */
+    getTopic() {
+        return this.topic;
+    }
+}
+exports.KafkaMetadataBroker = KafkaMetadataBroker;
+//# sourceMappingURL=KafkaMetadataBroker.js.map

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @veridot/kafka - Kafka implementation for Veridot metadata broker.
+ *
+ * This module provides a Kafka-based implementation of the MetadataBroker
+ * interface, using Kafka for distributed message propagation and LMDB for
+ * local persistent caching.
+ *
+ * @packageDocumentation
+ * @module @veridot/kafka
+ * @author Darlin TEUMA
+ * @since 2.2.0
+ */
+export { KafkaMetadataBroker, KafkaMetadataBrokerOptions } from './KafkaMetadataBroker';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,18 @@
+"use strict";
+/**
+ * @veridot/kafka - Kafka implementation for Veridot metadata broker.
+ *
+ * This module provides a Kafka-based implementation of the MetadataBroker
+ * interface, using Kafka for distributed message propagation and LMDB for
+ * local persistent caching.
+ *
+ * @packageDocumentation
+ * @module @veridot/kafka
+ * @author Darlin TEUMA
+ * @since 2.2.0
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaMetadataBroker = void 0;
+var KafkaMetadataBroker_1 = require("./KafkaMetadataBroker");
+Object.defineProperty(exports, "KafkaMetadataBroker", { enumerable: true, get: function () { return KafkaMetadataBroker_1.KafkaMetadataBroker; } });
+//# sourceMappingURL=index.js.map

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@veridot/kafka",
+  "version": "2.2.0",
+  "description": "Kafka implementation module for Veridot - Provides Kafka-based metadata broker",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist",
+    "prepublishOnly": "pnpm run clean && pnpm run build"
+  },
+  "keywords": [
+    "veridot",
+    "kafka",
+    "metadata",
+    "broker",
+    "distributed"
+  ],
+  "author": "Darlin TEUMA <darlinteuma2@gmail.com>",
+  "contributors": [
+    "Frank KOSSI <frank.kossi@kunrin.com> (Original Veridot creator and architect)"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@veridot/core": "workspace:*",
+    "kafkajs": "^2.2.4",
+    "lmdb": "^3.2.6"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.3.0"
+  }
+}
+