@midwayjs/kafka 4.0.0-alpha.1 → 4.0.0-beta.2

package/README.md

@@ -12,4 +12,4 @@ Document: [https://midwayjs.org](https://midwayjs.org)
 
 ## License
 
-[MIT]((http://github.com/midwayjs/midway/blob/master/LICENSE))
+[MIT]((https://github.com/midwayjs/midway/blob/master/LICENSE))

package/dist/configuration.d.ts

@@ -1,3 +1,5 @@
+import { IMidwayContainer } from '@midwayjs/core';
 export declare class KafkaConfiguration {
+    onReady(container: IMidwayContainer): Promise<void>;
 }
 //# sourceMappingURL=configuration.d.ts.map

package/dist/configuration.js

@@ -8,7 +8,11 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.KafkaConfiguration = void 0;
 const core_1 = require("@midwayjs/core");
+const service_1 = require("./service");
 let KafkaConfiguration = class KafkaConfiguration {
+    async onReady(container) {
+        await container.getAsync(service_1.KafkaProducerFactory);
+    }
 };
 exports.KafkaConfiguration = KafkaConfiguration;
 exports.KafkaConfiguration = KafkaConfiguration = __decorate([
@@ -17,9 +21,7 @@ exports.KafkaConfiguration = KafkaConfiguration = __decorate([
         importConfigs: [
             {
                 default: {
-                    kafka: {
-                        contextLoggerApplyLogger: 'kafkaLogger',
-                    },
+                    kafka: {},
                     midwayLogger: {
                         clients: {
                             kafkaLogger: {

package/dist/framework.d.ts

@@ -1,14 +1,13 @@
-import { BaseFramework, ILogger, TypedResourceManager } from '@midwayjs/core';
+import { BaseFramework, TypedResourceManager } from '@midwayjs/core';
 import { IKafkaConsumerInitOptions, IKafkaConsumer, IMidwayKafkaContext } from './interface';
 import { Kafka, Consumer } from 'kafkajs';
 export declare class MidwayKafkaFramework extends BaseFramework<any, IMidwayKafkaContext, any> {
     protected LogCreator: any;
     protected typedResourceManager: TypedResourceManager<Consumer, IKafkaConsumerInitOptions, IKafkaConsumer>;
+    protected frameworkLoggerName: string;
     configure(): any;
-    kafKaLogger: ILogger;
     applicationInitialize(): Promise<void>;
     run(): Promise<void>;
-    private loadLegacySubscriber;
     getConsumer(subscriberNameOrInstanceName: string): any;
     getKafka(instanceName: string): Kafka;
     getFrameworkName(): string;

package/dist/framework.js

@@ -5,13 +5,9 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
     else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
-var __metadata = (this && this.__metadata) || function (k, v) {
-    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MidwayKafkaFramework = void 0;
 const core_1 = require("@midwayjs/core");
-const kafka_1 = require("./kafka");
 const decorator_1 = require("./decorator");
 const kafkajs_1 = require("kafkajs");
 const manager_1 = require("./manager");
@@ -30,197 +26,86 @@ const toMidwayLogLevel = level => {
     }
 };
 let MidwayKafkaFramework = class MidwayKafkaFramework extends core_1.BaseFramework {
+    constructor() {
+        super(...arguments);
+        this.frameworkLoggerName = 'kafkaLogger';
+    }
     configure() {
         return this.configService.getConfiguration('kafka');
     }
     async applicationInitialize() {
         this.LogCreator = logLevel => {
-            const logger = this.kafKaLogger;
+            const logger = this.logger;
             return ({ level, log }) => {
                 const lvl = toMidwayLogLevel(level);
                 const { message, ...extra } = log;
                 logger?.[lvl](message, extra);
             };
         };
-        // Create a connection manager
-        if (this.configurationOptions['kafkaConfig']) {
-            this.app = new kafka_1.KafkaConsumerServer({
-                logger: this.kafKaLogger,
-                ...this.configurationOptions,
-            });
-        }
-        else {
-            this.app = {};
-        }
+        this.app = {};
     }
     async run() {
-        if (this.configurationOptions['kafkaConfig']) {
-            try {
-                await this.app.connect(this.configurationOptions.kafkaConfig, this.configurationOptions.consumerConfig);
-                await this.loadLegacySubscriber();
-                this.kafKaLogger.info('Kafka consumer server start success');
-            }
-            catch (error) {
-                this.kafKaLogger.error('Kafka consumer connect fail', error);
-                await this.app.closeConnection();
-            }
+        const { consumer } = this.configurationOptions;
+        if (!consumer)
+            return;
+        const subscriberMap = {};
+        // find subscriber
+        const subscriberModules = core_1.DecoratorManager.listModule(decorator_1.KAFKA_DECORATOR_KEY);
+        for (const subscriberModule of subscriberModules) {
+            const subscriberName = core_1.MetadataManager.getOwnMetadata(decorator_1.KAFKA_DECORATOR_KEY, subscriberModule);
+            subscriberMap[subscriberName] = subscriberModule;
         }
-        else {
-            const { consumer } = this.configurationOptions;
-            if (!consumer)
-                return;
-            const subscriberMap = {};
-            // find subscriber
-            const subscriberModules = core_1.DecoratorManager.listModule(decorator_1.KAFKA_DECORATOR_KEY);
-            for (const subscriberModule of subscriberModules) {
-                const subscriberName = core_1.MetadataManager.getOwnMetadata(decorator_1.KAFKA_DECORATOR_KEY, subscriberModule);
-                subscriberMap[subscriberName] = subscriberModule;
-            }
-            this.typedResourceManager = new core_1.TypedResourceManager({
-                initializeValue: consumer,
-                initializeClzProvider: subscriberMap,
-                resourceInitialize: async (resourceInitializeConfig, resourceName) => {
-                    let client;
-                    if (resourceInitializeConfig.kafkaInstanceRef) {
-                        client = manager_1.KafkaManager.getInstance().getKafkaInstance(resourceInitializeConfig.kafkaInstanceRef);
-                        if (!client) {
-                            throw new core_1.MidwayCommonError(`kafka instance ${resourceInitializeConfig.kafkaInstanceRef} not found`);
-                        }
+        this.typedResourceManager = new core_1.TypedResourceManager({
+            initializeValue: consumer,
+            initializeClzProvider: subscriberMap,
+            resourceInitialize: async (resourceInitializeConfig, resourceName) => {
+                let client;
+                if (resourceInitializeConfig.kafkaInstanceRef) {
+                    client = manager_1.KafkaManager.getInstance().getKafkaInstance(resourceInitializeConfig.kafkaInstanceRef);
+                    if (!client) {
+                        throw new core_1.MidwayCommonError(`kafka instance ${resourceInitializeConfig.kafkaInstanceRef} not found`);
                     }
-                    else {
-                        client = new kafkajs_1.Kafka({
-                            logCreator: this.LogCreator,
-                            ...resourceInitializeConfig.connectionOptions,
-                        });
-                        manager_1.KafkaManager.getInstance().addKafkaInstance(resourceName, client);
-                    }
-                    const consumer = client.consumer(resourceInitializeConfig.consumerOptions);
-                    await consumer.connect();
-                    await consumer.subscribe(resourceInitializeConfig.subscribeOptions);
-                    return consumer;
-                },
-                resourceBinding: async (ClzProvider, resourceInitializeConfig, consumer) => {
-                    const runMethod = ClzProvider.prototype['eachBatch']
-                        ? 'eachBatch'
-                        : 'eachMessage';
-                    const runConfig = {
-                        ...resourceInitializeConfig.consumerRunConfig,
-                    };
-                    runConfig[runMethod] = async (payload) => {
-                        const ctx = this.app.createAnonymousContext();
-                        const fn = await this.applyMiddleware(async (ctx) => {
-                            ctx.payload = payload;
-                            ctx.consumer = consumer;
-                            const instance = await ctx.requestContext.getAsync(ClzProvider);
-                            return await instance[runMethod].call(instance, payload, ctx);
-                        });
-                        return await fn(ctx);
-                    };
-                    return runConfig;
-                },
-                resourceStart: async (resource, resourceInitializeConfig, resourceBindingResult) => {
-                    await resource.run(resourceBindingResult);
-                },
-                resourceDestroy: async (resource) => {
-                    await resource.disconnect();
-                },
-            });
-            await this.typedResourceManager.init();
-            await this.typedResourceManager.start();
-        }
-    }
-    async loadLegacySubscriber() {
-        const subscriberModules = core_1.DecoratorManager.listModule(core_1.MS_CONSUMER_KEY, module => {
-            const metadata = core_1.MetadataManager.getOwnMetadata(core_1.MS_CONSUMER_KEY, module);
-            return metadata.type === core_1.MSListenerType.KAFKA;
-        });
-        for (const module of subscriberModules) {
-            const data = (0, core_1.listPropertyDataFromClass)(core_1.MS_CONSUMER_KEY, module);
-            const topicTitles = [...new Set(data.map(e => e[0].topic))];
-            const midwayConsumerConfigs = topicTitles.map(e => {
-                const midwayConsumerConfig = {
-                    topic: e,
-                    subscription: {},
-                    runConfig: {},
-                };
-                const consumerParams = data
-                    .map(value => {
-                    if (value[0].topic === midwayConsumerConfig.topic) {
-                        return Object.assign(midwayConsumerConfig, value[0]);
-                    }
-                })
-                    .filter(e => e && true);
-                if (consumerParams && Object.keys(consumerParams[0]).length > 0) {
-                    return consumerParams[0];
                 }
-                return midwayConsumerConfig;
-            });
-            midwayConsumerConfigs.forEach(async (e) => {
-                await this.app.connection.subscribe(Object.assign({
-                    topics: topicTitles,
-                }, e.subscription));
-                await this.app.connection.run(Object.assign(e.runConfig, {
-                    eachMessage: async (payload) => {
-                        const { topic, partition, message } = payload;
-                        let propertyKey;
-                        for (const methodBindListeners of data) {
-                            for (const listenerOptions of methodBindListeners) {
-                                if (topic === listenerOptions.topic) {
-                                    propertyKey = listenerOptions.propertyKey;
-                                    const ctx = {
-                                        topic: topic,
-                                        partition: partition,
-                                        message: message,
-                                        commitOffsets: (offset) => {
-                                            return this.app.connection.commitOffsets([
-                                                {
-                                                    topic: topic,
-                                                    partition: partition,
-                                                    offset,
-                                                },
-                                            ]);
-                                        },
-                                        payload,
-                                        consumer: this.app.connection,
-                                    };
-                                    this.app.createAnonymousContext(ctx);
-                                    if (typeof propertyKey === 'string') {
-                                        const isPassed = await this.app
-                                            .getFramework()
-                                            .runGuard(ctx, module, propertyKey);
-                                        if (!isPassed) {
-                                            throw new core_1.MidwayInvokeForbiddenError(propertyKey, module);
-                                        }
-                                    }
-                                    const ins = await ctx.requestContext.getAsync(module);
-                                    const fn = await this.applyMiddleware(async () => {
-                                        return await ins[propertyKey].call(ins, message);
-                                    });
-                                    try {
-                                        const result = await fn(ctx);
-                                        // 返回为undefined，下面永远不会执行
-                                        if (result) {
-                                            const res = await this.app.connection.commitOffsets([
-                                                {
-                                                    topic: topic,
-                                                    partition: partition,
-                                                    offset: message.offset,
-                                                },
-                                            ]);
-                                            return res;
-                                        }
-                                    }
-                                    catch (error) {
-                                        // 记录偏移量提交的异常情况
-                                        this.logger.error(error);
-                                    }
-                                }
-                            }
-                        }
-                    },
-                }));
-            });
-        }
+                else {
+                    client = new kafkajs_1.Kafka({
+                        logCreator: this.LogCreator,
+                        ...resourceInitializeConfig.connectionOptions,
+                    });
+                    manager_1.KafkaManager.getInstance().addKafkaInstance(resourceName, client);
+                }
+                const consumer = client.consumer(resourceInitializeConfig.consumerOptions);
+                await consumer.connect();
+                await consumer.subscribe(resourceInitializeConfig.subscribeOptions);
+                return consumer;
+            },
+            resourceBinding: async (ClzProvider, resourceInitializeConfig, consumer) => {
+                const runMethod = ClzProvider.prototype['eachBatch']
+                    ? 'eachBatch'
+                    : 'eachMessage';
+                const runConfig = {
+                    ...resourceInitializeConfig.consumerRunConfig,
+                };
+                runConfig[runMethod] = async (payload) => {
+                    const ctx = this.app.createAnonymousContext();
+                    const fn = await this.applyMiddleware(async (ctx) => {
+                        ctx.payload = payload;
+                        ctx.consumer = consumer;
+                        const instance = await ctx.requestContext.getAsync(ClzProvider);
+                        return await instance[runMethod].call(instance, payload, ctx);
+                    });
+                    return await fn(ctx);
+                };
+                return runConfig;
+            },
+            resourceStart: async (resource, resourceInitializeConfig, resourceBindingResult) => {
+                await resource.run(resourceBindingResult);
+            },
+            resourceDestroy: async (resource) => {
+                await resource.disconnect();
+            },
+        });
+        await this.typedResourceManager.init();
+        await this.typedResourceManager.start();
     }
     getConsumer(subscriberNameOrInstanceName) {
         if (this.typedResourceManager) {
@@ -240,10 +125,6 @@ let MidwayKafkaFramework = class MidwayKafkaFramework extends core_1.BaseFramewo
     }
 };
 exports.MidwayKafkaFramework = MidwayKafkaFramework;
-__decorate([
-    (0, core_1.Logger)('kafkaLogger'),
-    __metadata("design:type", Object)
-], MidwayKafkaFramework.prototype, "kafKaLogger", void 0);
 exports.MidwayKafkaFramework = MidwayKafkaFramework = __decorate([
     (0, core_1.Framework)()
 ], MidwayKafkaFramework);

package/dist/service.js

@@ -18,7 +18,9 @@ let KafkaProducerFactory = class KafkaProducerFactory extends core_1.ServiceFact
         return 'kafka:producer';
     }
     async init() {
-        await this.initClients(this.pubConfig);
+        await this.initClients(this.pubConfig, {
+            concurrent: true,
+        });
     }
     async createClient(config, clientName) {
         const { connectionOptions, producerOptions, kafkaInstanceRef } = config;
@@ -77,7 +79,9 @@ let KafkaAdminFactory = class KafkaAdminFactory extends core_1.ServiceFactory {
         return 'kafka:admin';
     }
     async init() {
-        await this.initClients(this.adminConfig);
+        await this.initClients(this.adminConfig, {
+            concurrent: true,
+        });
     }
     async createClient(config, clientName) {
         const { connectionOptions, adminOptions, kafkaInstanceRef } = config;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midwayjs/kafka",
-  "version": "4.0.0-alpha.1",
+  "version": "4.0.0-beta.2",
   "description": "Midway Framework for kafka",
   "main": "dist/index.js",
   "typings": "index.d.ts",
@@ -33,14 +33,14 @@
   },
   "homepage": "https://github.com/midwayjs/midway#readme",
   "devDependencies": {
-    "@midwayjs/core": "^4.0.0-alpha.1",
-    "@midwayjs/mock": "^4.0.0-alpha.1"
+    "@midwayjs/core": "^4.0.0-beta.2",
+    "@midwayjs/mock": "^4.0.0-beta.2"
   },
   "dependencies": {
     "kafkajs": "2.2.4"
   },
   "engines": {
-    "node": ">=12"
+    "node": ">=20"
   },
-  "gitHead": "14bb4da91805a1cf52f190c0d37a74b395dd6372"
+  "gitHead": "53bfef4c5279da5f09025e4610bdbf64f94f60bd"
 }

package/dist/kafka.d.ts

@@ -1,15 +0,0 @@
-/// <reference types="node" />
-/// <reference types="node" />
-import { Consumer, ConsumerConfig, KafkaConfig } from 'kafkajs';
-import { ILogger } from '@midwayjs/core';
-import { EventEmitter } from 'stream';
-export declare class KafkaConsumerServer extends EventEmitter {
-    protected loggers: ILogger;
-    connection: Consumer;
-    constructor(options?: any);
-    bindError(): void;
-    connect(options: KafkaConfig, consumerOptions: ConsumerConfig): Promise<void>;
-    protected closeConnection(): Promise<void>;
-    close(): Promise<void>;
-}
-//# sourceMappingURL=kafka.d.ts.map

package/dist/kafka.js

@@ -1,52 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.KafkaConsumerServer = void 0;
-const kafkajs_1 = require("kafkajs");
-const stream_1 = require("stream");
-class KafkaConsumerServer extends stream_1.EventEmitter {
-    constructor(options = {}) {
-        super();
-        this.connection = null;
-        this.loggers = options.logger;
-        this.bindError();
-    }
-    bindError() {
-        this.on('error', err => {
-            this.loggers.error(err);
-        });
-    }
-    async connect(options, consumerOptions) {
-        this.connection = new kafkajs_1.Kafka(options).consumer(consumerOptions);
-        this.connection.on('consumer.connect', () => {
-            this.loggers.info('Kafka consumer connected!');
-        });
-        this.connection.on('consumer.disconnect', err => {
-            if (err) {
-                this.loggers.error('Kafka consumer disconnected', err);
-            }
-            else {
-                this.loggers.info('Kafka consumer disconnected!');
-            }
-        });
-    }
-    async closeConnection() {
-        try {
-            if (this.connection) {
-                await this.connection.disconnect();
-            }
-            this.loggers.debug('Kafka consumer connection close success');
-        }
-        catch (err) {
-            this.loggers.error('Kafka consumer connection close error', err);
-        }
-        finally {
-            this.connection = null;
-        }
-    }
-    async close() {
-        this.loggers.debug('Kafka consumer will be close');
-        await this.closeConnection();
-    }
-}
-exports.KafkaConsumerServer = KafkaConsumerServer;
-//# sourceMappingURL=kafka.js.map