@midwayjs/kafka 1.0.0

package/README.md

@@ -0,0 +1,15 @@
+# midway framework for kafka
+
+[![Package Quality](http://npm.packagequality.com/shield/midway-web.svg)](http://packagequality.com/#?package=midway-web)
+[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/midwayjs/midway/pulls)
+
+this is a sub package for midway.
+
+
+this module use [kafkaJS](https://kafka.js.org)
+
+Document: [https://midwayjs.org](https://midwayjs.org)
+
+## License
+
+[MIT]((http://github.com/midwayjs/midway/blob/master/LICENSE))

package/dist/configuration.d.ts

@@ -0,0 +1,6 @@
+import { MidwayKafkaFramework } from './framework';
+export declare class KafkaConfiguration {
+    framework: MidwayKafkaFramework;
+    onReady(): Promise<void>;
+}
+//# sourceMappingURL=configuration.d.ts.map

package/dist/configuration.js

@@ -0,0 +1,35 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaConfiguration = void 0;
+const decorator_1 = require("@midwayjs/decorator");
+const framework_1 = require("./framework");
+let KafkaConfiguration = class KafkaConfiguration {
+    async onReady() { }
+};
+__decorate([
+    (0, decorator_1.Inject)(),
+    __metadata("design:type", framework_1.MidwayKafkaFramework)
+], KafkaConfiguration.prototype, "framework", void 0);
+KafkaConfiguration = __decorate([
+    (0, decorator_1.Configuration)({
+        namespace: 'kafka',
+        importConfigs: [
+            {
+                default: {
+                    kafka: {},
+                },
+            },
+        ],
+    })
+], KafkaConfiguration);
+exports.KafkaConfiguration = KafkaConfiguration;
+//# sourceMappingURL=configuration.js.map

package/dist/framework.d.ts

@@ -0,0 +1,12 @@
+import { BaseFramework } from '@midwayjs/core';
+import { MidwayFrameworkType } from '@midwayjs/decorator';
+export declare class MidwayKafkaFramework extends BaseFramework<any, any, any> {
+    configure(): any;
+    applicationInitialize(options: any): Promise<void>;
+    run(): Promise<void>;
+    private loadSubscriber;
+    getFrameworkName(): string;
+    protected beforeStop(): Promise<void>;
+    getFrameworkType(): MidwayFrameworkType;
+}
+//# sourceMappingURL=framework.d.ts.map

package/dist/framework.js

@@ -0,0 +1,95 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MidwayKafkaFramework = void 0;
+const core_1 = require("@midwayjs/core");
+const decorator_1 = require("@midwayjs/decorator");
+const kafka_1 = require("./kafka");
+let MidwayKafkaFramework = class MidwayKafkaFramework extends core_1.BaseFramework {
+    configure() {
+        return this.configService.getConfiguration('kafka');
+    }
+    async applicationInitialize(options) {
+        // Create a connection manager
+        this.app = new kafka_1.KafkaConsumerServer({
+            logger: this.logger,
+            ...this.configurationOptions,
+        });
+    }
+    async run() {
+        await this.app.connect(this.configurationOptions.kafkaConfig, this.configurationOptions.consumerConfig);
+        await this.loadSubscriber();
+        this.logger.info('Kafka consumer server start success');
+    }
+    async loadSubscriber() {
+        const subscriberModules = (0, decorator_1.listModule)(decorator_1.MS_CONSUMER_KEY, module => {
+            const metadata = (0, decorator_1.getClassMetadata)(decorator_1.MS_CONSUMER_KEY, module);
+            return metadata.type === decorator_1.MSListenerType.KAFKA;
+        });
+        for (const module of subscriberModules) {
+            const data = (0, decorator_1.listPropertyDataFromClass)(decorator_1.MS_CONSUMER_KEY, module);
+            for (const methodBindListeners of data) {
+                for (const listenerOptions of methodBindListeners) {
+                    await this.app.connection.subscribe({
+                        topic: listenerOptions.topic,
+                        // fromBeginning: true,
+                    });
+                    await this.app.connection.run({
+                        autoCommit: false,
+                        eachMessage: async ({ topic, partition, message }) => {
+                            const ctx = {
+                                topic: topic,
+                                partition: partition,
+                                message: message,
+                                commitOffsets: (offset) => {
+                                    return this.app.connection.commitOffsets([
+                                        {
+                                            topic: topic,
+                                            partition: partition,
+                                            offset,
+                                        },
+                                    ]);
+                                },
+                            };
+                            this.app.createAnonymousContext(ctx);
+                            const ins = await ctx.requestContext.getAsync(module);
+                            const fn = await this.applyMiddleware(async (ctx) => {
+                                return await ins[listenerOptions.propertyKey].call(ins, message);
+                            });
+                            try {
+                                const result = await fn(ctx);
+                                // 返回为undefined，下面永远不会执行
+                                if (result) {
+                                    const res = await this.app.connection.commitOffsets(message.offset);
+                                    return res;
+                                }
+                            }
+                            catch (error) {
+                                this.logger.error(error);
+                            }
+                        },
+                    });
+                }
+            }
+        }
+    }
+    getFrameworkName() {
+        return 'midway:kafka';
+    }
+    async beforeStop() {
+        await this.app.close();
+    }
+    getFrameworkType() {
+        return decorator_1.MidwayFrameworkType.MS_KAFKA;
+    }
+};
+MidwayKafkaFramework = __decorate([
+    (0, decorator_1.Framework)()
+], MidwayKafkaFramework);
+exports.MidwayKafkaFramework = MidwayKafkaFramework;
+//# sourceMappingURL=framework.js.map

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { MidwayKafkaFramework as Framework } from './framework';
+export * from './interface';
+export { KafkaConfiguration as Configuration } from './configuration';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,23 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Configuration = exports.Framework = void 0;
+var framework_1 = require("./framework");
+Object.defineProperty(exports, "Framework", { enumerable: true, get: function () { return framework_1.MidwayKafkaFramework; } });
+__exportStar(require("./interface"), exports);
+var configuration_1 = require("./configuration");
+Object.defineProperty(exports, "Configuration", { enumerable: true, get: function () { return configuration_1.KafkaConfiguration; } });
+//# sourceMappingURL=index.js.map

package/dist/interface.d.ts

@@ -0,0 +1,19 @@
+import { IConfigurationOptions, IMidwayApplication, IMidwayContext, NextFunction as BaseNextFunction } from '@midwayjs/core';
+import { Kafka, KafkaConfig } from 'kafkajs';
+export interface IKafkaApplication {
+}
+export declare type IMidwayKafkaApplication = IMidwayApplication<IMidwayKafkaContext> & IKafkaApplication;
+export interface IMidwayKafkaConfigurationOptions extends IConfigurationOptions, KafkaConfig {
+}
+export declare type IMidwayKafkaContext = IMidwayContext<{
+    topic: any;
+    partition: any;
+    message: any;
+    commitOffsets(data: any): void;
+}>;
+export declare type Application = IMidwayKafkaApplication;
+export interface Context extends IMidwayKafkaContext {
+}
+export declare type NextFunction = BaseNextFunction;
+export declare type DefaultConfig = string | Kafka;
+//# sourceMappingURL=interface.d.ts.map

package/dist/interface.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=interface.js.map

package/dist/kafka.d.ts

@@ -0,0 +1,15 @@
+/// <reference types="node" />
+import { Consumer, ConsumerConfig, KafkaConfig } from 'kafkajs';
+import { ILogger } from '@midwayjs/logger';
+import { IKafkaApplication } from './interface';
+import { EventEmitter } from 'stream';
+export declare class KafkaConsumerServer extends EventEmitter implements IKafkaApplication {
+    protected loggers: ILogger;
+    protected connection: Consumer;
+    constructor(options?: any);
+    bindError(): void;
+    connect(options: KafkaConfig, consumerOptions: ConsumerConfig): Promise<void>;
+    protected closeConnection(): Promise<void>;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=kafka.d.ts.map

package/dist/kafka.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaConsumerServer = void 0;
+const kafkajs_1 = require("kafkajs");
+const stream_1 = require("stream");
+class KafkaConsumerServer extends stream_1.EventEmitter {
+    constructor(options = {}) {
+        super();
+        this.connection = null;
+        this.loggers = options.logger;
+        this.bindError();
+    }
+    bindError() {
+        this.on('error', err => {
+            this.loggers.error(err);
+        });
+    }
+    async connect(options, consumerOptions) {
+        try {
+            this.connection = new kafkajs_1.Kafka(options).consumer(consumerOptions);
+            this.connection.on('consumer.connect', () => {
+                this.loggers.info('Kafka consumer connected!');
+            });
+            this.connection.on('consumer.disconnect', err => {
+                if (err) {
+                    this.loggers.error('Kafka consumer disconnected', err);
+                }
+                else {
+                    this.loggers.info('Kafka consumer disconnected!');
+                }
+            });
+        }
+        catch (error) {
+            this.loggers.error('Kafka consumer connect fail', error);
+            await this.closeConnection();
+        }
+    }
+    async closeConnection() {
+        try {
+            if (this.connection) {
+                await this.connection.disconnect();
+            }
+            this.loggers.debug('Kafka consumer connection close success');
+        }
+        catch (err) {
+            this.loggers.error('Kafka consumer connection close error', err);
+        }
+        finally {
+            this.connection = null;
+        }
+    }
+    async close() {
+        this.loggers.debug('Kafka consumer will be close');
+        await this.closeConnection();
+    }
+}
+exports.KafkaConsumerServer = KafkaConsumerServer;
+//# sourceMappingURL=kafka.js.map

package/index.d.ts

@@ -0,0 +1,8 @@
+import { IMidwayKafkaConfigurationOptions } from './dist';
+export * from './dist/index';
+
+declare module '@midwayjs/core/dist/interface' {
+  interface MidwayConfig {
+    kafka?: PowerPartial<IMidwayKafkaConfigurationOptions>;
+  }
+}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@midwayjs/kafka",
+  "version": "1.0.0",
+  "description": "Midway Framework for kafka",
+  "main": "dist/index",
+  "typings": "index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "test": "node --require=ts-node/register ../../node_modules/.bin/jest --runInBand",
+    "cov": "node --require=ts-node/register ../../node_modules/.bin/jest --runInBand --coverage --forceExit",
+    "lint:fix": "../../node_modules/.bin/mwts fix"
+  },
+  "keywords": [
+    "midway",
+    "IoC",
+    "ws",
+    "scene",
+    "kafka"
+  ],
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "index.d.ts"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/midwayjs/midway.git"
+  },
+  "author": "chenzhaozheng <389428742@qq.com>",
+  "bugs": {
+    "url": "https://github.com/midwayjs/midway/issues"
+  },
+  "homepage": "https://github.com/midwayjs/midway#readme",
+  "devDependencies": {
+    "@midwayjs/decorator": "^3.4.0-beta.1",
+    "@midwayjs/mock": "^3.4.0-beta.1",
+    "kafkajs": "^1.16.0"
+  },
+  "dependencies": {
+    "@midwayjs/core": "^3.4.0-beta.1",
+    "@midwayjs/logger": "^2.15.0",
+    "kafkajs": "^1.16.0"
+  },
+  "engines": {
+    "node": ">=12"
+  }
+}