@stepflowjs/trigger-kafka 0.0.1

package/dist/index.d.ts

@@ -0,0 +1,65 @@
+import { Trigger, TriggerHandler } from '@stepflowjs/core';
+
+interface KafkaTriggerConfig {
+    /** Kafka broker URLs */
+    brokers: string[];
+    /** Topic to consume from */
+    topic: string;
+    /** Consumer group ID */
+    groupId: string;
+    /** Client identifier (optional) */
+    clientId?: string;
+    /** Start from beginning (default: false) */
+    fromBeginning?: boolean;
+    /** Enable SSL */
+    ssl?: boolean;
+    /** SASL authentication */
+    sasl?: {
+        mechanism: "plain" | "scram-sha-256" | "scram-sha-512";
+        username: string;
+        password: string;
+    };
+}
+/**
+ * Kafka trigger for Stepflow workflows
+ *
+ * Consumes messages from a Kafka topic using consumer groups and triggers
+ * workflows for each message. Supports SSL and SASL authentication.
+ *
+ * @example
+ * ```typescript
+ * const trigger = new KafkaTrigger({
+ *   brokers: ['localhost:9092'],
+ *   topic: 'orders',
+ *   groupId: 'stepflow-orders',
+ *   fromBeginning: false,
+ * });
+ *
+ * await trigger.start(async (event) => {
+ *   await stepflow.trigger('process-order', event.data);
+ * });
+ * ```
+ */
+declare class KafkaTrigger implements Trigger<KafkaTriggerConfig> {
+    readonly config: KafkaTriggerConfig;
+    readonly type = "kafka";
+    private kafka?;
+    private consumer?;
+    private running;
+    constructor(config: KafkaTriggerConfig);
+    /**
+     * Start the Kafka consumer and begin processing messages
+     * @param handler Function to call for each Kafka message
+     */
+    start(handler: TriggerHandler): Promise<void>;
+    /**
+     * Stop the Kafka consumer and disconnect
+     */
+    stop(): Promise<void>;
+    /**
+     * Health check - returns true if consumer is running
+     */
+    healthCheck(): Promise<boolean>;
+}
+
+export { KafkaTrigger, type KafkaTriggerConfig };

package/dist/index.js

@@ -0,0 +1,114 @@
+// src/index.ts
+import { Kafka } from "kafkajs";
+var KafkaTrigger = class {
+  constructor(config) {
+    this.config = config;
+  }
+  type = "kafka";
+  kafka;
+  consumer;
+  running = false;
+  /**
+   * Start the Kafka consumer and begin processing messages
+   * @param handler Function to call for each Kafka message
+   */
+  async start(handler) {
+    if (this.running) {
+      throw new Error("KafkaTrigger is already running");
+    }
+    try {
+      const kafkaConfig = {
+        clientId: this.config.clientId || "stepflow-trigger",
+        brokers: this.config.brokers
+      };
+      if (this.config.ssl) {
+        kafkaConfig.ssl = true;
+      }
+      if (this.config.sasl) {
+        kafkaConfig.sasl = this.config.sasl;
+      }
+      this.kafka = new Kafka(kafkaConfig);
+      this.consumer = this.kafka.consumer({ groupId: this.config.groupId });
+      await this.consumer.connect();
+      await this.consumer.subscribe({
+        topic: this.config.topic,
+        fromBeginning: this.config.fromBeginning ?? false
+      });
+      this.running = true;
+      await this.consumer.run({
+        eachMessage: async ({ topic, partition, message }) => {
+          if (!this.running) {
+            return;
+          }
+          try {
+            let data = null;
+            if (message.value) {
+              try {
+                data = JSON.parse(message.value.toString());
+              } catch {
+                data = message.value.toString();
+              }
+            }
+            const event = {
+              id: crypto.randomUUID(),
+              type: this.type,
+              source: topic,
+              data,
+              metadata: {
+                topic,
+                partition,
+                offset: message.offset,
+                key: message.key?.toString(),
+                headers: message.headers ? Object.fromEntries(
+                  Object.entries(message.headers).map(([k, v]) => [
+                    k,
+                    v?.toString()
+                  ])
+                ) : void 0,
+                timestamp: message.timestamp
+              },
+              timestamp: /* @__PURE__ */ new Date()
+            };
+            await handler(event);
+          } catch (error) {
+            console.error(
+              `Error processing Kafka message from ${topic}:${partition}@${message.offset}:`,
+              error
+            );
+          }
+        }
+      });
+    } catch (error) {
+      this.running = false;
+      throw new Error(
+        `Failed to start KafkaTrigger: ${error.message}`
+      );
+    }
+  }
+  /**
+   * Stop the Kafka consumer and disconnect
+   */
+  async stop() {
+    this.running = false;
+    if (this.consumer) {
+      try {
+        await this.consumer.disconnect();
+      } catch (error) {
+        console.error("Error disconnecting Kafka consumer:", error);
+      } finally {
+        this.consumer = void 0;
+        this.kafka = void 0;
+      }
+    }
+  }
+  /**
+   * Health check - returns true if consumer is running
+   */
+  async healthCheck() {
+    return this.running && this.consumer !== void 0;
+  }
+};
+export {
+  KafkaTrigger
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type { Trigger, TriggerHandler, TriggerEvent } from \"@stepflowjs/core\";\nimport { Kafka, type Consumer, type KafkaConfig } from \"kafkajs\";\n\n// ============================================================================\n// Types\n// ============================================================================\n\nexport interface KafkaTriggerConfig {\n  /** Kafka broker URLs */\n  brokers: string[];\n  /** Topic to consume from */\n  topic: string;\n  /** Consumer group ID */\n  groupId: string;\n  /** Client identifier (optional) */\n  clientId?: string;\n  /** Start from beginning (default: false) */\n  fromBeginning?: boolean;\n  /** Enable SSL */\n  ssl?: boolean;\n  /** SASL authentication */\n  sasl?: {\n    mechanism: \"plain\" | \"scram-sha-256\" | \"scram-sha-512\";\n    username: string;\n    password: string;\n  };\n}\n\n// ============================================================================\n// KafkaTrigger Implementation\n// ============================================================================\n\n/**\n * Kafka trigger for Stepflow workflows\n *\n * Consumes messages from a Kafka topic using consumer groups and triggers\n * workflows for each message. Supports SSL and SASL authentication.\n *\n * @example\n * ```typescript\n * const trigger = new KafkaTrigger({\n *   brokers: ['localhost:9092'],\n *   topic: 'orders',\n *   groupId: 'stepflow-orders',\n *   fromBeginning: false,\n * });\n *\n * await trigger.start(async (event) => {\n *   await stepflow.trigger('process-order', event.data);\n * });\n * ```\n */\nexport class KafkaTrigger implements Trigger<KafkaTriggerConfig> {\n  readonly type = \"kafka\";\n  private kafka?: Kafka;\n  private consumer?: Consumer;\n  private running = false;\n\n  constructor(readonly config: KafkaTriggerConfig) {}\n\n  /**\n   * Start the Kafka consumer and begin processing messages\n   * @param handler Function to call for each Kafka message\n   */\n  async start(handler: TriggerHandler): Promise<void> {\n    if (this.running) {\n      throw new Error(\"KafkaTrigger is already running\");\n    }\n\n    try {\n      // Create Kafka client configuration\n      const kafkaConfig: KafkaConfig = {\n        clientId: this.config.clientId || \"stepflow-trigger\",\n        brokers: this.config.brokers,\n      };\n\n      // Add SSL if configured\n      if (this.config.ssl) {\n        kafkaConfig.ssl = true;\n      }\n\n      // Add SASL if configured\n      if (this.config.sasl) {\n        kafkaConfig.sasl = this.config.sasl as any;\n      }\n\n      // Create Kafka client and consumer\n      this.kafka = new Kafka(kafkaConfig);\n      this.consumer = this.kafka.consumer({ groupId: this.config.groupId });\n\n      // Connect to Kafka\n      await this.consumer.connect();\n\n      // Subscribe to topic\n      await this.consumer.subscribe({\n        topic: this.config.topic,\n        fromBeginning: this.config.fromBeginning ?? false,\n      });\n\n      this.running = true;\n\n      // Start consuming messages\n      await this.consumer.run({\n        eachMessage: async ({ topic, partition, message }) => {\n          if (!this.running) {\n            return;\n          }\n\n          try {\n            // Parse message value as JSON\n            let data: unknown = null;\n            if (message.value) {\n              try {\n                data = JSON.parse(message.value.toString());\n              } catch {\n                // If not JSON, use raw string\n                data = message.value.toString();\n              }\n            }\n\n            // Create trigger event\n            const event: TriggerEvent = {\n              id: crypto.randomUUID(),\n              type: this.type,\n              source: topic,\n              data,\n              metadata: {\n                topic,\n                partition,\n                offset: message.offset,\n                key: message.key?.toString(),\n                headers: message.headers\n                  ? Object.fromEntries(\n                      Object.entries(message.headers).map(([k, v]) => [\n                        k,\n                        v?.toString(),\n                      ]),\n                    )\n                  : undefined,\n                timestamp: message.timestamp,\n              },\n              timestamp: new Date(),\n            };\n\n            // Invoke handler\n            await handler(event);\n          } catch (error) {\n            console.error(\n              `Error processing Kafka message from ${topic}:${partition}@${message.offset}:`,\n              error,\n            );\n            // Don't throw - let Kafka handle retries via consumer group\n          }\n        },\n      });\n    } catch (error) {\n      this.running = false;\n      throw new Error(\n        `Failed to start KafkaTrigger: ${(error as Error).message}`,\n      );\n    }\n  }\n\n  /**\n   * Stop the Kafka consumer and disconnect\n   */\n  async stop(): Promise<void> {\n    this.running = false;\n\n    if (this.consumer) {\n      try {\n        await this.consumer.disconnect();\n      } catch (error) {\n        console.error(\"Error disconnecting Kafka consumer:\", error);\n      } finally {\n        this.consumer = undefined;\n        this.kafka = undefined;\n      }\n    }\n  }\n\n  /**\n   * Health check - returns true if consumer is running\n   */\n  async healthCheck(): Promise<boolean> {\n    return this.running && this.consumer !== undefined;\n  }\n}\n"],"mappings":";AACA,SAAS,aAA8C;AAmDhD,IAAM,eAAN,MAA0D;AAAA,EAM/D,YAAqB,QAA4B;AAA5B;AAAA,EAA6B;AAAA,EALzC,OAAO;AAAA,EACR;AAAA,EACA;AAAA,EACA,UAAU;AAAA;AAAA;AAAA;AAAA;AAAA,EAQlB,MAAM,MAAM,SAAwC;AAClD,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,QAAI;AAEF,YAAM,cAA2B;AAAA,QAC/B,UAAU,KAAK,OAAO,YAAY;AAAA,QAClC,SAAS,KAAK,OAAO;AAAA,MACvB;AAGA,UAAI,KAAK,OAAO,KAAK;AACnB,oBAAY,MAAM;AAAA,MACpB;AAGA,UAAI,KAAK,OAAO,MAAM;AACpB,oBAAY,OAAO,KAAK,OAAO;AAAA,MACjC;AAGA,WAAK,QAAQ,IAAI,MAAM,WAAW;AAClC,WAAK,WAAW,KAAK,MAAM,SAAS,EAAE,SAAS,KAAK,OAAO,QAAQ,CAAC;AAGpE,YAAM,KAAK,SAAS,QAAQ;AAG5B,YAAM,KAAK,SAAS,UAAU;AAAA,QAC5B,OAAO,KAAK,OAAO;AAAA,QACnB,eAAe,KAAK,OAAO,iBAAiB;AAAA,MAC9C,CAAC;AAED,WAAK,UAAU;AAGf,YAAM,KAAK,SAAS,IAAI;AAAA,QACtB,aAAa,OAAO,EAAE,OAAO,WAAW,QAAQ,MAAM;AACpD,cAAI,CAAC,KAAK,SAAS;AACjB;AAAA,UACF;AAEA,cAAI;AAEF,gBAAI,OAAgB;AACpB,gBAAI,QAAQ,OAAO;AACjB,kBAAI;AACF,uBAAO,KAAK,MAAM,QAAQ,MAAM,SAAS,CAAC;AAAA,cAC5C,QAAQ;AAEN,uBAAO,QAAQ,MAAM,SAAS;AAAA,cAChC;AAAA,YACF;AAGA,kBAAM,QAAsB;AAAA,cAC1B,IAAI,OAAO,WAAW;AAAA,cACtB,MAAM,KAAK;AAAA,cACX,QAAQ;AAAA,cACR;AAAA,cACA,UAAU;AAAA,gBACR;AAAA,gBACA;AAAA,gBACA,QAAQ,QAAQ;AAAA,gBAChB,KAAK,QAAQ,KAAK,SAAS;AAAA,gBAC3B,SAAS,QAAQ,UACb,OAAO;AAAA,kBACL,OAAO,QAAQ,QAAQ,OAAO,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM;AAAA,oBAC9C;AAAA,oBACA,GAAG,SAAS;AAAA,kBACd,CAAC;AAAA,gBACH,IACA;AAAA,gBACJ,WAAW,QAAQ;AAAA,cACrB;AAAA,cACA,WAAW,oBAAI,KAAK;AAAA,YACtB;AAGA,kBAAM,QAAQ,KAAK;AAAA,UACrB,SAAS,OAAO;AACd,oBAAQ;AAAA,cACN,uCAAuC,KAAK,IAAI,SAAS,IAAI,QAAQ,MAAM;AAAA,cAC3E;AAAA,YACF;AAAA,UAEF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,WAAK,UAAU;AACf,YAAM,IAAI;AAAA,QACR,iCAAkC,MAAgB,OAAO;AAAA,MAC3D;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,OAAsB;AAC1B,SAAK,UAAU;AAEf,QAAI,KAAK,UAAU;AACjB,UAAI;AACF,cAAM,KAAK,SAAS,WAAW;AAAA,MACjC,SAAS,OAAO;AACd,gBAAQ,MAAM,uCAAuC,KAAK;AAAA,MAC5D,UAAE;AACA,aAAK,WAAW;AAChB,aAAK,QAAQ;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,cAAgC;AACpC,WAAO,KAAK,WAAW,KAAK,aAAa;AAAA,EAC3C;AACF;","names":[]}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@stepflowjs/trigger-kafka",
+  "version": "0.0.1",
+  "description": "Kafka trigger for Stepflow with consumer group management",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "kafkajs": "^2.2.4",
+    "@stepflowjs/core": "0.0.1"
+  },
+  "devDependencies": {
+    "tsup": "^8.5.1",
+    "vitest": "^4.0.17"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "license": "MIT",
+  "author": "Stepflow Contributors",
+  "repository": {
+    "type": "git",
+    "url": "https://stepflow-production.up.railway.app",
+    "directory": "packages/triggers/kafka"
+  },
+  "homepage": "https://stepflow-production.up.railway.app",
+  "bugs": {
+    "url": "https://stepflow-production.up.railway.app"
+  },
+  "keywords": [
+    "stepflow",
+    "trigger",
+    "kafka",
+    "stream",
+    "workflow",
+    "orchestration"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest",
+    "clean": "rm -rf dist"
+  }
+}