@blokjs/trigger-pubsub 0.2.0

package/src/PubSubTrigger.ts

@@ -0,0 +1,337 @@
+/**
+ * PubSubTrigger - Base class for pub/sub-based workflow triggers
+ *
+ * Extends TriggerBase to support pub/sub triggers:
+ * - GCP Pub/Sub
+ * - AWS SNS (via SQS subscription)
+ * - Azure Service Bus
+ *
+ * Pattern:
+ * 1. loadNodes() - Load available nodes into NodeMap
+ * 2. loadWorkflows() - Load workflows with pubsub triggers
+ * 3. startSubscriber() - Connect to pub/sub and start receiving messages
+ * 4. For each message:
+ *    - Match workflow by trigger config (topic/subscription)
+ *    - Create context with this.createContext()
+ *    - Populate ctx.request with message data
+ *    - Execute workflow via this.run(ctx)
+ *    - Ack/nack based on response
+ */
+
+import type { HelperResponse, PubSubProvider, PubSubTriggerOpts } from "@blokjs/helper";
+import {
+	DefaultLogger,
+	type GlobalOptions,
+	type BlokService,
+	NodeMap,
+	TriggerBase,
+	type TriggerResponse,
+} from "@blokjs/runner";
+import type { Context, RequestContext } from "@blokjs/shared";
+import { type Span, SpanStatusCode, metrics, trace } from "@opentelemetry/api";
+import { v4 as uuid } from "uuid";
+
+/**
+ * Message received from pub/sub
+ */
+export interface PubSubMessage {
+	/** Unique message ID */
+	id: string;
+	/** Message body (parsed) */
+	body: unknown;
+	/** Message attributes/metadata */
+	attributes: Record<string, string>;
+	/** Original raw message from provider */
+	raw: unknown;
+	/** Topic name */
+	topic: string;
+	/** Subscription name */
+	subscription?: string;
+	/** Publish timestamp */
+	publishTime?: Date;
+	/** Acknowledge the message */
+	ack: () => Promise<void>;
+	/** Reject/nack the message */
+	nack: () => Promise<void>;
+}
+
+/**
+ * Pub/Sub adapter interface - implemented by each provider
+ */
+export interface PubSubAdapter {
+	/** Provider name */
+	readonly provider: PubSubProvider;
+
+	/** Connect to the pub/sub system */
+	connect(): Promise<void>;
+
+	/** Disconnect from the pub/sub system */
+	disconnect(): Promise<void>;
+
+	/** Subscribe to a topic and receive messages */
+	subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void>;
+
+	/** Unsubscribe from a topic */
+	unsubscribe(subscription: string): Promise<void>;
+
+	/** Check if connected */
+	isConnected(): boolean;
+
+	/** Health check */
+	healthCheck(): Promise<boolean>;
+}
+
+/**
+ * Workflow model with pub/sub trigger configuration
+ */
+interface PubSubWorkflowModel {
+	path: string;
+	config: {
+		name: string;
+		version: string;
+		trigger?: {
+			pubsub?: PubSubTriggerOpts;
+			[key: string]: unknown;
+		};
+		[key: string]: unknown;
+	};
+}
+
+/**
+ * PubSubTrigger - Abstract base class for pub/sub-based triggers
+ */
+export abstract class PubSubTrigger extends TriggerBase {
+	protected nodeMap: GlobalOptions = {} as GlobalOptions;
+	protected readonly tracer = trace.getTracer(
+		process.env.PROJECT_NAME || "trigger-pubsub-workflow",
+		process.env.PROJECT_VERSION || "0.0.1",
+	);
+	protected readonly logger = new DefaultLogger();
+	protected abstract adapter: PubSubAdapter;
+
+	// Subclasses provide these
+	protected abstract nodes: Record<string, BlokService<unknown>>;
+	protected abstract workflows: Record<string, HelperResponse>;
+
+	constructor() {
+		super();
+		this.loadNodes();
+		this.loadWorkflows();
+	}
+
+	/**
+	 * Load nodes into the node map
+	 */
+	loadNodes(): void {
+		this.nodeMap.nodes = new NodeMap();
+		const nodeKeys = Object.keys(this.nodes);
+		for (const key of nodeKeys) {
+			this.nodeMap.nodes.addNode(key, this.nodes[key]);
+		}
+	}
+
+	/**
+	 * Load workflows into the workflow map
+	 */
+	loadWorkflows(): void {
+		this.nodeMap.workflows = this.workflows;
+	}
+
+	/**
+	 * Start the pub/sub subscriber - main entry point
+	 */
+	async listen(): Promise<number> {
+		const startTime = this.startCounter();
+
+		try {
+			// Connect to pub/sub system
+			await this.adapter.connect();
+			this.logger.log(`Connected to ${this.adapter.provider} pub/sub system`);
+
+			// Find all workflows with pub/sub triggers
+			const pubsubWorkflows = this.getPubSubWorkflows();
+
+			if (pubsubWorkflows.length === 0) {
+				this.logger.log("No workflows with pub/sub triggers found");
+				return this.endCounter(startTime);
+			}
+
+			// Subscribe to each topic/subscription
+			for (const workflow of pubsubWorkflows) {
+				const config = workflow.config.trigger?.pubsub as PubSubTriggerOpts;
+				this.logger.log(
+					`Subscribing to topic: ${config.topic}, subscription: ${config.subscription} for workflow: ${workflow.path}`,
+				);
+
+				await this.adapter.subscribe(config, async (message) => {
+					await this.handleMessage(message, workflow, config);
+				});
+			}
+
+			this.logger.log(`Pub/Sub trigger started. Listening to ${pubsubWorkflows.length} subscription(s)`);
+
+			// Enable HMR in development mode
+			if (process.env.BLOK_HMR === "true" || process.env.NODE_ENV === "development") {
+				await this.enableHotReload();
+			}
+
+			return this.endCounter(startTime);
+		} catch (error) {
+			this.logger.error(`Failed to start pub/sub trigger: ${(error as Error).message}`);
+			throw error;
+		}
+	}
+
+	/**
+	 * Stop the pub/sub subscriber
+	 */
+	async stop(): Promise<void> {
+		await this.adapter.disconnect();
+		this.logger.log("Pub/Sub trigger stopped");
+	}
+
+	protected override async onHmrWorkflowChange(): Promise<void> {
+		this.logger.log("[HMR] Pub/Sub workflow changed, reloading...");
+		await this.waitForInFlightRequests();
+		await this.stop();
+		this.loadWorkflows();
+		await this.listen();
+	}
+
+	/**
+	 * Get all workflows that have pub/sub triggers
+	 */
+	protected getPubSubWorkflows(): PubSubWorkflowModel[] {
+		const workflows: PubSubWorkflowModel[] = [];
+
+		for (const [path, workflow] of Object.entries(this.nodeMap.workflows || {})) {
+			// HelperResponse has a protected _config property
+			const workflowConfig = (workflow as unknown as { _config: PubSubWorkflowModel["config"] })._config;
+
+			if (workflowConfig?.trigger) {
+				const triggerType = Object.keys(workflowConfig.trigger)[0];
+
+				if (triggerType === "pubsub" && workflowConfig.trigger.pubsub) {
+					workflows.push({
+						path,
+						config: workflowConfig,
+					});
+				}
+			}
+		}
+
+		return workflows;
+	}
+
+	/**
+	 * Handle an incoming message
+	 */
+	protected async handleMessage(
+		message: PubSubMessage,
+		workflow: PubSubWorkflowModel,
+		config: PubSubTriggerOpts,
+	): Promise<void> {
+		const id = message.id || uuid();
+		const defaultMeter = metrics.getMeter("default");
+		const pubsubMessages = defaultMeter.createCounter("pubsub_messages", {
+			description: "Pub/Sub messages processed",
+		});
+		const pubsubErrors = defaultMeter.createCounter("pubsub_errors", {
+			description: "Pub/Sub message processing errors",
+		});
+
+		await this.tracer.startActiveSpan(`pubsub:${config.topic}`, async (span: Span) => {
+			try {
+				const start = performance.now();
+
+				// Initialize configuration for this workflow
+				await this.configuration.init(workflow.path, this.nodeMap);
+
+				// Create context
+				const ctx: Context = this.createContext(undefined, workflow.path, id);
+
+				// Populate request with message data
+				ctx.request = {
+					body: message.body,
+					headers: message.attributes,
+					query: {},
+					params: {
+						topic: message.topic,
+						subscription: message.subscription || "",
+						messageId: message.id,
+					},
+				} as unknown as RequestContext;
+
+				// Store message metadata in context
+				if (!ctx.vars) ctx.vars = {};
+				ctx.vars["_pubsub_message"] = {
+					topic: message.topic,
+					subscription: message.subscription || "",
+					publishTime: message.publishTime?.toISOString() ?? "",
+					attributes: JSON.stringify(message.attributes),
+				};
+
+				ctx.logger.log(`Processing message from ${config.topic}: ${id}`);
+
+				// Execute workflow
+				const response: TriggerResponse = await this.run(ctx);
+				const end = performance.now();
+
+				// Set span attributes
+				span.setAttribute("success", true);
+				span.setAttribute("message_id", id);
+				span.setAttribute("topic", config.topic);
+				span.setAttribute("subscription", config.subscription);
+				span.setAttribute("provider", config.provider);
+				span.setAttribute("elapsed_ms", end - start);
+				span.setStatus({ code: SpanStatusCode.OK });
+
+				// Record metrics
+				pubsubMessages.add(1, {
+					env: process.env.NODE_ENV,
+					topic: config.topic,
+					subscription: config.subscription,
+					provider: config.provider,
+					workflow_name: this.configuration.name,
+					success: "true",
+				});
+
+				ctx.logger.log(`Message processed in ${(end - start).toFixed(2)}ms: ${id}`);
+
+				// Acknowledge message if configured
+				if (config.ack !== false) {
+					await message.ack();
+					ctx.logger.log(`Message acknowledged: ${id}`);
+				}
+			} catch (error) {
+				const errorMessage = (error as Error).message;
+
+				// Set span error
+				span.setAttribute("success", false);
+				span.recordException(error as Error);
+				span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage });
+
+				// Record error metrics
+				pubsubErrors.add(1, {
+					env: process.env.NODE_ENV,
+					topic: config.topic,
+					subscription: config.subscription,
+					provider: config.provider,
+					workflow_name: this.configuration?.name || "unknown",
+				});
+
+				this.logger.error(`Failed to process message ${id}: ${errorMessage}`, (error as Error).stack);
+
+				// Nack message
+				if (config.ack !== false) {
+					await message.nack();
+					this.logger.log(`Message nacked: ${id}`);
+				}
+			} finally {
+				span.end();
+			}
+		});
+	}
+}
+
+export default PubSubTrigger;

package/src/adapters/AWSSNSAdapter.ts

@@ -0,0 +1,258 @@
+/**
+ * AWSSNSAdapter - AWS SNS/SQS adapter for PubSubTrigger
+ *
+ * Uses AWS SDK v3 for SNS/SQS connectivity.
+ * SNS topics deliver to SQS queues, which this adapter polls.
+ *
+ * Requires: npm install @aws-sdk/client-sns @aws-sdk/client-sqs
+ *
+ * Environment variables:
+ * - AWS_REGION: AWS region (default: us-east-1)
+ * - AWS_ACCESS_KEY_ID: AWS access key (optional if using IAM roles)
+ * - AWS_SECRET_ACCESS_KEY: AWS secret key (optional if using IAM roles)
+ * - SQS_WAIT_TIME_SECONDS: Long polling wait time (default: 20)
+ * - SQS_MAX_MESSAGES: Max messages per receive (default: 10)
+ */
+
+import type { PubSubTriggerOpts } from "@blokjs/helper";
+import { v4 as uuid } from "uuid";
+import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
+
+/**
+ * AWS SNS/SQS configuration
+ */
+export interface AWSSNSConfig {
+	region: string;
+	waitTimeSeconds?: number;
+	maxNumberOfMessages?: number;
+}
+
+/**
+ * AWSSNSAdapter - AWS SNS implementation using SQS subscriptions
+ */
+export class AWSSNSAdapter implements PubSubAdapter {
+	readonly provider = "aws" as const;
+
+	private sqsClient: any;
+	private connected = false;
+	private config: AWSSNSConfig;
+	private pollingIntervals: Map<string, NodeJS.Timeout> = new Map();
+	private shouldStop = false;
+
+	constructor(config?: Partial<AWSSNSConfig>) {
+		this.config = {
+			region: config?.region || process.env.AWS_REGION || "us-east-1",
+			waitTimeSeconds: config?.waitTimeSeconds ?? Number.parseInt(process.env.SQS_WAIT_TIME_SECONDS || "20", 10),
+			maxNumberOfMessages: config?.maxNumberOfMessages ?? Number.parseInt(process.env.SQS_MAX_MESSAGES || "10", 10),
+		};
+	}
+
+	/**
+	 * Connect to AWS
+	 */
+	async connect(): Promise<void> {
+		if (this.connected) return;
+
+		try {
+			// Dynamic import of AWS SDK
+			const { SQSClient } = await import("@aws-sdk/client-sqs");
+
+			this.sqsClient = new SQSClient({
+				region: this.config.region,
+			});
+
+			this.connected = true;
+			this.shouldStop = false;
+			console.log(`[AWSSNSAdapter] Connected to AWS SNS/SQS: ${this.config.region}`);
+		} catch (error) {
+			throw new Error(
+				`Failed to connect to AWS: ${(error as Error).message}. ` +
+					`Make sure @aws-sdk/client-sqs is installed: npm install @aws-sdk/client-sqs`,
+			);
+		}
+	}
+
+	/**
+	 * Disconnect from AWS
+	 */
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+
+		this.shouldStop = true;
+
+		// Clear all polling intervals
+		for (const [queueUrl, interval] of this.pollingIntervals) {
+			clearTimeout(interval);
+		}
+		this.pollingIntervals.clear();
+
+		this.connected = false;
+		console.log("[AWSSNSAdapter] Disconnected from AWS SNS/SQS");
+	}
+
+	/**
+	 * Subscribe to an SNS topic via SQS queue
+	 * Note: The SQS queue should be pre-configured as an SNS subscription
+	 */
+	async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
+		if (!this.connected) {
+			throw new Error("Not connected to AWS. Call connect() first.");
+		}
+
+		// In AWS, subscription is the SQS queue URL that's subscribed to the SNS topic
+		const queueUrl = config.subscription;
+
+		// Start polling the SQS queue
+		this.poll(queueUrl, config, handler);
+
+		console.log(`[AWSSNSAdapter] Subscribed to queue: ${queueUrl} (topic: ${config.topic})`);
+	}
+
+	/**
+	 * Poll SQS queue for messages (long polling)
+	 */
+	private async poll(
+		queueUrl: string,
+		config: PubSubTriggerOpts,
+		handler: (message: PubSubMessage) => Promise<void>,
+	): Promise<void> {
+		if (this.shouldStop) return;
+
+		try {
+			const { ReceiveMessageCommand, DeleteMessageCommand } = await import("@aws-sdk/client-sqs");
+
+			const command = new ReceiveMessageCommand({
+				QueueUrl: queueUrl,
+				MaxNumberOfMessages: config.maxMessages || this.config.maxNumberOfMessages,
+				WaitTimeSeconds: this.config.waitTimeSeconds,
+				MessageAttributeNames: ["All"],
+				AttributeNames: ["All"],
+			});
+
+			const response = await this.sqsClient.send(command);
+
+			if (response.Messages && response.Messages.length > 0) {
+				for (const msg of response.Messages) {
+					// Parse SNS message wrapper
+					let snsMessage: any;
+					let body: unknown;
+
+					try {
+						snsMessage = JSON.parse(msg.Body || "{}");
+						// SNS wraps the actual message in a "Message" field
+						if (snsMessage.Type === "Notification" && snsMessage.Message) {
+							try {
+								body = JSON.parse(snsMessage.Message);
+							} catch {
+								body = snsMessage.Message;
+							}
+						} else {
+							body = snsMessage;
+						}
+					} catch {
+						body = msg.Body;
+						snsMessage = {};
+					}
+
+					// Extract attributes from both SQS and SNS
+					const attributes: Record<string, string> = {};
+
+					// SQS message attributes
+					if (msg.MessageAttributes) {
+						for (const [key, attr] of Object.entries(msg.MessageAttributes)) {
+							attributes[key] = (attr as any).StringValue || "";
+						}
+					}
+
+					// SNS message attributes (if present)
+					if (snsMessage.MessageAttributes) {
+						for (const [key, attr] of Object.entries(snsMessage.MessageAttributes)) {
+							attributes[`sns_${key}`] = (attr as any).Value || "";
+						}
+					}
+
+					// Create pub/sub message
+					const pubsubMessage: PubSubMessage = {
+						id: snsMessage.MessageId || msg.MessageId || uuid(),
+						body,
+						attributes,
+						raw: msg,
+						topic: snsMessage.TopicArn || config.topic,
+						subscription: queueUrl,
+						publishTime: snsMessage.Timestamp ? new Date(snsMessage.Timestamp) : new Date(),
+						ack: async () => {
+							const deleteCommand = new DeleteMessageCommand({
+								QueueUrl: queueUrl,
+								ReceiptHandle: msg.ReceiptHandle,
+							});
+							await this.sqsClient.send(deleteCommand);
+						},
+						nack: async () => {
+							// Let the visibility timeout expire to return the message
+							// Or change visibility to 0 for immediate retry
+							const { ChangeMessageVisibilityCommand } = await import("@aws-sdk/client-sqs");
+							const changeCommand = new ChangeMessageVisibilityCommand({
+								QueueUrl: queueUrl,
+								ReceiptHandle: msg.ReceiptHandle,
+								VisibilityTimeout: 0,
+							});
+							await this.sqsClient.send(changeCommand);
+						},
+					};
+
+					// Process message
+					try {
+						await handler(pubsubMessage);
+					} catch (error) {
+						console.error(`[AWSSNSAdapter] Error processing message: ${(error as Error).message}`);
+					}
+				}
+			}
+		} catch (error) {
+			console.error(`[AWSSNSAdapter] Polling error: ${(error as Error).message}`);
+		}
+
+		// Continue polling unless stopped
+		if (!this.shouldStop) {
+			const timeout = setTimeout(() => this.poll(queueUrl, config, handler), 0);
+			this.pollingIntervals.set(queueUrl, timeout);
+		}
+	}
+
+	/**
+	 * Unsubscribe from a queue (stops polling)
+	 */
+	async unsubscribe(queueUrl: string): Promise<void> {
+		const interval = this.pollingIntervals.get(queueUrl);
+		if (interval) {
+			clearTimeout(interval);
+			this.pollingIntervals.delete(queueUrl);
+			console.log(`[AWSSNSAdapter] Unsubscribed from queue: ${queueUrl}`);
+		}
+	}
+
+	/**
+	 * Check if connected to AWS
+	 */
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	/**
+	 * Health check - verify AWS connectivity
+	 */
+	async healthCheck(): Promise<boolean> {
+		if (!this.connected) return false;
+
+		try {
+			const { ListQueuesCommand } = await import("@aws-sdk/client-sqs");
+			const command = new ListQueuesCommand({ MaxResults: 1 });
+			await this.sqsClient.send(command);
+			return true;
+		} catch {
+			return false;
+		}
+	}
+}
+
+export default AWSSNSAdapter;