@blokjs/trigger-grpc 0.2.0

package/package.json

@@ -0,0 +1,60 @@
+{
+	"name": "@blokjs/trigger-grpc",
+	"version": "0.2.0",
+	"description": "",
+	"type": "module",
+	"engines": {
+		"node": ">=18.0.0"
+	},
+	"main": "dist/index.js",
+	"types": "dist/index.d.ts",
+	"repository": "https://github.com/well-prado/blok.git",
+	"author": "Marco A. Castillo Della Sera",
+	"license": "Apache-2.0",
+	"scripts": {
+		"dev": "bun --watch run src/index.ts",
+		"start": "bun run dist/index.js",
+		"reload": "bun --env-file=.env.local run src/index.ts",
+		"build": "rimraf ./dist && tsc && copyfiles -u 1 proto/** dist/proto",
+		"build:proto": "npx buf generate .",
+		"test": "vitest run",
+		"test:dev": "vitest"
+	},
+	"devDependencies": {
+		"@bufbuild/buf": "^1.64.0",
+		"@bufbuild/protoc-gen-es": "^2.11.0",
+		"@connectrpc/protoc-gen-connect-es": "^1.7.0",
+		"@types/node": "^22.15.21",
+		"copyfiles": "^2.4.1",
+		"rimraf": "^6.1.2",
+		"typescript": "^5.8.3",
+		"vitest": "^4.0.18"
+	},
+	"dependencies": {
+		"@bufbuild/protobuf": "^2.11.0",
+		"@connectrpc/connect": "^2.1.1",
+		"@connectrpc/connect-fastify": "^2.1.1",
+		"@connectrpc/connect-node": "^2.1.1",
+		"@blokjs/api-call": "workspace:*",
+		"@blokjs/helper": "workspace:*",
+		"@blokjs/if-else": "workspace:*",
+		"@blokjs/runner": "workspace:*",
+		"@blokjs/shared": "workspace:*",
+		"@opentelemetry/api": "^1.9.0",
+		"@opentelemetry/exporter-prometheus": "^0.57.2",
+		"@opentelemetry/resources": "^1.30.1",
+		"@opentelemetry/sdk-metrics": "^1.30.1",
+		"@opentelemetry/semantic-conventions": "^1.39.0",
+		"fast-xml-parser": "^5.3.4",
+		"fastify": "^5.7.2",
+		"uuid": "^11.1.0"
+	},
+	"peerDependencies": {
+		"@bufbuild/protoc-gen-es": "2.11.0",
+		"@connectrpc/connect": "2.1.1"
+	},
+	"private": false,
+	"publishConfig": {
+		"access": "public"
+	}
+}

package/proto/workflow.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+
+package blok.workflow.v1;
+
+message WorkflowRequest {
+    string Name = 1;
+    string Message = 2;
+    string Encoding = 3;
+    string Type = 4;
+}
+
+message WorkflowResponse {
+    string Message = 1;
+    string Encoding = 2;
+    string Type = 3;
+}
+
+enum MessageEncoding {
+    BASE64 = 0;
+    STRING = 1;
+}
+
+enum MessageType {
+    TEXT = 0;
+    JSON = 1;
+    XML = 2;
+    HTML = 3;
+    BINARY = 4;
+}
+
+service WorkflowService {
+    rpc ExecuteWorkflow (WorkflowRequest) returns (WorkflowResponse) {}
+}

package/src/GRpcTrigger.ts

@@ -0,0 +1,252 @@
+import type { ConnectRouter } from "@connectrpc/connect";
+import {
+	DefaultLogger,
+	type GlobalOptions,
+	NodeMap,
+	type ParamsDictionary,
+	TriggerBase,
+	type TriggerResponse,
+} from "@blokjs/runner";
+import { type Context, GlobalError } from "@blokjs/shared";
+import { type Span, SpanStatusCode, metrics, trace } from "@opentelemetry/api";
+import fastify from "fastify";
+import { v4 as uuid } from "uuid";
+import MessageDecode from "./MessageDecode";
+import nodes from "./Nodes";
+import workflows from "./Workflows";
+import type RuntimeWorkflow from "./types/RuntimeWorkflow";
+
+import { type Step, Workflow } from "@blokjs/helper";
+import type { TriggerOpts } from "@blokjs/helper/dist/types/TriggerOpts";
+import {
+	MessageEncoding,
+	MessageType,
+	type WorkflowRequest,
+	type WorkflowResponse,
+	WorkflowService,
+} from "./gen/workflow_pb";
+
+enum NodeTypes {
+	MODULE = "module",
+	LOCAL = "local",
+	PYTHON3 = "runtime.python3",
+}
+
+export default class GRpcTrigger extends TriggerBase {
+	private server = fastify({
+		http2: true,
+		// https: {
+		//     key: readFileSync("localhost+2-key.pem", "utf8"),
+		//     cert: readFileSync("localhost+2.pem", "utf8"),
+		// }
+	});
+	private nodeMap: GlobalOptions = <GlobalOptions>{};
+	protected tracer = trace.getTracer(
+		process.env.PROJECT_NAME || "trigger-grpc-workflow",
+		process.env.PROJECT_VERSION || "0.0.1",
+	);
+	private logger = new DefaultLogger();
+
+	constructor() {
+		super();
+		this.loadNodes();
+		this.loadWorkflows();
+	}
+
+	getApp() {
+		return this.server;
+	}
+
+	async listen(): Promise<number> {
+		return 0;
+	}
+
+	loadNodes() {
+		this.nodeMap.nodes = new NodeMap();
+		const nodeKeys = Object.keys(nodes);
+		for (const key of nodeKeys) {
+			this.nodeMap.nodes.addNode(key, nodes[key]);
+		}
+	}
+
+	loadWorkflows() {
+		this.nodeMap.workflows = workflows;
+	}
+
+	processRequest(router: ConnectRouter, trigger: GRpcTrigger) {
+		router.service(WorkflowService, {
+			executeWorkflow: (request: WorkflowRequest) => trigger.executeWorkflow(request),
+		});
+	}
+
+	async executeWorkflow(request: WorkflowRequest) {
+		const start = performance.now();
+		const coder = new MessageDecode();
+		let name = request.Name;
+		const messageContext: Context = coder.requestDecode(request);
+		const runtimeWorkflow = messageContext as unknown as RuntimeWorkflow;
+		const id: string = (messageContext.request.query?.requestId as string) || (uuid() as string);
+		let remoteNodeExecution = false;
+
+		const defaultMeter = metrics.getMeter("default");
+		const workflow_runner_errors = defaultMeter.createCounter("workflow_errors", {
+			description: "Workflow runner errors",
+		});
+
+		return await this.tracer.startActiveSpan(`${name}`, async (span: Span) => {
+			try {
+				if (runtimeWorkflow !== undefined) {
+					const workflowModel = runtimeWorkflow.workflow;
+					const node_type = (workflowModel.steps[0] as unknown as ParamsDictionary).type;
+					let set_node_type: NodeTypes = NodeTypes.MODULE;
+					switch (node_type) {
+						case "runtime.python3":
+							set_node_type = NodeTypes.PYTHON3;
+							break;
+						case "local":
+							set_node_type = NodeTypes.LOCAL;
+							break;
+						default:
+							set_node_type = NodeTypes.MODULE;
+							break;
+					}
+
+					const trigger = Object.keys(workflowModel.trigger)[0];
+					const trigger_config =
+						((workflowModel.trigger as unknown as ParamsDictionary)[trigger] as unknown as TriggerOpts) || {};
+
+					const step: Step = Workflow({
+						name: `Remote Node: ${name}`,
+						version: "1.0.0",
+						description: "Remote Node",
+					})
+						.addTrigger((trigger as unknown as "http") || "grpc", trigger_config)
+						.addStep({
+							name: "node",
+							node: name,
+							type: set_node_type,
+							inputs: ((workflowModel.nodes as unknown as ParamsDictionary).node as unknown as ParamsDictionary).inputs,
+						});
+
+					this.nodeMap.workflows[id] = step;
+					name = id;
+					remoteNodeExecution = true;
+				}
+
+				await this.configuration.init(name, this.nodeMap);
+				let ctx: Context = this.createContext(undefined, name, id);
+				ctx.request = messageContext.request;
+				ctx.logger.log(`Workflow: ${name}, Version: ${this.configuration.version}`);
+
+				const response: TriggerResponse = await this.run(ctx);
+				ctx = response.ctx;
+				const average = response.metrics;
+
+				if (ctx.response.contentType === undefined || ctx.response.contentType === "")
+					ctx.response.contentType = "application/json";
+
+				const end = performance.now();
+				ctx.logger.log(`Completed in ${(end - start).toFixed(2)}ms`);
+
+				span.setAttribute("success", true);
+				span.setAttribute("Content-Type", ctx.response.contentType as string);
+				span.setAttribute("workflow_request_id", `${ctx.id}`);
+				span.setAttribute("workflow_elapsed_time", `${end - start}`);
+				span.setAttribute("workflow_version", `${this.configuration.version}`);
+				span.setAttribute("workflow_name", `${this.configuration.name}`);
+				span.setAttribute("workflow_memory_avg_mb", `${average.memory.total}`);
+				span.setAttribute("workflow_memory_min_mb", `${average.memory.min}`);
+				span.setAttribute("workflow_memory_max_mb", `${average.memory.max}`);
+				span.setAttribute("workflow_cpu_percentage", `${average.cpu.average}`);
+				span.setAttribute("workflow_cpu_total", `${average.cpu.total}`);
+				span.setAttribute("workflow_cpu_usage", `${average.cpu.usage}`);
+				span.setAttribute("workflow_cpu_model", `${average.cpu.model}`);
+				span.setStatus({ code: SpanStatusCode.OK });
+
+				return coder.responseEncode(ctx, request.Encoding, request.Type);
+			} catch (e: unknown) {
+				span.setAttribute("success", false);
+				span.setAttribute("workflow_request_id", `${id}`);
+				span.recordException(e as Error);
+				let message: WorkflowResponse = <WorkflowResponse>{};
+				const base64Key = MessageEncoding[MessageEncoding.BASE64];
+				const textKey = MessageType[MessageType.TEXT];
+				const jsonKey = MessageType[MessageType.JSON];
+
+				if (e instanceof GlobalError) {
+					const error_context = e as GlobalError;
+
+					if (error_context.context.message === "{}" && error_context.context.json instanceof DOMException) {
+						workflow_runner_errors.add(1, {
+							env: process.env.NODE_ENV,
+							workflow_version: `${this.configuration.version || "unknown"}`,
+							workflow_name: `${name || this.configuration.name}`,
+						});
+						span.setStatus({
+							code: SpanStatusCode.ERROR,
+							message: (error_context.context.json as Error).toString(),
+						});
+
+						this.logger.error(`${(error_context.context.json as Error).toString()}`);
+						message = {
+							Message: coder.responseErrorEncode((error_context.context.json as Error).toString(), base64Key, textKey),
+							Encoding: base64Key,
+							Type: textKey,
+						} as WorkflowResponse;
+					} else {
+						if (error_context.context.code === undefined) error_context.setCode(500);
+
+						if (error_context.hasJson()) {
+							workflow_runner_errors.add(1, {
+								env: process.env.NODE_ENV,
+								workflow_version: `${this.configuration.version || "unknown"}`,
+								workflow_name: `${name || this.configuration.name}`,
+							});
+							span.setStatus({ code: SpanStatusCode.ERROR, message: JSON.stringify(error_context.context.json) });
+							this.logger.error(`${JSON.stringify(error_context.context.json)}`);
+							message = {
+								Message: coder.responseErrorEncode(JSON.stringify(error_context.context.json), base64Key, textKey),
+								Encoding: base64Key,
+								Type: jsonKey,
+							} as WorkflowResponse;
+						} else {
+							workflow_runner_errors.add(1, {
+								env: process.env.NODE_ENV,
+								workflow_version: `${this.configuration.version || "unknown"}`,
+								workflow_name: `${name || this.configuration.name}`,
+							});
+							span.setStatus({ code: SpanStatusCode.ERROR, message: error_context.message });
+							this.logger.error(`${error_context.message}`, error_context.stack?.replace(/\n/g, " "));
+							message = {
+								Message: coder.responseErrorEncode(error_context.message, base64Key, textKey),
+								Encoding: MessageEncoding[MessageEncoding.BASE64],
+								Type: textKey,
+							} as WorkflowResponse;
+						}
+					}
+				} else {
+					workflow_runner_errors.add(1, {
+						env: process.env.NODE_ENV,
+						workflow_version: `${this.configuration.version || "unknown"}`,
+						workflow_name: `${name || this.configuration.name}`,
+					});
+					span.setStatus({ code: SpanStatusCode.ERROR, message: (e as Error).message });
+					this.logger.error(`${(e as Error).message}`, `${(e as Error).stack?.replace(/\n/g, " ")}`);
+
+					message = {
+						Message: coder.responseErrorEncode((e as Error).message, base64Key, textKey),
+						Encoding: base64Key,
+						Type: textKey,
+					} as WorkflowResponse;
+				}
+
+				return message;
+			} finally {
+				if (remoteNodeExecution) {
+					delete this.nodeMap.workflows[id];
+				}
+				span.end();
+			}
+		});
+	}
+}

package/src/GrpcClient.ts

@@ -0,0 +1,87 @@
+import { type Transport, createClient } from "@connectrpc/connect";
+import { createGrpcTransport } from "@connectrpc/connect-node";
+import { createConnectTransport } from "@connectrpc/connect-node";
+import { createGrpcWebTransport } from "@connectrpc/connect-node";
+import { type WorkflowRequest, type WorkflowResponse, WorkflowService } from "./gen/workflow_pb";
+
+export type RpcOptions = {
+	host: string;
+	port: number;
+	protocol: string;
+	httpVersion: HttpVersionEnum;
+	transport: TransportEnum;
+};
+
+export type CallOptions = {
+	headers: Record<string, string>;
+};
+
+export enum TransportEnum {
+	GRPC = "grpc",
+	GRPC_WEB = "grpc-web",
+	CONNECT = "connect",
+}
+
+export enum HttpVersionEnum {
+	HTTP1 = "1.1",
+	HTTP2 = "2",
+}
+
+export type { WorkflowRequest, WorkflowResponse };
+
+export default class GrpcClient {
+	protected opts: RpcOptions;
+
+	constructor(options: RpcOptions) {
+		this.opts = options;
+	}
+
+	async call(message: WorkflowRequest, opts?: CallOptions): Promise<WorkflowResponse> {
+		const transport = this.transport();
+		const client = createClient(WorkflowService, transport);
+		return await client.executeWorkflow(message, opts);
+	}
+
+	transport(): Transport {
+		switch (this.opts.transport) {
+			case TransportEnum.GRPC:
+				return createGrpcTransport({
+					baseUrl: `${this.opts.protocol}://${this.opts.host}:${this.opts.port}/`,
+					interceptors: [],
+				});
+			case TransportEnum.GRPC_WEB:
+				return createGrpcWebTransport({
+					baseUrl: `${this.opts.protocol}://${this.opts.host}:${this.opts.port}/`,
+					httpVersion: this.opts.httpVersion,
+					interceptors: [],
+				});
+			case TransportEnum.CONNECT:
+				return createConnectTransport({
+					baseUrl: `${this.opts.protocol}://${this.opts.host}:${this.opts.port}/`,
+					httpVersion: this.opts.httpVersion,
+					interceptors: [],
+				});
+			default:
+				throw new Error("Invalid transport type");
+		}
+	}
+}
+
+// let client = new GrpcClient({
+//     host: "localhost",
+//     port: 8433,
+//     protocol: "http",
+//     httpVersion: HttpVersionEnum.HTTP2,
+//     transport: TransportEnum.GRPC,
+// });
+
+// client.call({
+//     "Name": "countries",
+//     "Message": "ewogICAgInJlcXVlc3QiOiB7CiAgICAgICAgImJvZHkiOiB7CiAgICAgICAgICAgICJjb3VudHJ5X25hbWUiOiAiQ2FuYWRhIgogICAgICAgIH0KICAgIH0KfQ==",
+//     "Encoding": "BASE64",
+//     "Type": "JSON"
+// } as WorkflowRequest).then((response) => {
+//     console.log(response);
+// }).catch((error) => {
+//     console.error(error);
+// });

package/src/GrpcServer.ts

@@ -0,0 +1,63 @@
+import type { ConnectRouter } from "@connectrpc/connect";
+import { fastifyConnectPlugin } from "@connectrpc/connect-fastify";
+import { DefaultLogger } from "@blokjs/runner";
+import { type Span, metrics, trace } from "@opentelemetry/api";
+import GRpcTrigger from "./GRpcTrigger";
+
+export type GrpcServerOptions = {
+	host: string;
+	port: number;
+};
+
+export default class GrpcServer {
+	protected opts: GrpcServerOptions;
+	protected tracer = trace.getTracer(
+		process.env.PROJECT_NAME || "trigger-grpc-server",
+		process.env.PROJECT_VERSION || "0.0.1",
+	);
+	protected app_cold_start = metrics.getMeter("default").createGauge("initialization", {
+		description: "Application cold start",
+	});
+	protected initializer = 0;
+	protected logger = new DefaultLogger();
+
+	constructor(opts: GrpcServerOptions) {
+		this.opts = opts;
+
+		if (this.opts.host === undefined) {
+			this.opts.host = "0.0.0.0";
+		}
+
+		if (this.opts.port === undefined) {
+			this.opts.port = 8443;
+		}
+
+		this.initializer = performance.now();
+	}
+
+	async start() {
+		this.tracer.startActiveSpan("initialization", async (span: Span) => {
+			const trigger = new GRpcTrigger();
+			const server = trigger.getApp();
+			const host = process.env.GRPC_HOST || this.opts.host;
+			let port: string | number = process.env.GRPC_PORT || this.opts.port;
+			if (typeof port === "string") {
+				port = Number.parseInt(port, 10);
+			}
+			await server.register(fastifyConnectPlugin, {
+				routes: (router: ConnectRouter) => trigger.processRequest(router, trigger),
+			});
+			await server.listen({ host, port: port });
+			this.logger.log(`Server is listening at ${JSON.stringify(server.addresses())}`);
+
+			this.initializer = performance.now() - this.initializer;
+			this.logger.log(`Server initialized in ${(this.initializer).toFixed(2)}ms`);
+			this.app_cold_start.record(this.initializer, {
+				pid: process.pid,
+				env: process.env.NODE_ENV,
+				app: process.env.APP_NAME,
+			});
+			span.end();
+		});
+	}
+}

package/src/MessageDecode.ts

@@ -0,0 +1,142 @@
+import type { JsonLikeObject } from "@blokjs/runner";
+import type { Context } from "@blokjs/shared";
+import { XMLBuilder, XMLParser } from "fast-xml-parser";
+import { MessageEncoding, MessageType, type WorkflowRequest, type WorkflowResponse } from "./gen/workflow_pb";
+
+export default class MessageDecode {
+	requestDecode(request: WorkflowRequest): Context {
+		let message: Context = <Context>{};
+
+		switch (request.Encoding) {
+			case MessageEncoding[MessageEncoding.BASE64]: {
+				const messageStr = Buffer.from(request.Message, "base64").toString("utf-8");
+				message = this.decodeType(messageStr, request.Type);
+				break;
+			}
+			case MessageEncoding[MessageEncoding.STRING]: {
+				message = this.decodeType(request.Message, request.Type);
+				break;
+			}
+			default:
+				throw new Error(`Unsupported encoding: ${request.Encoding}`);
+		}
+
+		return message;
+	}
+
+	decodeType(message: string, type: string): Context {
+		switch (type) {
+			case MessageType[MessageType.JSON]: {
+				return JSON.parse(message);
+			}
+			case MessageType[MessageType.XML]: {
+				return new XMLParser().parse(message);
+			}
+			default:
+				throw new Error(`Unsupported type: ${type}`);
+		}
+	}
+
+	responseEncode(ctx: Context, encoding: string, type: string): WorkflowResponse {
+		let message: string | object | Buffer<ArrayBuffer>;
+		const responseType = this.mapContentType(ctx.response.contentType as string);
+		switch (encoding) {
+			case MessageEncoding[MessageEncoding.BASE64]: {
+				if (responseType === MessageType.JSON) {
+					message = this.encodeType(ctx.response.data as JsonLikeObject, type);
+					message = Buffer.from(message).toString("base64");
+				}
+				if (responseType === MessageType.XML) {
+					message = this.encodeType(ctx.response.data as object, type);
+					message = Buffer.from(message).toString("base64");
+				} else {
+					message = this.encodeType(ctx.response.data as string, type);
+					message = Buffer.from(message).toString("base64");
+				}
+				break;
+			}
+			case MessageEncoding[MessageEncoding.STRING]: {
+				if (responseType === MessageType.JSON) {
+					message = this.encodeType(ctx.response.data as JsonLikeObject, type);
+				}
+				if (responseType === MessageType.XML) {
+					message = this.encodeType(ctx.response.data as object, type);
+				} else {
+					message = this.encodeType(ctx.response.data as string, type);
+				}
+				break;
+			}
+			default:
+				throw new Error(`Unsupported encoding: ${encoding}`);
+		}
+
+		return {
+			Message: message,
+			Encoding: encoding,
+			Type: MessageType[responseType],
+		} as WorkflowResponse;
+	}
+
+	responseErrorEncode(e: string | JsonLikeObject, encoding: string, type: string): string {
+		let message: string | object | Buffer<ArrayBuffer>;
+		switch (encoding) {
+			case MessageEncoding[MessageEncoding.BASE64]:
+				message = Buffer.from(this.encodeType(e, type)).toString("base64");
+				break;
+			case MessageEncoding[MessageEncoding.STRING]:
+				message = this.encodeType(e, type);
+				break;
+			default:
+				throw new Error(`Unsupported encoding: ${encoding}`);
+		}
+
+		return message as string;
+	}
+
+	responseDecode(response: WorkflowResponse): JsonLikeObject {
+		let message: JsonLikeObject = {};
+
+		switch (response.Encoding) {
+			case MessageEncoding[MessageEncoding.BASE64]: {
+				const messageStr = Buffer.from(response.Message, "base64").toString("utf-8");
+				message = this.decodeType(messageStr, response.Type) as unknown as JsonLikeObject;
+				break;
+			}
+			case MessageEncoding[MessageEncoding.STRING]: {
+				message = this.decodeType(response.Message, response.Type) as unknown as JsonLikeObject;
+				break;
+			}
+			default:
+				throw new Error(`Unsupported encoding: ${response.Encoding}`);
+		}
+
+		return message;
+	}
+
+	encodeType(message: string | object | Buffer<ArrayBuffer>, type: string): string {
+		switch (type) {
+			case MessageType[MessageType.JSON]:
+				return JSON.stringify(message);
+			case MessageType[MessageType.TEXT]:
+			case MessageType[MessageType.HTML]:
+				return message.toString();
+			case MessageType[MessageType.XML]:
+				return new XMLBuilder().build(message);
+			default:
+				throw new Error(`Unsupported type: ${type}`);
+		}
+	}
+
+	mapContentType(contentType: string): MessageType {
+		switch (contentType) {
+			case "application/json":
+				return MessageType.JSON;
+			case "text/html":
+				return MessageType.HTML;
+			case "text/xml":
+				return MessageType.XML;
+			default:
+				return MessageType.TEXT;
+		}
+	}
+}