@brandboostinggmbh/observable-workflows 0.8.6 → 0.10.0

package/dist/index.d.ts

@@ -406,11 +406,12 @@ declare const createLogAccessor: (context: {
       limit?: number;
       offset?: number;
       instanceId?: string;
+      populateData?: boolean;
     }): Promise<Step[]>;
   };
   getStep: (instanceId: string, stepName: string) => Promise<Step | null>;
   listWorkflows: (limit: number, offset: number, filter?: WorkflowFilter) => Promise<WorkflowRun[]>;
-  getWorkflow: (instanceId: string) => Promise<WorkflowRun | null>;
+  getWorkflow: (instanceId: string, populateData?: boolean) => Promise<WorkflowRun | null>;
   getWorkflowTypesByTenantId: (tenantId: string) => Promise<string[]>;
   getPropertiesKeys: (instanceId?: string) => Promise<WorkflowPropertyDefinition[]>;
 };
@@ -419,6 +420,12 @@ declare const createLogAccessor: (context: {
 //#region src/observableWorkflows/createQueueWorkflowContext.d.ts
 declare function createQueueWorkflowContext(options: QueueWorkflowContextOptions): {
   enqueueWorkflow: <I>(workflow: WorkflowFunction<I>, tenantId: string, input: I, initialName: string) => Promise<void>;
+  enqueueWorkflowBatch: <I>(workflows: Array<{
+    workflow: WorkflowFunction<I>;
+    tenantId: string;
+    input: I;
+    initialName: string;
+  }>) => Promise<void>;
   enqueueRetryWorkflow: <I>(workflow: WorkflowFunction<I>, tenantId: string, oldInstanceId: string) => Promise<void>;
   handleWorkflowQueueMessage: (message: WorkflowQueueMessage, env: {
     LOG_DB: D1Database;

package/dist/index.js

@@ -247,6 +247,49 @@ function pushLogToDB(options, { instanceId, stepName, message, timestamp, type,
            VALUES (?, ?, ?, ?, ?, ?, ?)`
 ).bind(instanceId, stepName, message, timestamp, type, logOrder, tenantId).run();
 }
+var LogBatcher = class {
+	batch = [];
+	flushTimer = null;
+	isDestroyed = false;
+	constructor(options, batchSize = 100, flushInterval = 2e3) {
+		this.options = options;
+		this.batchSize = batchSize;
+		this.flushInterval = flushInterval;
+	}
+	addLog(entry) {
+		if (this.isDestroyed) return pushLogToDB(this.options, entry);
+		this.batch.push(entry);
+		if (!this.flushTimer) this.flushTimer = setTimeout(() => {
+			this.flush();
+		}, this.flushInterval);
+		if (this.batch.length >= this.batchSize) return this.flush();
+		return Promise.resolve();
+	}
+	async flush() {
+		if (this.flushTimer) {
+			clearTimeout(this.flushTimer);
+			this.flushTimer = null;
+		}
+		if (this.batch.length === 0) return;
+		const logsToFlush = this.batch.splice(0);
+		try {
+			const statements = logsToFlush.map((entry) => this.options.D1.prepare(
+				/* sql */
+				`INSERT INTO LogTable
+                 (instanceId, stepName, log, timestamp, type, logOrder, tenantId)
+                 VALUES (?, ?, ?, ?, ?, ?, ?)`
+).bind(entry.instanceId, entry.stepName, entry.message, entry.timestamp, entry.type, entry.logOrder, entry.tenantId));
+			await this.options.D1.batch(statements);
+		} catch (error) {
+			console.error("Batch log insert failed, falling back to individual inserts:", error);
+			await Promise.all(logsToFlush.map((entry) => pushLogToDB(this.options, entry)));
+		}
+	}
+	async destroy() {
+		this.isDestroyed = true;
+		await this.flush();
+	}
+};
 function trySerializeObj(obj, serializer) {
 	try {
 		return serializer.serialize(obj);
@@ -401,46 +444,44 @@ const createLogAccessor = (context) => {
 		const selectClause = "SELECT DISTINCT w.*";
 		const fromClause = "FROM WorkflowTable w";
 		const propertyWhereConditions = [];
-		if (filter) {
-			if (filter.properties && Object.keys(filter.properties).length > 0) {
-				let propIndex = 0;
-				for (const [key, propertyFilter] of Object.entries(filter.properties)) {
-					const propAlias = `p${propIndex}`;
-					joinClause += ` LEFT JOIN WorkflowProperties ${propAlias} ON w.instanceId = ${propAlias}.instanceId AND ${propAlias}.key = ? AND ${propAlias}.tenantId = ?`;
-					bindings.push(key, context.tenantId);
-					if (propertyFilter.equals !== void 0) propertyWhereConditions.push({
-						condition: `${propAlias}.value = ?`,
-						binding: String(propertyFilter.equals)
-					});
-					if (propertyFilter.contains !== void 0) propertyWhereConditions.push({
-						condition: `${propAlias}.value LIKE ?`,
-						binding: `%${propertyFilter.contains}%`
-					});
-					if (propertyFilter.gt !== void 0) propertyWhereConditions.push({
-						condition: `CAST(${propAlias}.value AS REAL) > ?`,
-						binding: propertyFilter.gt
-					});
-					if (propertyFilter.gte !== void 0) propertyWhereConditions.push({
-						condition: `CAST(${propAlias}.value AS REAL) >= ?`,
-						binding: propertyFilter.gte
-					});
-					if (propertyFilter.lt !== void 0) propertyWhereConditions.push({
-						condition: `CAST(${propAlias}.value AS REAL) < ?`,
-						binding: propertyFilter.lt
+		if (filter && filter.properties && Object.keys(filter.properties).length > 0) {
+			let propIndex = 0;
+			for (const [key, propertyFilter] of Object.entries(filter.properties)) {
+				const propAlias = `p${propIndex}`;
+				joinClause += ` LEFT JOIN WorkflowProperties ${propAlias} ON w.instanceId = ${propAlias}.instanceId AND ${propAlias}.key = ? AND ${propAlias}.tenantId = ?`;
+				bindings.push(key, context.tenantId);
+				if (propertyFilter.equals !== void 0) propertyWhereConditions.push({
+					condition: `${propAlias}.value = ?`,
+					binding: String(propertyFilter.equals)
+				});
+				if (propertyFilter.contains !== void 0) propertyWhereConditions.push({
+					condition: `${propAlias}.value LIKE ?`,
+					binding: `%${propertyFilter.contains}%`
+				});
+				if (propertyFilter.gt !== void 0) propertyWhereConditions.push({
+					condition: `CAST(${propAlias}.value AS REAL) > ?`,
+					binding: propertyFilter.gt
+				});
+				if (propertyFilter.gte !== void 0) propertyWhereConditions.push({
+					condition: `CAST(${propAlias}.value AS REAL) >= ?`,
+					binding: propertyFilter.gte
+				});
+				if (propertyFilter.lt !== void 0) propertyWhereConditions.push({
+					condition: `CAST(${propAlias}.value AS REAL) < ?`,
+					binding: propertyFilter.lt
+				});
+				if (propertyFilter.lte !== void 0) propertyWhereConditions.push({
+					condition: `CAST(${propAlias}.value AS REAL) <= ?`,
+					binding: propertyFilter.lte
+				});
+				if (propertyFilter.in !== void 0 && Array.isArray(propertyFilter.in)) {
+					const placeholders = propertyFilter.in.map(() => "?").join(", ");
+					propertyWhereConditions.push({
+						condition: `${propAlias}.value IN (${placeholders})`,
+						bindings: propertyFilter.in.map(String)
 					});
-					if (propertyFilter.lte !== void 0) propertyWhereConditions.push({
-						condition: `CAST(${propAlias}.value AS REAL) <= ?`,
-						binding: propertyFilter.lte
-					});
-					if (propertyFilter.in !== void 0 && Array.isArray(propertyFilter.in)) {
-						const placeholders = propertyFilter.in.map(() => "?").join(", ");
-						propertyWhereConditions.push({
-							condition: `${propAlias}.value IN (${placeholders})`,
-							bindings: propertyFilter.in.map(String)
-						});
-					}
-					propIndex++;
 				}
+				propIndex++;
 			}
 		}
 		propertyWhereConditions.forEach((item) => {
@@ -533,6 +574,7 @@ const createLogAccessor = (context) => {
 		let limit;
 		let actualOffset;
 		let actualInstanceId;
+		let populateData = void 0;
 		if (typeof limitOrOptions === "number") {
 			limit = limitOrOptions;
 			actualOffset = offset;
@@ -542,7 +584,9 @@ const createLogAccessor = (context) => {
 			limit = options.limit;
 			actualOffset = options.offset;
 			actualInstanceId = options.instanceId;
+			populateData = options.populateData;
 		}
+		if (populateData === void 0) populateData = false;
 		let result;
 		if (limit !== void 0 && actualOffset !== void 0) result = await context.D1.prepare(
 			/* sql */
@@ -558,6 +602,12 @@ const createLogAccessor = (context) => {
 ).bind(context.tenantId, actualInstanceId ?? null, actualInstanceId ?? null).all();
 		if (result.results) {
 			const steps = await Promise.all(result.results.map(async (row) => {
+				let deserializedResult = null;
+				let deserializedError = null;
+				if (populateData) {
+					deserializedResult = await deserializeWithExternalStorage(row.result, row.resultRef, internalSerializer, context.externalBlobStorage);
+					deserializedError = await deserializeWithExternalStorage(row.error, row.errorRef, internalSerializer, context.externalBlobStorage);
+				}
 				return {
 					instanceId: row.instanceId,
 					name: row.stepName,
@@ -565,8 +615,8 @@ const createLogAccessor = (context) => {
 					status: row.stepStatus,
 					startTime: row.startTime,
 					endTime: row.endTime,
-					result: null,
-					error: null
+					result: deserializedResult,
+					error: deserializedError
 				};
 			}));
 			return steps;
@@ -623,10 +673,13 @@ const createLogAccessor = (context) => {
 		const workflow = await workflowTableRowToWorkflowRun(result, internalSerializer, context.externalBlobStorage);
 		return workflow;
 	};
-	const getWorkflow = async (instanceId) => {
+	const getWorkflow = async (instanceId, populateData = false) => {
 		const workflow = await getWorkflowShallow(instanceId);
 		if (!workflow) return null;
-		const steps = await listSteps({ instanceId });
+		const steps = await listSteps({
+			instanceId,
+			populateData
+		});
 		const retryWorkflows = await getWorkflowByParentId(instanceId);
 		const parentWorkflow = workflow.parentInstanceId ? await getWorkflowShallow(workflow.parentInstanceId) : null;
 		workflow.retries = retryWorkflows;
@@ -732,6 +785,15 @@ function createQueueWorkflowContext(options) {
 			tenantId
 		});
 	};
+	const enqueueWorkflowBatch = async (workflows) => {
+		await options.QUEUE.sendBatch(workflows.map(({ workflow, tenantId, input, initialName }) => ({ body: {
+			type: "workflow-run",
+			workflowType: workflow.workflowType,
+			workflowName: initialName,
+			input,
+			tenantId
+		} })));
+	};
 	const handleWorkflowQueueMessage = async (message, env, workflowResolver) => {
 		const workflowFunction = workflowResolver(message.workflowType);
 		if (!workflowFunction) throw new Error(`Workflow ${message.workflowType} not found`);
@@ -751,6 +813,7 @@ function createQueueWorkflowContext(options) {
 	};
 	return {
 		enqueueWorkflow,
+		enqueueWorkflowBatch,
 		enqueueRetryWorkflow,
 		handleWorkflowQueueMessage
 	};
@@ -766,13 +829,10 @@ async function createStepContext(context) {
 		const stepNameParam = typeof step$1 === "string" ? step$1 : step$1.name;
 		const stepMetadataParam = typeof step$1 === "string" ? void 0 : step$1.metadata;
 		if (context.parentInstanceId && reuseSuccessfulSteps) {
-			console.warn("temp: try to reuse successful steps");
 			const existingStep = await getStepRecord(context, stepNameParam, context.parentInstanceId);
 			if (existingStep) {
-				console.warn("temp: found existing step", existingStep);
 				const row = existingStep;
 				if (row.status === "completed") {
-					console.warn("temp: found existing step with completed status", row);
 					const { data: resultData, externalRef: resultRef } = await serializeWithExternalStorage(row.result, context.serializer, context.externalBlobStorage);
 					const { data: errorData, externalRef: errorRef } = await serializeWithExternalStorage(row.error, context.serializer, context.externalBlobStorage);
 					await insertStepRecordFull(context, {
@@ -788,10 +848,10 @@ async function createStepContext(context) {
 						errorRef
 					});
 					return row.result;
-				} else console.warn("temp: found existing step with different status", row);
-			} else console.warn("temp: no existing step found");
-		} else console.warn("temp: not trying to reuse successful steps");
-		const waitFor = [];
+				}
+			}
+		}
+		const logBatcher = new LogBatcher(context);
 		const startTime = Date.now();
 		const stepStatus = "pending";
 		const stepMetadata = context.serializer.serialize(stepMetadataParam || {});
@@ -812,7 +872,7 @@ async function createStepContext(context) {
 		const log = (message, type = "info") => {
 			logOrder++;
 			const timestamp = Date.now();
-			const logPromise = pushLogToDB(context, {
+			logBatcher.addLog({
 				stepName: stepNameParam,
 				instanceId,
 				message,
@@ -821,7 +881,6 @@ async function createStepContext(context) {
 				logOrder,
 				tenantId: context.tenantId
 			});
-			waitFor.push(logPromise);
 		};
 		const ctx = { console: {
 			log: (message, ...optionalParams) => {
@@ -849,7 +908,7 @@ async function createStepContext(context) {
           SET stepStatus = ?, endTime = ?, result = ?, error = ?, resultRef = ?, errorRef = ?
           WHERE instanceId = ? AND stepName = ?`
 ).bind(stepStatus$1, endTime, resultData, stepError$1, resultRef, null, instanceId, stepName).run();
-			await Promise.allSettled(waitFor);
+			await logBatcher.destroy();
 			return result;
 		} catch (error) {
 			const endTime = Date.now();
@@ -862,7 +921,7 @@ async function createStepContext(context) {
           SET stepStatus = ?, endTime = ?, result = ?, error = ?, resultRef = ?, errorRef = ?
           WHERE instanceId = ? AND stepName = ?`
 ).bind(stepStatus$1, endTime, stepResult$1, errorData, null, errorRef, instanceId, stepName).run();
-			await Promise.allSettled(waitFor);
+			await logBatcher.destroy();
 			throw error;
 		}
 	};
@@ -897,12 +956,12 @@ function createWorkflowContext(options) {
 			parentInstanceId,
 			tenantId
 		});
-		const waitFor = [];
+		const logBatcher = new LogBatcher(internalContext);
 		let logOrder = 0;
 		const log = (message, type = "info") => {
 			logOrder++;
 			const timestamp = Date.now();
-			const logPromise = pushLogToDB(internalContext, {
+			logBatcher.addLog({
 				instanceId,
 				stepName: null,
 				message,
@@ -911,7 +970,6 @@ function createWorkflowContext(options) {
 				logOrder,
 				tenantId
 			});
-			waitFor.push(logPromise);
 		};
 		const stepContext = await createStepContext({
 			D1: options.D1,
@@ -961,7 +1019,7 @@ function createWorkflowContext(options) {
 				endTime,
 				instanceId
 			});
-			await Promise.allSettled(waitFor);
+			await logBatcher.destroy();
 		} catch (error) {
 			const endTime = Date.now();
 			const workflowStatus = "failed";
@@ -970,7 +1028,7 @@ function createWorkflowContext(options) {
 				endTime,
 				instanceId
 			});
-			await Promise.allSettled(waitFor);
+			await logBatcher.destroy();
 			throw error;
 		}
 	};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brandboostinggmbh/observable-workflows",
-  "version": "0.8.6",
+  "version": "0.10.0",
   "description": "My awesome typescript library",
   "type": "module",
   "license": "MIT",