@brandboostinggmbh/observable-workflows 0.8.5 → 0.9.0

package/dist/index.d.ts

@@ -406,11 +406,12 @@ declare const createLogAccessor: (context: {
       limit?: number;
       offset?: number;
       instanceId?: string;
+      populateData?: boolean;
     }): Promise<Step[]>;
   };
   getStep: (instanceId: string, stepName: string) => Promise<Step | null>;
   listWorkflows: (limit: number, offset: number, filter?: WorkflowFilter) => Promise<WorkflowRun[]>;
-  getWorkflow: (instanceId: string) => Promise<WorkflowRun | null>;
+  getWorkflow: (instanceId: string, populateData?: boolean) => Promise<WorkflowRun | null>;
   getWorkflowTypesByTenantId: (tenantId: string) => Promise<string[]>;
   getPropertiesKeys: (instanceId?: string) => Promise<WorkflowPropertyDefinition[]>;
 };

package/dist/index.js

@@ -207,7 +207,7 @@ async function insertWorkflowRecord(options, { instanceId, workflowType, workflo
 		`INSERT INTO WorkflowTable
         (instanceId, workflowType, workflowName, workflowMetadata, input, inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId)
         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
-).bind(instanceId, workflowType, workflowName, options.serializer.serialize(workflowMetadata), inputData ?? null, inputRef, tenantId, workflowStatus, startTime, endTime ?? null, parentInstanceId ?? null).run();
+).bind(instanceId, workflowType, workflowName, options.serializer.serialize(workflowMetadata), inputData, inputRef, tenantId, workflowStatus, startTime, endTime ?? null, parentInstanceId ?? null).run();
 }
 function insertStepRecordFull(context, { instanceId, name, status, metadata, startTime, endTime, result, error, resultRef, errorRef }) {
 	return context.D1.prepare(
@@ -247,6 +247,49 @@ function pushLogToDB(options, { instanceId, stepName, message, timestamp, type,
            VALUES (?, ?, ?, ?, ?, ?, ?)`
 ).bind(instanceId, stepName, message, timestamp, type, logOrder, tenantId).run();
 }
+var LogBatcher = class {
+	batch = [];
+	flushTimer = null;
+	isDestroyed = false;
+	constructor(options, batchSize = 100, flushInterval = 2e3) {
+		this.options = options;
+		this.batchSize = batchSize;
+		this.flushInterval = flushInterval;
+	}
+	addLog(entry) {
+		if (this.isDestroyed) return pushLogToDB(this.options, entry);
+		this.batch.push(entry);
+		if (!this.flushTimer) this.flushTimer = setTimeout(() => {
+			this.flush();
+		}, this.flushInterval);
+		if (this.batch.length >= this.batchSize) return this.flush();
+		return Promise.resolve();
+	}
+	async flush() {
+		if (this.flushTimer) {
+			clearTimeout(this.flushTimer);
+			this.flushTimer = null;
+		}
+		if (this.batch.length === 0) return;
+		const logsToFlush = this.batch.splice(0);
+		try {
+			const statements = logsToFlush.map((entry) => this.options.D1.prepare(
+				/* sql */
+				`INSERT INTO LogTable
+                 (instanceId, stepName, log, timestamp, type, logOrder, tenantId)
+                 VALUES (?, ?, ?, ?, ?, ?, ?)`
+).bind(entry.instanceId, entry.stepName, entry.message, entry.timestamp, entry.type, entry.logOrder, entry.tenantId));
+			await this.options.D1.batch(statements);
+		} catch (error) {
+			console.error("Batch log insert failed, falling back to individual inserts:", error);
+			await Promise.all(logsToFlush.map((entry) => pushLogToDB(this.options, entry)));
+		}
+	}
+	async destroy() {
+		this.isDestroyed = true;
+		await this.flush();
+	}
+};
 function trySerializeObj(obj, serializer) {
 	try {
 		return serializer.serialize(obj);
@@ -401,46 +444,44 @@ const createLogAccessor = (context) => {
 		const selectClause = "SELECT DISTINCT w.*";
 		const fromClause = "FROM WorkflowTable w";
 		const propertyWhereConditions = [];
-		if (filter) {
-			if (filter.properties && Object.keys(filter.properties).length > 0) {
-				let propIndex = 0;
-				for (const [key, propertyFilter] of Object.entries(filter.properties)) {
-					const propAlias = `p${propIndex}`;
-					joinClause += ` LEFT JOIN WorkflowProperties ${propAlias} ON w.instanceId = ${propAlias}.instanceId AND ${propAlias}.key = ? AND ${propAlias}.tenantId = ?`;
-					bindings.push(key, context.tenantId);
-					if (propertyFilter.equals !== void 0) propertyWhereConditions.push({
-						condition: `${propAlias}.value = ?`,
-						binding: String(propertyFilter.equals)
-					});
-					if (propertyFilter.contains !== void 0) propertyWhereConditions.push({
-						condition: `${propAlias}.value LIKE ?`,
-						binding: `%${propertyFilter.contains}%`
+		if (filter && filter.properties && Object.keys(filter.properties).length > 0) {
+			let propIndex = 0;
+			for (const [key, propertyFilter] of Object.entries(filter.properties)) {
+				const propAlias = `p${propIndex}`;
+				joinClause += ` LEFT JOIN WorkflowProperties ${propAlias} ON w.instanceId = ${propAlias}.instanceId AND ${propAlias}.key = ? AND ${propAlias}.tenantId = ?`;
+				bindings.push(key, context.tenantId);
+				if (propertyFilter.equals !== void 0) propertyWhereConditions.push({
+					condition: `${propAlias}.value = ?`,
+					binding: String(propertyFilter.equals)
+				});
+				if (propertyFilter.contains !== void 0) propertyWhereConditions.push({
+					condition: `${propAlias}.value LIKE ?`,
+					binding: `%${propertyFilter.contains}%`
+				});
+				if (propertyFilter.gt !== void 0) propertyWhereConditions.push({
+					condition: `CAST(${propAlias}.value AS REAL) > ?`,
+					binding: propertyFilter.gt
+				});
+				if (propertyFilter.gte !== void 0) propertyWhereConditions.push({
+					condition: `CAST(${propAlias}.value AS REAL) >= ?`,
+					binding: propertyFilter.gte
+				});
+				if (propertyFilter.lt !== void 0) propertyWhereConditions.push({
+					condition: `CAST(${propAlias}.value AS REAL) < ?`,
+					binding: propertyFilter.lt
+				});
+				if (propertyFilter.lte !== void 0) propertyWhereConditions.push({
+					condition: `CAST(${propAlias}.value AS REAL) <= ?`,
+					binding: propertyFilter.lte
+				});
+				if (propertyFilter.in !== void 0 && Array.isArray(propertyFilter.in)) {
+					const placeholders = propertyFilter.in.map(() => "?").join(", ");
+					propertyWhereConditions.push({
+						condition: `${propAlias}.value IN (${placeholders})`,
+						bindings: propertyFilter.in.map(String)
 					});
-					if (propertyFilter.gt !== void 0) propertyWhereConditions.push({
-						condition: `CAST(${propAlias}.value AS REAL) > ?`,
-						binding: propertyFilter.gt
-					});
-					if (propertyFilter.gte !== void 0) propertyWhereConditions.push({
-						condition: `CAST(${propAlias}.value AS REAL) >= ?`,
-						binding: propertyFilter.gte
-					});
-					if (propertyFilter.lt !== void 0) propertyWhereConditions.push({
-						condition: `CAST(${propAlias}.value AS REAL) < ?`,
-						binding: propertyFilter.lt
-					});
-					if (propertyFilter.lte !== void 0) propertyWhereConditions.push({
-						condition: `CAST(${propAlias}.value AS REAL) <= ?`,
-						binding: propertyFilter.lte
-					});
-					if (propertyFilter.in !== void 0 && Array.isArray(propertyFilter.in)) {
-						const placeholders = propertyFilter.in.map(() => "?").join(", ");
-						propertyWhereConditions.push({
-							condition: `${propAlias}.value IN (${placeholders})`,
-							bindings: propertyFilter.in.map(String)
-						});
-					}
-					propIndex++;
 				}
+				propIndex++;
 			}
 		}
 		propertyWhereConditions.forEach((item) => {
@@ -533,6 +574,7 @@ const createLogAccessor = (context) => {
 		let limit;
 		let actualOffset;
 		let actualInstanceId;
+		let populateData = void 0;
 		if (typeof limitOrOptions === "number") {
 			limit = limitOrOptions;
 			actualOffset = offset;
@@ -542,7 +584,9 @@ const createLogAccessor = (context) => {
 			limit = options.limit;
 			actualOffset = options.offset;
 			actualInstanceId = options.instanceId;
+			populateData = options.populateData;
 		}
+		if (populateData === void 0) populateData = false;
 		let result;
 		if (limit !== void 0 && actualOffset !== void 0) result = await context.D1.prepare(
 			/* sql */
@@ -558,8 +602,12 @@ const createLogAccessor = (context) => {
 ).bind(context.tenantId, actualInstanceId ?? null, actualInstanceId ?? null).all();
 		if (result.results) {
 			const steps = await Promise.all(result.results.map(async (row) => {
-				const result$1 = await deserializeWithExternalStorage(row.result, row.resultRef, internalSerializer, context.externalBlobStorage);
-				const error = await deserializeWithExternalStorage(row.error, row.errorRef, internalSerializer, context.externalBlobStorage);
+				let deserializedResult = null;
+				let deserializedError = null;
+				if (populateData) {
+					deserializedResult = await deserializeWithExternalStorage(row.result, row.resultRef, internalSerializer, context.externalBlobStorage);
+					deserializedError = await deserializeWithExternalStorage(row.error, row.errorRef, internalSerializer, context.externalBlobStorage);
+				}
 				return {
 					instanceId: row.instanceId,
 					name: row.stepName,
@@ -567,8 +615,8 @@ const createLogAccessor = (context) => {
 					status: row.stepStatus,
 					startTime: row.startTime,
 					endTime: row.endTime,
-					result: result$1,
-					error
+					result: deserializedResult,
+					error: deserializedError
 				};
 			}));
 			return steps;
@@ -625,10 +673,13 @@ const createLogAccessor = (context) => {
 		const workflow = await workflowTableRowToWorkflowRun(result, internalSerializer, context.externalBlobStorage);
 		return workflow;
 	};
-	const getWorkflow = async (instanceId) => {
+	const getWorkflow = async (instanceId, populateData = false) => {
 		const workflow = await getWorkflowShallow(instanceId);
 		if (!workflow) return null;
-		const steps = await listSteps({ instanceId });
+		const steps = await listSteps({
+			instanceId,
+			populateData
+		});
 		const retryWorkflows = await getWorkflowByParentId(instanceId);
 		const parentWorkflow = workflow.parentInstanceId ? await getWorkflowShallow(workflow.parentInstanceId) : null;
 		workflow.retries = retryWorkflows;
@@ -768,13 +819,10 @@ async function createStepContext(context) {
 		const stepNameParam = typeof step$1 === "string" ? step$1 : step$1.name;
 		const stepMetadataParam = typeof step$1 === "string" ? void 0 : step$1.metadata;
 		if (context.parentInstanceId && reuseSuccessfulSteps) {
-			console.warn("temp: try to reuse successful steps");
 			const existingStep = await getStepRecord(context, stepNameParam, context.parentInstanceId);
 			if (existingStep) {
-				console.warn("temp: found existing step", existingStep);
 				const row = existingStep;
 				if (row.status === "completed") {
-					console.warn("temp: found existing step with completed status", row);
 					const { data: resultData, externalRef: resultRef } = await serializeWithExternalStorage(row.result, context.serializer, context.externalBlobStorage);
 					const { data: errorData, externalRef: errorRef } = await serializeWithExternalStorage(row.error, context.serializer, context.externalBlobStorage);
 					await insertStepRecordFull(context, {
@@ -790,10 +838,10 @@ async function createStepContext(context) {
 						errorRef
 					});
 					return row.result;
-				} else console.warn("temp: found existing step with different status", row);
-			} else console.warn("temp: no existing step found");
-		} else console.warn("temp: not trying to reuse successful steps");
-		const waitFor = [];
+				}
+			}
+		}
+		const logBatcher = new LogBatcher(context);
 		const startTime = Date.now();
 		const stepStatus = "pending";
 		const stepMetadata = context.serializer.serialize(stepMetadataParam || {});
@@ -814,7 +862,7 @@ async function createStepContext(context) {
 		const log = (message, type = "info") => {
 			logOrder++;
 			const timestamp = Date.now();
-			const logPromise = pushLogToDB(context, {
+			logBatcher.addLog({
 				stepName: stepNameParam,
 				instanceId,
 				message,
@@ -823,7 +871,6 @@ async function createStepContext(context) {
 				logOrder,
 				tenantId: context.tenantId
 			});
-			waitFor.push(logPromise);
 		};
 		const ctx = { console: {
 			log: (message, ...optionalParams) => {
@@ -851,7 +898,7 @@ async function createStepContext(context) {
           SET stepStatus = ?, endTime = ?, result = ?, error = ?, resultRef = ?, errorRef = ?
           WHERE instanceId = ? AND stepName = ?`
 ).bind(stepStatus$1, endTime, resultData, stepError$1, resultRef, null, instanceId, stepName).run();
-			await Promise.allSettled(waitFor);
+			await logBatcher.destroy();
 			return result;
 		} catch (error) {
 			const endTime = Date.now();
@@ -864,7 +911,7 @@ async function createStepContext(context) {
           SET stepStatus = ?, endTime = ?, result = ?, error = ?, resultRef = ?, errorRef = ?
           WHERE instanceId = ? AND stepName = ?`
 ).bind(stepStatus$1, endTime, stepResult$1, errorData, null, errorRef, instanceId, stepName).run();
-			await Promise.allSettled(waitFor);
+			await logBatcher.destroy();
 			throw error;
 		}
 	};
@@ -899,12 +946,12 @@ function createWorkflowContext(options) {
 			parentInstanceId,
 			tenantId
 		});
-		const waitFor = [];
+		const logBatcher = new LogBatcher(internalContext);
 		let logOrder = 0;
 		const log = (message, type = "info") => {
 			logOrder++;
 			const timestamp = Date.now();
-			const logPromise = pushLogToDB(internalContext, {
+			logBatcher.addLog({
 				instanceId,
 				stepName: null,
 				message,
@@ -913,7 +960,6 @@ function createWorkflowContext(options) {
 				logOrder,
 				tenantId
 			});
-			waitFor.push(logPromise);
 		};
 		const stepContext = await createStepContext({
 			D1: options.D1,
@@ -963,7 +1009,7 @@ function createWorkflowContext(options) {
 				endTime,
 				instanceId
 			});
-			await Promise.allSettled(waitFor);
+			await logBatcher.destroy();
 		} catch (error) {
 			const endTime = Date.now();
 			const workflowStatus = "failed";
@@ -972,7 +1018,7 @@ function createWorkflowContext(options) {
 				endTime,
 				instanceId
 			});
-			await Promise.allSettled(waitFor);
+			await logBatcher.destroy();
 			throw error;
 		}
 	};
@@ -983,14 +1029,15 @@ function createWorkflowContext(options) {
 		}
 		const oldRun = await options.D1.prepare(
 			/* sql */
-			`SELECT input, workflowName, tenantId FROM WorkflowTable WHERE instanceId = ? `
+			`SELECT input, workflowName, tenantId, inputRef FROM WorkflowTable WHERE instanceId = ? `
 ).bind(retryInstanceId).first();
 		const oldWorkflowName = oldRun?.workflowName;
 		const tenantId = oldRun?.tenantId;
 		if (!tenantId) throw new Error(`No tenantId found for instanceId ${retryInstanceId}`);
+		const inputRef = oldRun?.inputRef;
 		const encodedInput = oldRun?.input;
-		if (!encodedInput) throw new Error(`No input found for instanceId ${retryInstanceId}`);
-		const input = internalContext.serializer.deserialize(encodedInput);
+		if (inputRef === void 0 || encodedInput === void 0) throw new Error(`No input found for instanceId ${retryInstanceId}`);
+		const input = await deserializeWithExternalStorage(encodedInput, inputRef, internalContext.serializer, options.externalBlobStorage);
 		await call({
 			workflow,
 			input,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brandboostinggmbh/observable-workflows",
-  "version": "0.8.5",
+  "version": "0.9.0",
   "description": "My awesome typescript library",
   "type": "module",
   "license": "MIT",