@brandboostinggmbh/observable-workflows 0.14.0 → 0.15.0

package/dist/index.d.ts

@@ -351,6 +351,7 @@ type ExternalBlobStorage = {
   threshold: number;
   set: (data: string) => Promise<string>;
   get: (id: string) => Promise<string>;
+  delete: (...keys: string[]) => Promise<number>;
 };
 type WorkflowContextOptions = {
   D1: D1Database;
@@ -545,6 +546,7 @@ type DeleteConfig = {
     /** Workflows that are still in progress after a long time, might have failed non-gracefully and are therefore not correctly marked as failed.*/
     inProgressWorkflows: boolean;
   };
+  deleteRefsFromExternalStorage: boolean;
 };
 declare const createCleanupManager: (context: {
   D1: D1Database;
@@ -552,11 +554,29 @@ declare const createCleanupManager: (context: {
   serializer?: Serializer;
   externalBlobStorage?: ExternalBlobStorage;
 }) => {
-  countAffectedWorkflows: (config: DeleteConfig) => Promise<{
+  countAffectedWorkflows: (config: DeleteConfig, limit: number) => Promise<{
     count: number;
+    affectedSteps: {
+      instanceId: string;
+      stepName: string;
+      stepStatus: string;
+      errorRef: string | null;
+      resultRef: string | null;
+      workflowName: string;
+    }[];
+    externalStorageKeysCount: number;
   }>;
-  deleteOldWorkflows: (config: DeleteConfig) => Promise<{
+  deleteOldWorkflows: (config: DeleteConfig, limit: number) => Promise<{
     deletedCount: number;
+    deletedSteps: {
+      instanceId: string;
+      stepName: string;
+      stepStatus: string;
+      errorRef: string | null;
+      resultRef: string | null;
+      workflowName: string;
+    }[];
+    deletedExternalStorageKeysCount: number;
   }>;
 };
 

package/dist/index.js

@@ -1307,6 +1307,23 @@ function createR2ExternalBlobStorage(options) {
 			} catch (error) {
 				throw new Error(`Failed to retrieve data from R2: ${error instanceof Error ? error.message : String(error)}`);
 			}
+		},
+		async delete(...keys) {
+			if (keys.length === 0) return 0;
+			try {
+				await bucket.delete(keys);
+				return keys.length;
+			} catch (error) {
+				console.warn(`Bulk delete failed, falling back to individual deletes: ${error instanceof Error ? error.message : String(error)}`);
+				let deletedCount = 0;
+				for (const key of keys) try {
+					await bucket.delete(key);
+					deletedCount++;
+				} catch (individualError) {
+					console.warn(`Failed to delete key ${key} from R2: ${individualError instanceof Error ? individualError.message : String(individualError)}`);
+				}
+				return deletedCount;
+			}
 		}
 	};
 }
@@ -1314,42 +1331,114 @@ function createR2ExternalBlobStorage(options) {
 //#endregion
 //#region src/observableWorkflows/createCleanupManager.ts
 const createCleanupManager = (context) => {
-	const countAffectedWorkflows = async (config) => {
+	const buildStatusesAndPlaceholders = (config) => {
 		const statuses = [];
 		if (config.delete.successfulWorkflows) statuses.push("completed");
 		if (config.delete.failedWorkflows) statuses.push("failed");
 		if (config.delete.inProgressWorkflows) statuses.push("pending");
-		if (statuses.length === 0) return { count: 0 };
 		const statusPlaceholders = statuses.map(() => "?").join(", ");
+		return {
+			statuses,
+			statusPlaceholders
+		};
+	};
+	const getAffectedWorkflows = async (config, limit) => {
+		const { statuses, statusPlaceholders } = buildStatusesAndPlaceholders(config);
+		if (statuses.length === 0) return [];
 		const result = await context.D1.prepare(
 			/* sql */
 			`
-      SELECT COUNT(*) as count 
-      FROM WorkflowTable 
+      SELECT instanceId, workflowName, inputRef
+      FROM WorkflowTable
       WHERE tenantId = ? 
         AND startTime < (strftime('%s', 'now', '-' || ? || ' days') * 1000)
         AND workflowStatus IN (${statusPlaceholders})
+      ORDER BY workflowName, instanceId
+      LIMIT ?
     `
-).bind(context.tenantId, config.deleteWorkflowsOlderThanDays, ...statuses).first();
-		return result;
+).bind(context.tenantId, config.deleteWorkflowsOlderThanDays, ...statuses, limit).all();
+		return result.results;
 	};
-	const deleteOldWorkflows = async (config) => {
-		const statuses = [];
-		if (config.delete.successfulWorkflows) statuses.push("completed");
-		if (config.delete.failedWorkflows) statuses.push("failed");
-		if (config.delete.inProgressWorkflows) statuses.push("pending");
-		if (statuses.length === 0) return { deletedCount: 0 };
-		const statusPlaceholders = statuses.map(() => "?").join(", ");
+	const getAffectedSteps = async (workflows) => {
+		if (workflows.length === 0) return [];
+		const workflowIds = workflows.map(() => "?").join(", ");
 		const result = await context.D1.prepare(
 			/* sql */
 			`
-      DELETE FROM WorkflowTable 
-      WHERE tenantId = ? 
-        AND startTime < (strftime('%s', 'now', '-' || ? || ' days') * 1000)
-        AND workflowStatus IN (${statusPlaceholders})
+      SELECT s.instanceId, s.stepName, s.stepStatus, s.errorRef, s.resultRef, w.workflowName
+      FROM StepTable s
+      JOIN WorkflowTable w ON s.instanceId = w.instanceId
+      WHERE s.instanceId IN (${workflowIds})
+      ORDER BY w.workflowName, s.instanceId, s.stepName
     `
-).bind(context.tenantId, config.deleteWorkflowsOlderThanDays, ...statuses).run();
-		return { deletedCount: result.meta.changes || 0 };
+).bind(...workflows.map((w) => w.instanceId)).all();
+		return result.results;
+	};
+	const collectExternalStorageKeys = (workflows, steps) => {
+		const keys = [];
+		for (const workflow of workflows) if (workflow.inputRef) keys.push(workflow.inputRef);
+		for (const step of steps) {
+			if (step.errorRef) keys.push(step.errorRef);
+			if (step.resultRef) keys.push(step.resultRef);
+		}
+		return keys;
+	};
+	const countAffectedWorkflows = async (config, limit) => {
+		if (buildStatusesAndPlaceholders(config).statuses.length === 0) return {
+			count: 0,
+			affectedSteps: [],
+			externalStorageKeysCount: 0
+		};
+		const affectedWorkflows = await getAffectedWorkflows(config, limit);
+		const affectedSteps = await getAffectedSteps(affectedWorkflows);
+		let externalStorageKeysCount = 0;
+		if (config.deleteRefsFromExternalStorage && context.externalBlobStorage) {
+			const externalKeys = collectExternalStorageKeys(affectedWorkflows, affectedSteps);
+			externalStorageKeysCount = externalKeys.length;
+		}
+		return {
+			count: affectedWorkflows.length,
+			affectedSteps,
+			externalStorageKeysCount
+		};
+	};
+	const deleteOldWorkflows = async (config, limit) => {
+		if (buildStatusesAndPlaceholders(config).statuses.length === 0) return {
+			deletedCount: 0,
+			deletedSteps: [],
+			deletedExternalStorageKeysCount: 0
+		};
+		const deletedWorkflows = await getAffectedWorkflows(config, limit);
+		const deletedSteps = await getAffectedSteps(deletedWorkflows);
+		let deletedExternalStorageKeysCount = 0;
+		if (config.deleteRefsFromExternalStorage && context.externalBlobStorage) {
+			const externalKeys = collectExternalStorageKeys(deletedWorkflows, deletedSteps);
+			if (externalKeys.length > 0) try {
+				deletedExternalStorageKeysCount = await context.externalBlobStorage.delete(...externalKeys);
+			} catch (error) {
+				console.warn(`Failed to delete some external storage keys: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		if (deletedWorkflows.length > 0) {
+			const workflowIds = deletedWorkflows.map(() => "?").join(", ");
+			const result = await context.D1.prepare(
+				/* sql */
+				`
+        DELETE FROM WorkflowTable 
+        WHERE instanceId IN (${workflowIds})
+      `
+).bind(...deletedWorkflows.map((w) => w.instanceId)).run();
+			return {
+				deletedCount: result.meta.changes || 0,
+				deletedSteps,
+				deletedExternalStorageKeysCount
+			};
+		}
+		return {
+			deletedCount: 0,
+			deletedSteps,
+			deletedExternalStorageKeysCount
+		};
 	};
 	return {
 		countAffectedWorkflows,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brandboostinggmbh/observable-workflows",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "My awesome typescript library",
   "type": "module",
   "license": "MIT",