@brandboostinggmbh/observable-workflows 0.20.0-beta.5 → 0.20.0-beta.6

package/dist/index.d.ts

@@ -62,43 +62,8 @@ declare function finalizeWorkflowRecord(options: InternalWorkflowContextOptions,
   instanceId: string;
   result?: any;
 }): Promise<D1Result<Record<string, unknown>>>;
-/**
- * Insert a new workflow record into the database.
- * Optionally creates workflow dependencies atomically in the same transaction.
- *
- * @example
- * ```typescript
- * // Create a workflow with dependencies
- * await insertWorkflowRecord(context, {
- *   instanceId: 'workflow-c',
- *   workflowType: 'process-data',
- *   workflowName: 'Data Processing',
- *   workflowMetadata: {},
- *   input: { data: 'value' },
- *   workflowStatus: 'pending',
- *   startTime: Date.now(),
- *   tenantId: 'tenant-1',
- *   dependencies: [
- *     { instanceId: 'workflow-a' },
- *     { instanceId: 'workflow-b' }
- *   ] // This workflow depends on A and B
- * })
- * ```
- */
-declare function insertWorkflowRecord(options: InternalWorkflowContextOptions, {
-  instanceId,
-  workflowType,
-  workflowName,
-  workflowMetadata,
-  input,
-  workflowStatus,
-  startTime,
-  endTime,
-  parentInstanceId,
-  tenantId,
-  triggerId,
-  dependencies
-}: {
+/** Parameters for preparing workflow insert statements */
+interface PrepareWorkflowInsertParams {
   /** Unique identifier for this workflow instance */
   instanceId: string;
   /** Type/category of the workflow */
@@ -123,11 +88,36 @@ declare function insertWorkflowRecord(options: InternalWorkflowContextOptions, {
   triggerId?: string | null;
   /**
    * Optional array of workflow dependencies that this workflow depends on.
-   * When provided, dependency relationships are created atomically with the workflow insert.
-   * This ensures that the workflow and all its dependencies are created in a single transaction.
+   * When provided, dependency insert statements are included in the returned statements array.
    */
   dependencies?: Array<WorkflowDependency>;
-}): Promise<{
+}
+/** Result from preparing workflow insert statements */
+
+/**
+ * Insert a new workflow record into the database.
+ * Optionally creates workflow dependencies atomically in the same transaction.
+ *
+ * @example
+ * ```typescript
+ * // Create a workflow with dependencies
+ * await insertWorkflowRecord(context, {
+ *   instanceId: 'workflow-c',
+ *   workflowType: 'process-data',
+ *   workflowName: 'Data Processing',
+ *   workflowMetadata: {},
+ *   input: { data: 'value' },
+ *   workflowStatus: 'pending',
+ *   startTime: Date.now(),
+ *   tenantId: 'tenant-1',
+ *   dependencies: [
+ *     { instanceId: 'workflow-a' },
+ *     { instanceId: 'workflow-b' }
+ *   ] // This workflow depends on A and B
+ * })
+ * ```
+ */
+declare function insertWorkflowRecord(options: InternalWorkflowContextOptions, params: PrepareWorkflowInsertParams): Promise<{
   success: boolean;
   meta: any;
 }>;

package/dist/index.js

@@ -444,6 +444,55 @@ async function finalizeWorkflowRecord(options, { workflowStatus, endTime, instan
             WHERE instanceId = ?`).bind(workflowStatus, endTime, instanceId).run(), options.retryConfig);
 }
 /**
+* Prepare D1 statements for inserting a workflow record and its dependencies.
+* This function handles input serialization (including external blob storage for large inputs)
+* and returns prepared statements that can be executed individually or batched with other statements.
+*
+* This is an internal helper used by `insertWorkflowRecord` and `enqueueWorkflowBatch` to enable
+* efficient batching of multiple workflow inserts into a single D1 batch operation.
+*
+* @param options - The internal workflow context options containing D1, serializer, etc.
+* @param params - The workflow record parameters
+* @returns Object containing prepared statements, instanceId, and workflowType
+*
+* @example
+* ```typescript
+* // Prepare statements for multiple workflows, then batch execute
+* const prepared = await Promise.all([
+*   prepareWorkflowInsertStatements(options, workflow1Params),
+*   prepareWorkflowInsertStatements(options, workflow2Params),
+* ])
+* const allStatements = prepared.flatMap(p => p.statements)
+* await options.D1.batch(allStatements)
+* ```
+*
+* @internal
+*/
+async function prepareWorkflowInsertStatements(options, { instanceId, workflowType, workflowName, workflowMetadata, input, workflowStatus, startTime, endTime, parentInstanceId, tenantId, triggerId, dependencies }) {
+	const { data: inputData, externalRef: inputRef } = await serializeWithExternalStorage(input, options.serializer, options.externalBlobStorage);
+	const insertWorkflowStatement = options.D1.prepare(`INSERT INTO WorkflowTable
+        (instanceId, workflowType, workflowName, workflowMetadata, input, inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId, triggerId)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).bind(instanceId, workflowType, workflowName, options.serializer.serialize(workflowMetadata), inputData, inputRef, tenantId, workflowStatus, startTime, endTime ?? null, parentInstanceId ?? null, triggerId ?? null);
+	if (!dependencies || dependencies.length === 0) return {
+		statements: [insertWorkflowStatement],
+		instanceId,
+		workflowType
+	};
+	const createdAt = Date.now();
+	const dependencyStatements = dependencies.map((dep) => prepareWorkflowDependencyStatement({
+		D1: options.D1,
+		dependencyWorkflowId: dep.instanceId,
+		dependentWorkflowId: instanceId,
+		tenantId,
+		createdAt
+	}));
+	return {
+		statements: [insertWorkflowStatement, ...dependencyStatements],
+		instanceId,
+		workflowType
+	};
+}
+/**
 * Insert a new workflow record into the database.
 * Optionally creates workflow dependencies atomically in the same transaction.
 *
@@ -466,27 +515,16 @@ async function finalizeWorkflowRecord(options, { workflowStatus, endTime, instan
 * })
 * ```
 */
-async function insertWorkflowRecord(options, { instanceId, workflowType, workflowName, workflowMetadata, input, workflowStatus, startTime, endTime, parentInstanceId, tenantId, triggerId, dependencies }) {
-	const { data: inputData, externalRef: inputRef } = await serializeWithExternalStorage(input, options.serializer, options.externalBlobStorage);
-	const insertWorkflowStatement = options.D1.prepare(`INSERT INTO WorkflowTable
-        (instanceId, workflowType, workflowName, workflowMetadata, input, inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId, triggerId)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).bind(instanceId, workflowType, workflowName, options.serializer.serialize(workflowMetadata), inputData, inputRef, tenantId, workflowStatus, startTime, endTime ?? null, parentInstanceId ?? null, triggerId ?? null);
-	if (!dependencies || dependencies.length === 0) {
-		const result = await retryD1Operation(() => insertWorkflowStatement.run(), options.retryConfig);
+async function insertWorkflowRecord(options, params) {
+	const { statements } = await prepareWorkflowInsertStatements(options, params);
+	if (statements.length === 1) {
+		const result = await retryD1Operation(() => statements[0].run(), options.retryConfig);
 		return {
 			success: result.success,
 			meta: result.meta
 		};
 	}
-	const createdAt = Date.now();
-	const dependencyStatements = dependencies.map((dep) => prepareWorkflowDependencyStatement({
-		D1: options.D1,
-		dependencyWorkflowId: dep.instanceId,
-		dependentWorkflowId: instanceId,
-		tenantId,
-		createdAt
-	}));
-	const results = await retryD1Operation(() => options.D1.batch([insertWorkflowStatement, ...dependencyStatements]), options.retryConfig);
+	const results = await retryD1Operation(() => options.D1.batch(statements), options.retryConfig);
 	const allSucceeded = results.every((result) => result.success);
 	return {
 		success: allSucceeded,
@@ -1784,68 +1822,79 @@ function createQueueWorkflowContext(options) {
 			queueIdentifier: options.queueIdentifier
 		});
 	};
+	/**
+	* Enqueue multiple workflows in a single batch operation.
+	*
+	* This method is optimized for efficiency: it prepares all workflow insert statements
+	* in parallel (which may involve external blob storage for large inputs), then executes
+	* them in a single D1 batch operation, followed by a single queue batch send.
+	*
+	* **Network requests:** 2 + M, where M is the number of workflows with large inputs
+	* that require external blob storage.
+	*
+	* **D1 batch limit:** Cloudflare D1 has a limit of ~100 statements per batch.
+	* If you have many workflows with dependencies, this limit may be exceeded.
+	* For very large batches, consider splitting into smaller chunks.
+	*
+	* @param workflows - Array of workflow items to enqueue
+	* @returns Array of scheduled workflow stubs in the same order as inputs
+	*/
 	const enqueueWorkflowBatch = async (workflows) => {
-		const results = await Promise.all(workflows.map(async ({ workflow, tenantId, input, initialName, dependencies }) => {
+		const startTime = Date.now();
+		console.log(`enqueueWorkflowBatch: Starting batch of ${workflows.length} workflows`);
+		const preparedWorkflows = await Promise.all(workflows.map(async ({ workflow, tenantId, input, initialName, dependencies }) => {
 			const instanceId = idFactory();
 			const triggerId = idFactory();
-			const startTime = Date.now();
-			if (dependencies && dependencies.length > 0) {
-				await insertWorkflowRecord(internalContext, {
-					instanceId,
-					workflowType: workflow.workflowType,
-					workflowName: initialName,
-					workflowMetadata: workflow.metadata,
-					input,
-					workflowStatus: "waiting",
-					startTime,
-					endTime: null,
-					parentInstanceId: void 0,
-					tenantId,
-					triggerId,
-					dependencies
-				});
-				return {
-					instanceId,
-					workflowType: workflow.workflowType,
-					shouldQueue: false,
-					queueMessage: null
-				};
-			} else {
-				await insertWorkflowRecord(internalContext, {
-					instanceId,
-					workflowType: workflow.workflowType,
-					workflowName: initialName,
-					workflowMetadata: workflow.metadata,
-					input,
-					workflowStatus: "scheduled",
-					startTime,
-					endTime: null,
-					parentInstanceId: void 0,
-					tenantId,
-					triggerId
-				});
-				return {
-					instanceId,
-					workflowType: workflow.workflowType,
-					shouldQueue: true,
-					queueMessage: {
-						type: "workflow-run",
-						workflowType: workflow.workflowType,
-						workflowName: initialName,
-						input,
-						tenantId,
-						triggerId,
-						queueIdentifier: options.queueIdentifier,
-						scheduledInstanceId: instanceId
-					}
-				};
-			}
+			const hasDependencies = dependencies && dependencies.length > 0;
+			const prepared = await prepareWorkflowInsertStatements(internalContext, {
+				instanceId,
+				workflowType: workflow.workflowType,
+				workflowName: initialName,
+				workflowMetadata: workflow.metadata,
+				input,
+				workflowStatus: hasDependencies ? "waiting" : "scheduled",
+				startTime,
+				endTime: null,
+				parentInstanceId: void 0,
+				tenantId,
+				triggerId,
+				dependencies
+			});
+			return {
+				...prepared,
+				tenantId,
+				triggerId,
+				initialName,
+				shouldQueue: !hasDependencies
+			};
 		}));
-		const messagesToQueue = results.filter((r) => r.shouldQueue && r.queueMessage).map((r) => ({ body: r.queueMessage }));
-		if (messagesToQueue.length > 0) await options.QUEUE.sendBatch(messagesToQueue);
-		return results.map((r) => ({
-			instanceId: r.instanceId,
-			workflowType: r.workflowType
+		const workflowsWithDeps = preparedWorkflows.filter((p) => !p.shouldQueue).length;
+		const workflowsWithoutDeps = preparedWorkflows.filter((p) => p.shouldQueue).length;
+		console.log(`enqueueWorkflowBatch: Prepared ${preparedWorkflows.length} workflows (${workflowsWithoutDeps} to queue, ${workflowsWithDeps} waiting on dependencies)`);
+		const allStatements = preparedWorkflows.flatMap((p) => p.statements);
+		if (allStatements.length > 0) {
+			console.log(`enqueueWorkflowBatch: Executing D1 batch with ${allStatements.length} statements`);
+			await retryD1Operation(() => options.D1.batch(allStatements), options.retryConfig);
+		}
+		const messagesToQueue = preparedWorkflows.filter((p) => p.shouldQueue).map((p) => ({ body: {
+			type: "workflow-run",
+			workflowType: p.workflowType,
+			workflowName: p.initialName,
+			input: workflows.find((w) => w.workflow.workflowType === p.workflowType && w.initialName === p.initialName)?.input,
+			tenantId: p.tenantId,
+			triggerId: p.triggerId,
+			queueIdentifier: options.queueIdentifier,
+			scheduledInstanceId: p.instanceId
+		} }));
+		if (messagesToQueue.length > 0) {
+			console.log(`enqueueWorkflowBatch: Sending ${messagesToQueue.length} messages to queue`);
+			await options.QUEUE.sendBatch(messagesToQueue);
+		}
+		const elapsed = Date.now() - startTime;
+		console.log(`enqueueWorkflowBatch: Completed in ${elapsed}ms`);
+		return preparedWorkflows.map((p) => ({
+			instanceId: p.instanceId,
+			workflowType: p.workflowType
 		}));
 	};
 	const handleWorkflowQueueMessage = async ({ message, env, workflowResolver }) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brandboostinggmbh/observable-workflows",
-  "version": "0.20.0-beta.5",
+  "version": "0.20.0-beta.6",
   "description": "My awesome typescript library",
   "type": "module",
   "license": "MIT",