@brandboostinggmbh/observable-workflows 0.10.0 → 0.11.0

package/dist/index.d.ts

@@ -37,7 +37,8 @@ declare function insertWorkflowRecord(options: InternalWorkflowContextOptions, {
   startTime,
   endTime,
   parentInstanceId,
-  tenantId
+  tenantId,
+  triggerId
 }: {
   instanceId: string;
   workflowType: string;
@@ -49,6 +50,7 @@ declare function insertWorkflowRecord(options: InternalWorkflowContextOptions, {
   endTime?: number | null;
   parentInstanceId?: string | null;
   tenantId: string;
+  triggerId?: string | null;
 }): Promise<D1Result<Record<string, unknown>>>;
 declare function insertStepRecordFull(context: StepContextOptions, {
   instanceId,
@@ -133,6 +135,7 @@ declare function workflowTableRowToWorkflowRun(row: {
   startTime: number;
   endTime: number | null;
   parentInstanceId: string | null;
+  triggerId: string | null;
 }, serializer: Serializer, externalBlobStorage?: ExternalBlobStorage): Promise<WorkflowRun>;
 declare function updateWorkflowName(context: {
   D1: D1Database;
@@ -213,6 +216,7 @@ type WorkflowRun = {
   startTime: number;
   endTime: number | null;
   parentInstanceId: string | null;
+  triggerId: string | null;
   steps?: Step[];
   /** All Logs associated with the workflow. Includes logs that are part of steps */
   logs?: Log[];
@@ -346,13 +350,7 @@ type WorkflowContextOptions = {
 };
 type InternalWorkflowContextOptions = WorkflowContextOptions & Required<Pick<WorkflowContextOptions, 'serializer' | 'idFactory'>>;
 type WorkflowContextInstance = {
-  call: <I>(args: {
-    workflow: WorkflowFunction<I>;
-    input: I;
-    workflowName: string;
-    tenantId: string;
-    parentInstanceId?: string | undefined;
-  }) => Promise<void>;
+  call: <I>(args: WorkflowCallParams<I>) => Promise<void>;
   retry: <I>(workflow: WorkflowFunction<I>, retryInstanceId: string, retryOptions?: RetryWorkflowOptions | undefined) => Promise<void>;
 };
 type QueueWorkflowContextOptions = {
@@ -391,6 +389,17 @@ type RetryWorkflowOptions = {
   /** If true the retry will attept to reuse all results from successful steps. Defaults to True */
   reuseSuccessfulSteps?: boolean;
 };
+type WorkflowCallParams<I> = {
+  workflow: WorkflowFunction<I>;
+  input: I;
+  workflowName: string;
+  tenantId: string;
+  parentInstanceId?: string | undefined;
+  /** If true and a parentInstanceId is provided, this run will attept to reuse all results from successful steps. Defaults to true  */
+  reuseSuccessfulSteps?: boolean;
+  /** Optional trigger identifier for workflow correlation */
+  triggerId?: string | null;
+};
 
 //#endregion
 //#region src/observableWorkflows/createLogAccessor.d.ts
@@ -413,6 +422,7 @@ declare const createLogAccessor: (context: {
   listWorkflows: (limit: number, offset: number, filter?: WorkflowFilter) => Promise<WorkflowRun[]>;
   getWorkflow: (instanceId: string, populateData?: boolean) => Promise<WorkflowRun | null>;
   getWorkflowTypesByTenantId: (tenantId: string) => Promise<string[]>;
+  getWorkflowByTriggerId: (triggerId: string) => Promise<WorkflowRun | null>;
   getPropertiesKeys: (instanceId?: string) => Promise<WorkflowPropertyDefinition[]>;
 };
 

package/dist/index.js

@@ -7,8 +7,10 @@ async function detectSchemaVersion(db) {
 	let workflowTable = "missing";
 	if (workflowTableInfo) {
 		const hasInputRef = workflowTableInfo.sql.includes("inputRef");
+		const hasTriggerId = workflowTableInfo.sql.includes("triggerId");
 		const inputHasNotNull = workflowTableInfo.sql.includes("input TEXT NOT NULL");
-		if (hasInputRef && !inputHasNotNull) workflowTable = "v2+";
+		if (hasTriggerId && hasInputRef && !inputHasNotNull) workflowTable = "v4";
+		else if (hasInputRef && !inputHasNotNull && !hasTriggerId) workflowTable = "v2";
 		else if (!hasInputRef && inputHasNotNull) workflowTable = "v1";
 		else workflowTable = "v1";
 	}
@@ -17,7 +19,7 @@ async function detectSchemaVersion(db) {
 	if (stepTableInfo) {
 		const hasResultRef = stepTableInfo.sql.includes("resultRef");
 		const hasErrorRef = stepTableInfo.sql.includes("errorRef");
-		if (hasResultRef && hasErrorRef) stepTable = "v2+";
+		if (hasResultRef && hasErrorRef) stepTable = "v2";
 		else stepTable = "v1";
 	}
 	const logTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='LogTable'`).first();
@@ -34,6 +36,83 @@ async function detectSchemaVersion(db) {
 	};
 }
 /**
+* Migrate WorkflowTable from V1 to V2 schema
+* Adds inputRef column and makes input nullable
+*/
+async function migrateWorkflowTableV1ToV2(db) {
+	const workflowTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first();
+	const hasInputRef = workflowTableInfo.sql.includes("inputRef");
+	const inputHasNotNull = workflowTableInfo.sql.includes("input TEXT NOT NULL");
+	if (!hasInputRef || inputHasNotNull) await db.batch([
+		db.prepare(
+			/* sql */
+			`CREATE TABLE WorkflowTable_new (
+        instanceId TEXT NOT NULL,
+        workflowType TEXT NOT NULL,
+        workflowName TEXT NOT NULL,
+        workflowMetadata TEXT NOT NULL,
+        input TEXT,
+        inputRef TEXT,
+        tenantId TEXT NOT NULL,
+        workflowStatus TEXT NOT NULL,
+        startTime INTEGER NOT NULL,
+        endTime INTEGER,
+        parentInstanceId TEXT,
+        PRIMARY KEY (instanceId)
+      )`
+),
+		db.prepare(
+			/* sql */
+			`INSERT INTO WorkflowTable_new 
+        SELECT instanceId, workflowType, workflowName, workflowMetadata, 
+               CASE WHEN input = '' THEN NULL ELSE input END as input,
+               NULL as inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId
+        FROM WorkflowTable`
+),
+		db.prepare(`DROP TABLE WorkflowTable`),
+		db.prepare(`ALTER TABLE WorkflowTable_new RENAME TO WorkflowTable`)
+	]);
+}
+/**
+* Migrate WorkflowTable from V2/V3 to V4 schema
+* Adds triggerId column with UNIQUE constraint
+*/
+async function migrateWorkflowTableV2V3ToV4(db) {
+	const workflowTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first();
+	const hasTriggerId = workflowTableInfo.sql.includes("triggerId");
+	if (!hasTriggerId) await db.batch([
+		db.prepare(
+			/* sql */
+			`CREATE TABLE WorkflowTable_new (
+        instanceId TEXT NOT NULL,
+        workflowType TEXT NOT NULL,
+        workflowName TEXT NOT NULL,
+        workflowMetadata TEXT NOT NULL,
+        input TEXT,
+        inputRef TEXT,
+        tenantId TEXT NOT NULL,
+        workflowStatus TEXT NOT NULL,
+        startTime INTEGER NOT NULL,
+        endTime INTEGER,
+        parentInstanceId TEXT,
+        triggerId TEXT,
+        PRIMARY KEY (instanceId),
+        UNIQUE (triggerId)
+      )`
+),
+		db.prepare(
+			/* sql */
+			`INSERT INTO WorkflowTable_new 
+        SELECT instanceId, workflowType, workflowName, workflowMetadata, 
+               input, inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId,
+               NULL as triggerId
+        FROM WorkflowTable`
+),
+		db.prepare(`DROP TABLE WorkflowTable`),
+		db.prepare(`ALTER TABLE WorkflowTable_new RENAME TO WorkflowTable`)
+	]);
+}
+/**
 * Create or migrate WorkflowTable to the latest schema
 */
 async function migrateWorkflowTable(db, currentVersion) {
@@ -52,45 +131,21 @@ async function migrateWorkflowTable(db, currentVersion) {
           startTime INTEGER NOT NULL,
           endTime INTEGER,
           parentInstanceId TEXT,
-          PRIMARY KEY (instanceId)
+          triggerId TEXT,
+          PRIMARY KEY (instanceId),
+          UNIQUE (triggerId)
         )`
 ).run();
 		return;
 	}
 	if (currentVersion === "v1") {
-		const workflowTableInfo = await db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='WorkflowTable'`).first();
-		const hasInputRef = workflowTableInfo.sql.includes("inputRef");
-		if (!hasInputRef) await db.prepare(`ALTER TABLE WorkflowTable ADD COLUMN inputRef TEXT`).run();
-		const inputHasNotNull = workflowTableInfo.sql.includes("input TEXT NOT NULL");
-		if (inputHasNotNull) await db.batch([
-			db.prepare(
-				/* sql */
-				`CREATE TABLE WorkflowTable_new (
-          instanceId TEXT NOT NULL,
-          workflowType TEXT NOT NULL,
-          workflowName TEXT NOT NULL,
-          workflowMetadata TEXT NOT NULL,
-          input TEXT,
-          inputRef TEXT,
-          tenantId TEXT NOT NULL,
-          workflowStatus TEXT NOT NULL,
-          startTime INTEGER NOT NULL,
-          endTime INTEGER,
-          parentInstanceId TEXT,
-          PRIMARY KEY (instanceId)
-        )`
-),
-			db.prepare(
-				/* sql */
-				`INSERT INTO WorkflowTable_new 
-          SELECT instanceId, workflowType, workflowName, workflowMetadata, 
-                 CASE WHEN input = '' THEN NULL ELSE input END as input,
-                 inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId 
-          FROM WorkflowTable`
-),
-			db.prepare(`DROP TABLE WorkflowTable`),
-			db.prepare(`ALTER TABLE WorkflowTable_new RENAME TO WorkflowTable`)
-		]);
+		await migrateWorkflowTableV1ToV2(db);
+		await migrateWorkflowTableV2V3ToV4(db);
+		return;
+	}
+	if (currentVersion === "v2") {
+		await migrateWorkflowTableV2V3ToV4(db);
+		return;
 	}
 }
 /**
@@ -173,6 +228,7 @@ async function migrateWorkflowPropertiesTable(db, currentVersion) {
 */
 async function createIndexes(db) {
 	await db.prepare(`CREATE INDEX IF NOT EXISTS idx_workflows_parent_instance_id ON WorkflowTable (parentInstanceId)`).run();
+	await db.prepare(`CREATE INDEX IF NOT EXISTS idx_workflows_trigger_id ON WorkflowTable (triggerId)`).run();
 }
 /**
 * Main migration function that ensures all tables exist and are up-to-date.
@@ -200,14 +256,14 @@ function finalizeWorkflowRecord(options, { workflowStatus, endTime, instanceId }
           WHERE instanceId = ?`
 ).bind(workflowStatus, endTime, instanceId).run();
 }
-async function insertWorkflowRecord(options, { instanceId, workflowType, workflowName, workflowMetadata, input, workflowStatus, startTime, endTime, parentInstanceId, tenantId }) {
+async function insertWorkflowRecord(options, { instanceId, workflowType, workflowName, workflowMetadata, input, workflowStatus, startTime, endTime, parentInstanceId, tenantId, triggerId }) {
 	const { data: inputData, externalRef: inputRef } = await serializeWithExternalStorage(input, options.serializer, options.externalBlobStorage);
 	return options.D1.prepare(
 		/* sql */
 		`INSERT INTO WorkflowTable
-        (instanceId, workflowType, workflowName, workflowMetadata, input, inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
-).bind(instanceId, workflowType, workflowName, options.serializer.serialize(workflowMetadata), inputData, inputRef, tenantId, workflowStatus, startTime, endTime ?? null, parentInstanceId ?? null).run();
+        (instanceId, workflowType, workflowName, workflowMetadata, input, inputRef, tenantId, workflowStatus, startTime, endTime, parentInstanceId, triggerId)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+).bind(instanceId, workflowType, workflowName, options.serializer.serialize(workflowMetadata), inputData, inputRef, tenantId, workflowStatus, startTime, endTime ?? null, parentInstanceId ?? null, triggerId ?? null).run();
 }
 function insertStepRecordFull(context, { instanceId, name, status, metadata, startTime, endTime, result, error, resultRef, errorRef }) {
 	return context.D1.prepare(
@@ -239,13 +295,68 @@ async function getStepRecord(context, stepName, instanceId) {
 		};
 	} else return null;
 }
+/**
+* FUTURE ENHANCEMENT: External Blob Storage for Logs
+*
+* Currently, log messages are stored directly in the LogTable's `log` TEXT column.
+* For very large log messages, we may want to implement external blob storage similar
+* to how step results and errors are handled.
+*
+* To implement this, the following changes would be needed:
+*
+* 1. Database Schema Migration:
+*    - Add `logRef` TEXT column to LogTable
+*    - Update migrateLogTable() in migrations.ts to handle schema version upgrade
+*
+* 2. Storage Functions:
+*    - Modify pushLogToDB() to use serializeWithExternalStorage() for log messages
+*    - Update LogBatcher to handle external storage for large logs
+*    - Consider batch operations for external storage efficiency
+*
+* 3. Retrieval Functions:
+*    - Update log retrieval in createLogAccessor.ts to use deserializeWithExternalStorage()
+*    - Handle both direct and external log data when populating Log objects
+*
+* 4. Configuration:
+*    - Consider separate threshold for logs vs step data (logs might need smaller threshold)
+*    - Add optional log-specific external storage configuration
+*
+* 5. Migration Strategy:
+*    - Decide how to handle existing large logs during schema migration
+*    - Consider background migration process for large log datasets
+*/
+const MAX_LOG_SIZE = 1024 * 64;
+function truncateLogMessage(message) {
+	const encoder = new TextEncoder();
+	const messageBytes = encoder.encode(message);
+	if (messageBytes.length <= MAX_LOG_SIZE) return message;
+	const truncateMarker = "\n... [LOG TRUNCATED - Original size: " + messageBytes.length + " bytes] ...";
+	const markerBytes = encoder.encode(truncateMarker);
+	const availableBytes = MAX_LOG_SIZE - markerBytes.length;
+	let truncatedBytes = messageBytes.slice(0, availableBytes);
+	const decoder = new TextDecoder("utf-8", {
+		fatal: false,
+		ignoreBOM: false
+	});
+	let truncatedMessage = decoder.decode(truncatedBytes);
+	if (truncatedMessage.includes("�")) for (let i = availableBytes - 3; i > availableBytes - 10 && i > 0; i--) {
+		truncatedBytes = messageBytes.slice(0, i);
+		const testMessage = decoder.decode(truncatedBytes);
+		if (!testMessage.includes("�")) {
+			truncatedMessage = testMessage;
+			break;
+		}
+	}
+	return truncatedMessage + truncateMarker;
+}
 function pushLogToDB(options, { instanceId, stepName, message, timestamp, type, logOrder, tenantId }) {
+	const truncatedMessage = truncateLogMessage(message);
 	return options.D1.prepare(
 		/* sql */
 		`INSERT INTO LogTable
            (instanceId, stepName, log, timestamp, type, logOrder, tenantId)
            VALUES (?, ?, ?, ?, ?, ?, ?)`
-).bind(instanceId, stepName, message, timestamp, type, logOrder, tenantId).run();
+).bind(instanceId, stepName, truncatedMessage, timestamp, type, logOrder, tenantId).run();
 }
 var LogBatcher = class {
 	batch = [];
@@ -256,9 +367,16 @@ var LogBatcher = class {
 		this.batchSize = batchSize;
 		this.flushInterval = flushInterval;
 	}
-	addLog(entry) {
-		if (this.isDestroyed) return pushLogToDB(this.options, entry);
-		this.batch.push(entry);
+	async addLog(entry) {
+		if (this.isDestroyed) {
+			await pushLogToDB(this.options, entry);
+			return Promise.resolve();
+		}
+		const truncatedEntry = {
+			...entry,
+			message: truncateLogMessage(entry.message)
+		};
+		this.batch.push(truncatedEntry);
 		if (!this.flushTimer) this.flushTimer = setTimeout(() => {
 			this.flush();
 		}, this.flushInterval);
@@ -398,7 +516,8 @@ async function workflowTableRowToWorkflowRun(row, serializer, externalBlobStorag
 		workflowStatus: row.workflowStatus,
 		startTime: row.startTime,
 		endTime: row.endTime,
-		parentInstanceId: row.parentInstanceId
+		parentInstanceId: row.parentInstanceId,
+		triggerId: row.triggerId
 	};
 }
 async function updateWorkflowName(context, instanceId, newWorkflowName) {
@@ -643,6 +762,17 @@ const createLogAccessor = (context) => {
 		}
 		return null;
 	};
+	const getWorkflowByTriggerId = async (triggerId) => {
+		const result = await context.D1.prepare(
+			/* sql */
+			`SELECT * FROM WorkflowTable WHERE triggerId = ? AND tenantId = ?`
+).bind(triggerId, context.tenantId).first();
+		if (result) {
+			const workflow = await workflowTableRowToWorkflowRun(result, internalSerializer, context.externalBlobStorage);
+			return workflow;
+		}
+		return null;
+	};
 	const getWorkflowTypesByTenantId = async (tenantId) => {
 		const result = await context.D1.prepare(
 			/* sql */
@@ -761,6 +891,7 @@ const createLogAccessor = (context) => {
 		listWorkflows,
 		getWorkflow,
 		getWorkflowTypesByTenantId,
+		getWorkflowByTriggerId,
 		getPropertiesKeys
 	};
 };
@@ -937,11 +1068,21 @@ function createWorkflowContext(options) {
 		serializer: options.serializer ?? defaultSerializer,
 		idFactory: options.idFactory ?? defaultIdFactory
 	};
-	const call = async ({ workflow, input, workflowName, tenantId, parentInstanceId, reuseSuccessfulSteps }) => {
+	const call = async ({ workflow, input, workflowName, tenantId, parentInstanceId, reuseSuccessfulSteps, triggerId }) => {
 		if (!ensuredTables) {
 			await ensureTables(options.D1);
 			ensuredTables = true;
 		}
+		if (triggerId) {
+			const logAccessor = createLogAccessor({
+				D1: internalContext.D1,
+				externalBlobStorage: internalContext.externalBlobStorage,
+				serializer: internalContext.serializer,
+				tenantId
+			});
+			const existingWorkflow = await logAccessor.getWorkflowByTriggerId(triggerId);
+			throw new Error(`Workflow with triggerId ${triggerId} already exists: ${existingWorkflow?.instanceId}`);
+		}
 		const instanceId = internalContext.idFactory();
 		const startTime = Date.now();
 		await insertWorkflowRecord(internalContext, {
@@ -954,7 +1095,8 @@ function createWorkflowContext(options) {
 			startTime,
 			endTime: null,
 			parentInstanceId,
-			tenantId
+			tenantId,
+			triggerId
 		});
 		const logBatcher = new LogBatcher(internalContext);
 		let logOrder = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brandboostinggmbh/observable-workflows",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "My awesome typescript library",
   "type": "module",
   "license": "MIT",