@mastra/dynamodb 1.4.1 → 1.4.2-alpha.0

package/dist/index.js

@@ -1,7 +1,7 @@
 import { DescribeTableCommand, DynamoDBClient } from "@aws-sdk/client-dynamodb";
 import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb";
 import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
-import { BackgroundTasksStorage, MastraCompositeStore, MemoryStorage, SCORERS_SCHEMA, ScoresStorage, TABLE_BACKGROUND_TASKS, TABLE_MESSAGES, TABLE_RESOURCES, TABLE_SCORERS, TABLE_THREADS, TABLE_WORKFLOW_SNAPSHOT, WorkflowsStorage, calculatePagination, createStorageErrorId, matchesExpectedWorkflowStatus, normalizePerPage, storageMessageMatchesMetadataFilter, validateStorageMetadataFilter } from "@mastra/core/storage";
+import { BackgroundTasksStorage, MastraCompositeStore, MemoryStorage, RunFenceConflictError, SCORERS_SCHEMA, ScoresStorage, TABLE_BACKGROUND_TASKS, TABLE_MEMORY_RUN_FENCES, TABLE_MESSAGES, TABLE_RESOURCES, TABLE_SCORERS, TABLE_THREADS, TABLE_WORKFLOW_RUN_OWNERS, TABLE_WORKFLOW_SNAPSHOT, WorkflowsStorage, calculatePagination, createStorageErrorId, isRunFenceConflictError, matchesExpectedWorkflowStatus, matchesRunFence, normalizePerPage, resolveRunFence, storageMessageMatchesMetadataFilter, validateStorageMetadataFilter } from "@mastra/core/storage";
 import { Entity, Service } from "electrodb";
 import { MessageList } from "@mastra/core/agent";
 import { saveScorePayloadSchema } from "@mastra/core/evals";
@@ -498,6 +498,80 @@ const resourceEntity = new Entity({
 	} }
 });
 //#endregion
+//#region src/entities/run-fencing.ts
+/**
+* The current claim on a durable workflow run. `leaseExpiresAt` holds the
+* lease's expiry in epoch milliseconds and is removed when the owner releases
+* the run; `generation` and `ownerId` are kept. `updatedAt` is the epoch
+* milliseconds of the last claim, renewal or release.
+*/
+const workflowRunOwnerEntity = new Entity({
+	model: {
+		entity: "workflow_run_owner",
+		version: "1",
+		service: "mastra"
+	},
+	attributes: {
+		entity: {
+			type: "string",
+			required: true
+		},
+		run_id: {
+			type: "string",
+			required: true
+		},
+		generation: { type: "number" },
+		ownerId: { type: "string" },
+		leaseExpiresAt: { type: "number" },
+		updatedAt: { type: "number" }
+	},
+	indexes: { primary: {
+		pk: {
+			field: "pk",
+			composite: ["entity", "run_id"]
+		},
+		sk: {
+			field: "sk",
+			composite: ["entity"]
+		}
+	} }
+});
+/**
+* The newest fence memory has seen for a run. Fenced memory writes check it.
+* `retiredAt` is set, in epoch milliseconds, once the execution holding the
+* fence settles, and removed when a fence is raised again.
+*/
+const memoryRunFenceEntity = new Entity({
+	model: {
+		entity: "memory_run_fence",
+		version: "1",
+		service: "mastra"
+	},
+	attributes: {
+		entity: {
+			type: "string",
+			required: true
+		},
+		run_id: {
+			type: "string",
+			required: true
+		},
+		generation: { type: "number" },
+		ownerId: { type: "string" },
+		retiredAt: { type: "number" }
+	},
+	indexes: { primary: {
+		pk: {
+			field: "pk",
+			composite: ["entity", "run_id"]
+		},
+		sk: {
+			field: "sk",
+			composite: ["entity"]
+		}
+	} }
+});
+//#endregion
 //#region src/entities/score.ts
 const scoreEntity = new Entity({
 	model: {
@@ -1128,7 +1202,9 @@ function getElectroDbService(client, tableName) {
 		workflow_snapshot: workflowSnapshotEntity,
 		resource: resourceEntity,
 		score: scoreEntity,
-		background_task: backgroundTaskEntity
+		background_task: backgroundTaskEntity,
+		workflow_run_owner: workflowRunOwnerEntity,
+		memory_run_fence: memoryRunFenceEntity
 	}, {
 		client,
 		table: tableName
@@ -1166,7 +1242,9 @@ const ENTITY_MAP = {
 	[TABLE_RESOURCES]: "resource",
 	[TABLE_WORKFLOW_SNAPSHOT]: "workflow_snapshot",
 	[TABLE_SCORERS]: "score",
-	[TABLE_BACKGROUND_TASKS]: "background_task"
+	[TABLE_BACKGROUND_TASKS]: "background_task",
+	[TABLE_WORKFLOW_RUN_OWNERS]: "workflow_run_owner",
+	[TABLE_MEMORY_RUN_FENCES]: "memory_run_fence"
 };
 /**
 * Gets the primary key fields for an entity based on the scanned item
@@ -1185,6 +1263,10 @@ function getDeleteKey(entityName, item) {
 			key.workflow_name = item.workflow_name;
 			key.run_id = item.run_id;
 			break;
+		case "workflow_run_owner":
+		case "memory_run_fence":
+			key.run_id = item.run_id;
+			break;
 		default: key.id = item.id;
 	}
 	return key;
@@ -1520,6 +1602,91 @@ function getTtlProps(entityName, ttlConfig, customTtlSeconds) {
 	return { [getTtlAttributeName(entityName, ttlConfig)]: ttlValue };
 }
 //#endregion
+//#region src/storage/domains/run-fencing.ts
+const MAX_CONFLICT_RETRIES = 8;
+const BASE_DELAY_MS$1 = 20;
+async function backoff(attempt) {
+	const delay = BASE_DELAY_MS$1 * 2 ** attempt;
+	await new Promise((resolve) => setTimeout(resolve, delay + Math.random() * delay * .5));
+}
+function hasErrorName(error, errorName) {
+	let current = error;
+	for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
+		const { name, code } = current;
+		if (name === errorName || code === errorName) return true;
+		current = current.cause;
+	}
+	return false;
+}
+function isTransactionConflict(error) {
+	return hasErrorName(error, "TransactionConflictException");
+}
+/** Whether a write failed its condition, either directly or wrapped by ElectroDB. */
+function isConditionalCheckFailed(error) {
+	return hasErrorName(error, "ConditionalCheckFailedException");
+}
+/**
+* Runs a single-item write, retrying while a transaction is writing the same
+* item. DynamoDB rejects such a write with TransactionConflictException
+* instead of waiting for the transaction, and the SDK does not retry it.
+*/
+async function retryOnTransactionConflict(write) {
+	for (let attempt = 0;; attempt++) try {
+		return await write();
+	} catch (error) {
+		if (attempt >= MAX_CONFLICT_RETRIES || !isTransactionConflict(error)) throw error;
+		await backoff(attempt);
+	}
+}
+/**
+* Rejects a fenced write that turns out to have nothing to write when its
+* fence is no longer current, as the write itself would have been rejected.
+*/
+async function assertRunFence(service, check) {
+	if (!check) return;
+	const { claims, fence, operation } = check;
+	const { data } = await service.entities[claims].get({
+		entity: claims,
+		run_id: fence.runId
+	}).go({ consistent: true });
+	if (!matchesRunFence(data, fence)) throw new RunFenceConflictError(fence, operation);
+}
+/**
+* Runs a write, fenced when `check` is set. A fenced write is a transaction
+* whose first item is a condition check on the run's claim, matching it on
+* the fence. A fence that is no longer current fails the check, and the write
+* throws `RunFenceConflictError` without writing anything. Transactions are
+* serializable with the single-item writes that claim a run, so a takeover
+* lands either before the transaction (failing its check) or after it.
+*
+* A fenced write whose own condition fails throws an error named
+* `ConditionalCheckFailedException`, as the unfenced write would.
+*/
+async function goWithRunFence(service, check, write) {
+	if (!check) {
+		await write.go();
+		return;
+	}
+	const { claims, fence, operation } = check;
+	for (let attempt = 0;; attempt++) {
+		const result = await service.transaction.write((entities) => [entities[claims].check({
+			entity: claims,
+			run_id: fence.runId
+		}).where((attr, op) => `${op.eq(attr.generation, fence.generation)} AND ${op.eq(attr.ownerId, fence.ownerId)}`).commit(), write.commit()]).go();
+		if (!result.canceled) return;
+		const [claim, target] = result.data;
+		if (claim?.code === "ConditionalCheckFailed") throw new RunFenceConflictError(fence, operation);
+		if (target?.code === "ConditionalCheckFailed") {
+			const error = new Error(target.message ?? "The conditional request failed");
+			error.name = "ConditionalCheckFailedException";
+			throw error;
+		}
+		const reasons = result.data.map((item) => item?.code ?? "None");
+		if (attempt >= MAX_CONFLICT_RETRIES || !reasons.includes("TransactionConflict")) throw new Error(`DynamoDB canceled the fenced ${operation} transaction: ${reasons.join(", ")}`);
+		await backoff(attempt);
+	}
+}
+//#endregion
 //#region src/storage/domains/memory/index.ts
 var MemoryStorageDynamoDB = class extends MemoryStorage {
 	supportsPartialThreadUpdate = true;
@@ -1531,15 +1698,81 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 		this.service = resolved.service;
 		this.ttlConfig = resolved.ttl;
 	}
+	supportsRunFencing() {
+		return true;
+	}
 	async dangerouslyClearAll() {
 		await deleteTableData(this.service, TABLE_THREADS);
 		await deleteTableData(this.service, TABLE_MESSAGES);
 		await deleteTableData(this.service, TABLE_RESOURCES);
+		await deleteTableData(this.service, TABLE_MEMORY_RUN_FENCES);
 	}
-	async deleteMessages(messageIds) {
+	async raiseRunFence(fence) {
+		const fences = this.service.entities.memory_run_fence;
+		const key = {
+			entity: "memory_run_fence",
+			run_id: fence.runId
+		};
+		try {
+			await retryOnTransactionConflict(() => fences.update(key).set({
+				generation: fence.generation,
+				ownerId: fence.ownerId
+			}).remove(["retiredAt"]).where((attr, op) => `${op.notExists(attr.generation)} OR ${op.lt(attr.generation, fence.generation)} OR (${op.eq(attr.generation, fence.generation)} AND ${op.eq(attr.ownerId, fence.ownerId)})`).go());
+			return true;
+		} catch (error) {
+			try {
+				if (!isConditionalCheckFailed(error)) throw error;
+				const { data } = await fences.get(key).go({ consistent: true });
+				return matchesRunFence(data, fence);
+			} catch (cause) {
+				throw new MastraError({
+					id: createStorageErrorId("DYNAMODB", "RAISE_RUN_FENCE", "FAILED"),
+					domain: ErrorDomain.STORAGE,
+					category: ErrorCategory.THIRD_PARTY,
+					details: { runId: fence.runId }
+				}, cause);
+			}
+		}
+	}
+	async retireRunFence(fence) {
+		try {
+			await retryOnTransactionConflict(() => this.service.entities.memory_run_fence.patch({
+				entity: "memory_run_fence",
+				run_id: fence.runId
+			}).set({ retiredAt: Date.now() }).where((attr, op) => `${op.eq(attr.generation, fence.generation)} AND ${op.eq(attr.ownerId, fence.ownerId)}`).go());
+			return true;
+		} catch (error) {
+			if (isConditionalCheckFailed(error)) return false;
+			throw new MastraError({
+				id: createStorageErrorId("DYNAMODB", "RETIRE_RUN_FENCE", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { runId: fence.runId }
+			}, error);
+		}
+	}
+	/** The check a write must pass: its own fence, otherwise the one in scope. */
+	#runFenceCheck(fence, operation) {
+		const resolved = resolveRunFence(this, fence);
+		return resolved && {
+			claims: "memory_run_fence",
+			fence: resolved,
+			operation
+		};
+	}
+	/** Bumps a thread's updatedAt after a write to its messages. */
+	#touchThread(threadId, check) {
+		return goWithRunFence(this.service, check, this.service.entities.thread.update({
+			entity: "thread",
+			id: threadId
+		}).set({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }));
+	}
+	async deleteMessages(messageIds, options) {
 		if (!messageIds || messageIds.length === 0) return;
 		this.logger.debug("Deleting messages", { count: messageIds.length });
+		const check = this.#runFenceCheck(options?.fence, "deleteMessages");
 		try {
+			let deleted = 0;
 			const threadIds = /* @__PURE__ */ new Set();
 			const batchSize = 25;
 			for (let i = 0; i < messageIds.length; i += batchSize) {
@@ -1552,18 +1785,17 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 				}));
 				for (const message of messagesToDelete) if (message) {
 					if (message.threadId) threadIds.add(message.threadId);
-					await this.service.entities.message.delete({
+					await goWithRunFence(this.service, check, this.service.entities.message.delete({
 						entity: "message",
 						id: message.id
-					}).go();
+					}));
+					deleted++;
 				}
 			}
-			const now = (/* @__PURE__ */ new Date()).toISOString();
-			for (const threadId of threadIds) await this.service.entities.thread.update({
-				entity: "thread",
-				id: threadId
-			}).set({ updatedAt: now }).go();
+			if (deleted === 0) await assertRunFence(this.service, check);
+			for (const threadId of threadIds) await this.#touchThread(threadId, check);
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "DELETE_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -1617,7 +1849,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 			}, error);
 		}
 	}
-	async saveThread({ thread }) {
+	async saveThread({ thread, fence }) {
 		this.logger.debug("Saving thread", { threadId: thread.id });
 		const now = /* @__PURE__ */ new Date();
 		const threadData = {
@@ -1631,7 +1863,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 			...getTtlProps("thread", this.ttlConfig)
 		};
 		try {
-			await this.service.entities.thread.upsert(threadData).go();
+			await goWithRunFence(this.service, this.#runFenceCheck(fence, "saveThread"), this.service.entities.thread.upsert(threadData));
 			return {
 				id: thread.id,
 				resourceId: thread.resourceId,
@@ -1641,6 +1873,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 				metadata: thread.metadata
 			};
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "SAVE_THREAD", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -1649,8 +1882,9 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 			}, error);
 		}
 	}
-	async updateThread({ id, title, metadata }) {
+	async updateThread({ id, title, metadata, fence }) {
 		this.logger.debug("Updating thread", { threadId: id });
+		const check = this.#runFenceCheck(fence, "updateThread");
 		try {
 			const existingThread = await this.getThreadById({ threadId: id });
 			if (!existingThread) throw new Error(`Thread not found: ${id}`);
@@ -1664,10 +1898,10 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 				};
 				updateData.metadata = JSON.stringify(mergedMetadata);
 			}
-			await this.service.entities.thread.update({
+			await goWithRunFence(this.service, check, this.service.entities.thread.update({
 				entity: "thread",
 				id
-			}).set(updateData).go();
+			}).set(updateData));
 			return {
 				...existingThread,
 				title: title || existingThread.title,
@@ -1678,6 +1912,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 				updatedAt: now
 			};
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "UPDATE_THREAD", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -1925,6 +2160,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 				...getTtlProps("message", this.ttlConfig)
 			};
 		});
+		const check = this.#runFenceCheck(args.fence, "saveMessages");
 		try {
 			const savedMessageIds = [];
 			for (const messageData of messagesToSave) {
@@ -1933,14 +2169,14 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 					throw new Error("Internal error: Missing entity property during saveMessages");
 				}
 				try {
-					await this.service.entities.message.put(messageData).go();
+					await goWithRunFence(this.service, check, this.service.entities.message.put(messageData));
 					savedMessageIds.push(messageData.id);
 				} catch (error) {
-					for (const savedId of savedMessageIds) try {
-						await this.service.entities.message.delete({
+					for (const savedId of isRunFenceConflictError(error) ? [] : savedMessageIds) try {
+						await goWithRunFence(this.service, check, this.service.entities.message.delete({
 							entity: "message",
 							id: savedId
-						}).go();
+						}));
 					} catch (rollbackError) {
 						this.logger.error("Failed to rollback message during save error", {
 							messageId: savedId,
@@ -1950,12 +2186,10 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 					throw error;
 				}
 			}
-			await this.service.entities.thread.update({
-				entity: "thread",
-				id: threadId
-			}).set({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).go();
+			await this.#touchThread(threadId, check);
 			return { messages: new MessageList().add(messages, "memory").get.all.db() };
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "SAVE_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -2109,6 +2343,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 		const { messages } = args;
 		this.logger.debug("Updating messages", { count: messages.length });
 		if (!messages.length) return [];
+		const check = this.#runFenceCheck(args.fence, "updateMessages");
 		const updatedMessages = [];
 		const affectedThreadIds = /* @__PURE__ */ new Set();
 		try {
@@ -2144,22 +2379,21 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 					if ("parts" in updates.content && updates.content.parts !== void 0) newContent.parts = updates.content.parts;
 					updatePayload.content = JSON.stringify(newContent);
 				}
-				await this.service.entities.message.update({
+				await goWithRunFence(this.service, check, this.service.entities.message.update({
 					entity: "message",
 					id
-				}).set(updatePayload).go();
+				}).set(updatePayload));
 				const updatedMessage = await this.service.entities.message.get({
 					entity: "message",
 					id
 				}).go();
 				if (updatedMessage.data) updatedMessages.push(this.parseMessageData(updatedMessage.data));
 			}
-			for (const threadId of affectedThreadIds) await this.service.entities.thread.update({
-				entity: "thread",
-				id: threadId
-			}).set({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).go();
+			if (affectedThreadIds.size === 0) await assertRunFence(this.service, check);
+			for (const threadId of affectedThreadIds) await this.#touchThread(threadId, check);
 			return updatedMessages;
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "UPDATE_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -2193,6 +2427,9 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 		}
 	}
 	async saveResource({ resource }) {
+		return this.#saveResource(resource, void 0);
+	}
+	async #saveResource(resource, check) {
 		this.logger.debug("Saving resource", { resourceId: resource.id });
 		const now = /* @__PURE__ */ new Date();
 		const resourceData = {
@@ -2205,7 +2442,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 			...getTtlProps("resource", this.ttlConfig)
 		};
 		try {
-			await this.service.entities.resource.upsert(resourceData).go();
+			await goWithRunFence(this.service, check, this.service.entities.resource.upsert(resourceData));
 			return {
 				id: resource.id,
 				workingMemory: resource.workingMemory,
@@ -2214,6 +2451,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 				updatedAt: now
 			};
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "SAVE_RESOURCE", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -2222,8 +2460,9 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 			}, error);
 		}
 	}
-	async updateResource({ resourceId, workingMemory, metadata }) {
+	async updateResource({ resourceId, workingMemory, metadata, fence }) {
 		this.logger.debug("Updating resource", { resourceId });
+		const check = this.#runFenceCheck(fence, "updateResource");
 		try {
 			const existingResource = await this.getResourceById({ resourceId });
 			if (!existingResource) {
@@ -2234,7 +2473,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 					createdAt: /* @__PURE__ */ new Date(),
 					updatedAt: /* @__PURE__ */ new Date()
 				};
-				return this.saveResource({ resource: newResource });
+				return this.#saveResource(newResource, check);
 			}
 			const now = /* @__PURE__ */ new Date();
 			const updateData = { updatedAt: now.toISOString() };
@@ -2246,10 +2485,10 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 				};
 				updateData.metadata = JSON.stringify(mergedMetadata);
 			}
-			await this.service.entities.resource.update({
+			await goWithRunFence(this.service, check, this.service.entities.resource.update({
 				entity: "resource",
 				id: resourceId
-			}).set(updateData).go();
+			}).set(updateData));
 			return {
 				...existingResource,
 				workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
@@ -2260,6 +2499,7 @@ var MemoryStorageDynamoDB = class extends MemoryStorage {
 				updatedAt: now
 			};
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "UPDATE_RESOURCE", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -2581,6 +2821,15 @@ function formatWorkflowRun(snapshotData) {
 		resourceId: snapshotData.resourceId
 	};
 }
+function toRunOwnershipRecord(item) {
+	return {
+		runId: item.run_id,
+		generation: Number(item.generation),
+		ownerId: item.ownerId,
+		leaseExpiresAt: item.leaseExpiresAt === void 0 ? null : new Date(item.leaseExpiresAt),
+		live: item.leaseExpiresAt !== void 0 && item.leaseExpiresAt > Date.now()
+	};
+}
 const MAX_RETRIES = 5;
 const BASE_DELAY_MS = 50;
 var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
@@ -2595,8 +2844,126 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 	supportsConcurrentUpdates() {
 		return true;
 	}
+	supportsRunFencing() {
+		return true;
+	}
 	async dangerouslyClearAll() {
 		await deleteTableData(this.service, TABLE_WORKFLOW_SNAPSHOT);
+		await deleteTableData(this.service, TABLE_WORKFLOW_RUN_OWNERS);
+	}
+	#runFenceCheck(fence, runId, operation) {
+		const resolved = resolveRunFence(this, fence, runId);
+		return resolved && {
+			claims: "workflow_run_owner",
+			fence: resolved,
+			operation
+		};
+	}
+	#runOwnerKey(runId) {
+		return {
+			entity: "workflow_run_owner",
+			run_id: runId
+		};
+	}
+	async #readRunOwner(runId) {
+		const { data } = await this.service.entities.workflow_run_owner.get(this.#runOwnerKey(runId)).go({ consistent: true });
+		return data ? toRunOwnershipRecord(data) : null;
+	}
+	#ownershipError(operation, runId, error) {
+		return new MastraError({
+			id: createStorageErrorId("DYNAMODB", operation, "FAILED"),
+			domain: ErrorDomain.STORAGE,
+			category: ErrorCategory.THIRD_PARTY,
+			details: { runId }
+		}, error);
+	}
+	async claimRunOwnership({ runId, ownerId, leaseMs, force, expectedGeneration }) {
+		try {
+			const owners = this.service.entities.workflow_run_owner;
+			const now = Date.now();
+			if (!expectedGeneration) {
+				try {
+					const item = {
+						...this.#runOwnerKey(runId),
+						generation: Math.max(1, now),
+						ownerId,
+						leaseExpiresAt: now + leaseMs,
+						updatedAt: now
+					};
+					await retryOnTransactionConflict(() => owners.create(item).go());
+					return {
+						acquired: true,
+						record: toRunOwnershipRecord(item)
+					};
+				} catch (error) {
+					if (!this.isConditionalCheckFailed(error)) throw error;
+				}
+				if (expectedGeneration === 0) return {
+					acquired: false,
+					record: await this.#readRunOwner(runId)
+				};
+			}
+			try {
+				const { data } = await retryOnTransactionConflict(() => owners.patch(this.#runOwnerKey(runId)).set({
+					ownerId,
+					leaseExpiresAt: now + leaseMs,
+					updatedAt: now
+				}).add({ generation: 1 }).where((attr, op) => [expectedGeneration !== void 0 ? op.eq(attr.generation, expectedGeneration) : "", force ? "" : `(${op.notExists(attr.leaseExpiresAt)} OR ${op.lte(attr.leaseExpiresAt, now)})`].filter(Boolean).join(" AND ")).go({ response: "all_new" }));
+				return {
+					acquired: true,
+					record: toRunOwnershipRecord(data)
+				};
+			} catch (error) {
+				if (!this.isConditionalCheckFailed(error)) throw error;
+				return {
+					acquired: false,
+					record: await this.#readRunOwner(runId)
+				};
+			}
+		} catch (error) {
+			throw this.#ownershipError("CLAIM_RUN_OWNERSHIP", runId, error);
+		}
+	}
+	async renewRunOwnership({ leaseMs, ...fence }) {
+		try {
+			const { data } = await retryOnTransactionConflict(() => {
+				const now = Date.now();
+				return this.service.entities.workflow_run_owner.patch(this.#runOwnerKey(fence.runId)).set({
+					leaseExpiresAt: now + leaseMs,
+					updatedAt: now
+				}).where((attr, op) => `${op.eq(attr.generation, fence.generation)} AND ${op.eq(attr.ownerId, fence.ownerId)} AND ${op.exists(attr.leaseExpiresAt)}`).go({ response: "all_new" });
+			});
+			return {
+				renewed: true,
+				record: toRunOwnershipRecord(data)
+			};
+		} catch (error) {
+			if (!this.isConditionalCheckFailed(error)) throw this.#ownershipError("RENEW_RUN_OWNERSHIP", fence.runId, error);
+			try {
+				return {
+					renewed: false,
+					record: await this.#readRunOwner(fence.runId)
+				};
+			} catch (readError) {
+				throw this.#ownershipError("RENEW_RUN_OWNERSHIP", fence.runId, readError);
+			}
+		}
+	}
+	async releaseRunOwnership(fence) {
+		try {
+			await retryOnTransactionConflict(() => this.service.entities.workflow_run_owner.patch(this.#runOwnerKey(fence.runId)).set({ updatedAt: Date.now() }).remove(["leaseExpiresAt"]).where((attr, op) => `${op.eq(attr.generation, fence.generation)} AND ${op.eq(attr.ownerId, fence.ownerId)}`).go());
+			return true;
+		} catch (error) {
+			if (this.isConditionalCheckFailed(error)) return false;
+			throw this.#ownershipError("RELEASE_RUN_OWNERSHIP", fence.runId, error);
+		}
+	}
+	async getRunOwnership({ runId }) {
+		try {
+			return await this.#readRunOwner(runId);
+		} catch (error) {
+			throw this.#ownershipError("GET_RUN_OWNERSHIP", runId, error);
+		}
 	}
 	/**
 	* Helper to check if an error is a DynamoDB conditional check failure
@@ -2621,7 +2988,8 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 		const jitter = Math.random() * backoff * .5;
 		await new Promise((resolve) => setTimeout(resolve, backoff + jitter));
 	}
-	async updateWorkflowResults({ workflowName, runId, stepId, result, requestContext, state }) {
+	async updateWorkflowResults({ workflowName, runId, stepId, result, requestContext, state, fence }) {
+		const check = this.#runFenceCheck(fence, runId, "updateWorkflowResults");
 		for (let attempt = 0; attempt < MAX_RETRIES; attempt++) try {
 			const existingRecord = await this.service.entities.workflow_snapshot.get({
 				entity: "workflow_snapshot",
@@ -2664,8 +3032,8 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 				updatedAt: now.toISOString(),
 				...getTtlProps("workflow_snapshot", this.ttlConfig)
 			};
-			if (previousUpdatedAt) await this.service.entities.workflow_snapshot.upsert(data).where((attr, op) => op.eq(attr.updatedAt, previousUpdatedAt)).go();
-			else await this.service.entities.workflow_snapshot.create(data).where((attr, op) => op.notExists(attr.run_id)).go();
+			const write = previousUpdatedAt ? this.service.entities.workflow_snapshot.upsert(data).where((attr, op) => op.eq(attr.updatedAt, previousUpdatedAt)) : this.service.entities.workflow_snapshot.create(data).where((attr, op) => op.notExists(attr.run_id));
+			await goWithRunFence(this.service, check, write);
 			return snapshot.context;
 		} catch (error) {
 			if (this.isConditionalCheckFailed(error)) {
@@ -2698,19 +3066,21 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 			}
 		}, /* @__PURE__ */ new Error("Max retries exceeded for optimistic locking"));
 	}
-	async updateWorkflowState({ workflowName, runId, opts }) {
+	async updateWorkflowState({ workflowName, runId, opts, fence }) {
+		const check = this.#runFenceCheck(fence, runId, "updateWorkflowState");
 		for (let attempt = 0; attempt < MAX_RETRIES; attempt++) try {
 			const existingRecord = await this.service.entities.workflow_snapshot.get({
 				entity: "workflow_snapshot",
 				workflow_name: workflowName,
 				run_id: runId
 			}).go();
-			if (!existingRecord.data) return;
-			const existingSnapshot = existingRecord.data.snapshot;
-			if (!existingSnapshot || !existingSnapshot.context) return;
-			const previousUpdatedAt = existingRecord.data.updatedAt;
+			const existingSnapshot = existingRecord.data?.snapshot;
 			const { expectedStatus, ...state } = opts;
-			if (!matchesExpectedWorkflowStatus(existingSnapshot.status, expectedStatus)) return;
+			if (!existingSnapshot?.context || !matchesExpectedWorkflowStatus(existingSnapshot.status, expectedStatus)) {
+				await assertRunFence(this.service, check);
+				return;
+			}
+			const previousUpdatedAt = existingRecord.data.updatedAt;
 			const updatedSnapshot = {
 				...existingSnapshot,
 				...state
@@ -2726,7 +3096,7 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 				resourceId: existingRecord.data.resourceId,
 				...getTtlProps("workflow_snapshot", this.ttlConfig)
 			};
-			await this.service.entities.workflow_snapshot.upsert(data).where((attr, op) => op.eq(attr.updatedAt, previousUpdatedAt)).go();
+			await goWithRunFence(this.service, check, this.service.entities.workflow_snapshot.upsert(data).where((attr, op) => op.eq(attr.updatedAt, previousUpdatedAt)));
 			return updatedSnapshot;
 		} catch (error) {
 			if (this.isConditionalCheckFailed(error)) {
@@ -2757,11 +3127,12 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 			}
 		}, /* @__PURE__ */ new Error("Max retries exceeded for optimistic locking"));
 	}
-	async persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot, createdAt, updatedAt }) {
+	async persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot, createdAt, updatedAt, fence }) {
 		this.logger.debug("Persisting workflow snapshot", {
 			workflowName,
 			runId
 		});
+		const check = this.#runFenceCheck(fence, runId, "persistWorkflowSnapshot");
 		try {
 			const now = /* @__PURE__ */ new Date();
 			const createdAtValue = (createdAt ?? now).toISOString();
@@ -2777,10 +3148,10 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 				...getTtlProps("workflow_snapshot", this.ttlConfig)
 			};
 			try {
-				await this.service.entities.workflow_snapshot.create(data).where((attr, op) => op.notExists(attr.run_id)).go();
+				await goWithRunFence(this.service, check, this.service.entities.workflow_snapshot.create(data).where((attr, op) => op.notExists(attr.run_id)));
 			} catch (error) {
 				if (!this.isConditionalCheckFailed(error)) throw error;
-				await this.service.entities.workflow_snapshot.update({
+				await goWithRunFence(this.service, check, this.service.entities.workflow_snapshot.update({
 					entity: "workflow_snapshot",
 					workflow_name: workflowName,
 					run_id: runId
@@ -2789,9 +3160,10 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 					updatedAt: updatedAtValue,
 					resourceId,
 					...getTtlProps("workflow_snapshot", this.ttlConfig)
-				}).go();
+				}));
 			}
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "PERSIST_WORKFLOW_SNAPSHOT", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -2950,18 +3322,19 @@ var WorkflowStorageDynamoDB = class extends WorkflowsStorage {
 			}, error);
 		}
 	}
-	async deleteWorkflowRunById({ runId, workflowName }) {
+	async deleteWorkflowRunById({ runId, workflowName, fence }) {
 		this.logger.debug("Deleting workflow run by ID", {
 			runId,
 			workflowName
 		});
 		try {
-			await this.service.entities.workflow_snapshot.delete({
+			await goWithRunFence(this.service, this.#runFenceCheck(fence, runId, "deleteWorkflowRunById"), this.service.entities.workflow_snapshot.delete({
 				entity: "workflow_snapshot",
 				workflow_name: workflowName,
 				run_id: runId
-			}).go();
+			}));
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("DYNAMODB", "DELETE_WORKFLOW_RUN_BY_ID", "FAILED"),
 				domain: ErrorDomain.STORAGE,