@mastra/upstash 1.5.2-alpha.0 → 1.5.3

package/dist/index.js

@@ -1,6 +1,6 @@
 import { RedisServerCache, RedisServerCache as RedisServerCache$1, upstashPreset, upstashPreset as upstashPreset$1 } from "@mastra/redis";
 import { Redis } from "@upstash/redis";
-import { BackgroundTasksStorage, MastraCompositeStore, MemoryStorage, ScoresStorage, TABLE_BACKGROUND_TASKS, TABLE_MESSAGES, TABLE_RESOURCES, TABLE_SCORERS, TABLE_THREADS, TABLE_WORKFLOW_SNAPSHOT, WorkflowsStorage, calculatePagination, createStorageErrorId, createVectorErrorId, ensureDate, filterByDateRange, normalizePerPage, serializeDate, storageMessageMatchesMetadataFilter, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
+import { BackgroundTasksStorage, MastraCompositeStore, MemoryStorage, RunFenceConflictError, ScoresStorage, TABLE_BACKGROUND_TASKS, TABLE_MEMORY_RUN_FENCES, TABLE_MESSAGES, TABLE_RESOURCES, TABLE_SCORERS, TABLE_THREADS, TABLE_WORKFLOW_RUN_OWNERS, TABLE_WORKFLOW_SNAPSHOT, WorkflowsStorage, calculatePagination, createStorageErrorId, createVectorErrorId, ensureDate, filterByDateRange, isRunFenceConflictError, normalizePerPage, resolveRunFence, serializeDate, storageMessageMatchesMetadataFilter, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
 import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
 import { MessageList } from "@mastra/core/agent";
 import { saveScorePayloadSchema } from "@mastra/core/evals";
@@ -350,6 +350,239 @@ var BackgroundTasksUpstash = class extends BackgroundTasksStorage {
 	}
 };
 //#endregion
+//#region src/storage/domains/run-fencing.ts
+function runClaimKey(claims, runId) {
+	return `${claims}:run_id:${runId}`;
+}
+const RUN_FENCE_CONFLICT = "MASTRA_RUN_FENCE_CONFLICT";
+const RUN_CLAIM_RETENTION_MS = 10080 * 60 * 1e3;
+const FENCE_GUARD = `
+local fenceClaim = redis.call('HMGET', KEYS[#KEYS], 'generation', 'ownerId')
+if tonumber(fenceClaim[1]) ~= tonumber(ARGV[#ARGV - 1]) or fenceClaim[2] ~= ARGV[#ARGV] then
+  return '${RUN_FENCE_CONFLICT}'
+end
+`;
+const OWNER = `
+local function owner(flag)
+  local r = redis.call('HMGET', KEYS[1], 'generation', 'ownerId', 'leaseExpiresAt')
+  return cjson.encode({ flag, r[1], r[2], r[3] })
+end
+`;
+const CLAIM = `${OWNER}
+local generation = tonumber(redis.call('HGET', KEYS[1], 'generation'))
+local expected = ARGV[3]
+if not generation then
+  if expected ~= '' and expected ~= '0' then return owner(0) end
+  -- The first generation comes from the clock, so a claim after this hash is
+  -- removed still outranks a surviving fence.
+  redis.call('HSET', KEYS[1], 'generation', string.format('%d', math.max(1, tonumber(ARGV[4]))), 'ownerId', ARGV[1], 'leaseExpiresAt', ARGV[5], 'updatedAt', ARGV[4])
+  redis.call('PEXPIRE', KEYS[1], ARGV[6])
+  return owner(1)
+end
+if expected ~= '' and generation ~= tonumber(expected) then return owner(0) end
+if ARGV[2] ~= '1' then
+  local expires = tonumber(redis.call('HGET', KEYS[1], 'leaseExpiresAt'))
+  if expires and expires > tonumber(ARGV[4]) then return owner(0) end
+end
+redis.call('HSET', KEYS[1], 'generation', string.format('%d', generation + 1), 'ownerId', ARGV[1], 'leaseExpiresAt', ARGV[5], 'updatedAt', ARGV[4])
+redis.call('PEXPIRE', KEYS[1], ARGV[6])
+return owner(1)
+`;
+const RENEW = `${OWNER}
+local r = redis.call('HMGET', KEYS[1], 'generation', 'ownerId', 'leaseExpiresAt')
+if tonumber(r[1]) ~= tonumber(ARGV[1]) or r[2] ~= ARGV[2] or not r[3] then return owner(0) end
+redis.call('HSET', KEYS[1], 'leaseExpiresAt', ARGV[3], 'updatedAt', ARGV[4])
+redis.call('PEXPIRE', KEYS[1], ARGV[5])
+return owner(1)
+`;
+const RELEASE = `
+local r = redis.call('HMGET', KEYS[1], 'generation', 'ownerId')
+if tonumber(r[1]) ~= tonumber(ARGV[1]) or r[2] ~= ARGV[2] then return 0 end
+redis.call('HDEL', KEYS[1], 'leaseExpiresAt')
+redis.call('HSET', KEYS[1], 'updatedAt', ARGV[3])
+redis.call('PEXPIRE', KEYS[1], ARGV[4])
+return 1
+`;
+const READ = `${OWNER}
+return owner(1)
+`;
+const RAISE = `
+local r = redis.call('HMGET', KEYS[1], 'generation', 'ownerId')
+local current = tonumber(r[1])
+if not current or current < tonumber(ARGV[1]) then
+  redis.call('HSET', KEYS[1], 'generation', ARGV[1], 'ownerId', ARGV[2])
+elseif current ~= tonumber(ARGV[1]) or r[2] ~= ARGV[2] then
+  return 0
+end
+-- Every write un-retires the fence.
+redis.call('HDEL', KEYS[1], 'retiredAt')
+redis.call('PERSIST', KEYS[1])
+return 1
+`;
+const RETIRE = `
+local r = redis.call('HMGET', KEYS[1], 'generation', 'ownerId')
+if tonumber(r[1]) ~= tonumber(ARGV[1]) or r[2] ~= ARGV[2] then return 0 end
+redis.call('HSET', KEYS[1], 'retiredAt', ARGV[3])
+redis.call('PEXPIRE', KEYS[1], ARGV[4])
+return 1
+`;
+const FENCED_WRITE = `
+local i = 1
+while i <= #ARGV - 2 do
+  local n = tonumber(ARGV[i])
+  redis.call(unpack(ARGV, i + 1, i + n))
+  i = i + n + 1
+end
+return 1
+`;
+/**
+* Runs `script`, first checking the run's claim against `check`'s fence in
+* the same script when `check` is set. A stale fence throws
+* `RunFenceConflictError` before the script does anything.
+*/
+async function evalFenced(client, script, keys, args, check) {
+	if (!check) return client.eval(script, keys, args);
+	const { claims, fence, operation } = check;
+	const result = await client.eval(FENCE_GUARD + script, [...keys, runClaimKey(claims, fence.runId)], [
+		...args,
+		String(fence.generation),
+		fence.ownerId
+	]);
+	if (result === RUN_FENCE_CONFLICT) throw new RunFenceConflictError(fence, operation);
+	return result;
+}
+function toRunOwnershipRecord(runId, reply, now) {
+	const [, generation, ownerId, leaseExpiresAt] = reply;
+	if (!generation) return null;
+	const expiresAt = leaseExpiresAt ? Number(leaseExpiresAt) : null;
+	return {
+		runId,
+		generation: Number(generation),
+		ownerId: String(ownerId),
+		leaseExpiresAt: expiresAt === null ? null : new Date(expiresAt),
+		live: expiresAt !== null && expiresAt > now
+	};
+}
+async function evalOwner(client, script, runId, args, now) {
+	const raw = await client.eval(script, [runClaimKey(TABLE_WORKFLOW_RUN_OWNERS, runId)], args);
+	const reply = typeof raw === "string" ? JSON.parse(raw) : raw;
+	return {
+		applied: Number(reply[0]) === 1,
+		record: toRunOwnershipRecord(runId, reply, now)
+	};
+}
+async function claimRunOwnership(client, { runId, ownerId, leaseMs, force, expectedGeneration }) {
+	const now = Date.now();
+	const { applied, record } = await evalOwner(client, CLAIM, runId, [
+		ownerId,
+		force ? "1" : "0",
+		expectedGeneration === void 0 ? "" : String(expectedGeneration),
+		String(now),
+		String(now + leaseMs),
+		String(RUN_CLAIM_RETENTION_MS)
+	], now);
+	return applied ? {
+		acquired: true,
+		record
+	} : {
+		acquired: false,
+		record
+	};
+}
+async function renewRunOwnership(client, { leaseMs, ...fence }) {
+	const now = Date.now();
+	const { applied, record } = await evalOwner(client, RENEW, fence.runId, [
+		String(fence.generation),
+		fence.ownerId,
+		String(now + leaseMs),
+		String(now),
+		String(RUN_CLAIM_RETENTION_MS)
+	], now);
+	return applied ? {
+		renewed: true,
+		record
+	} : {
+		renewed: false,
+		record
+	};
+}
+async function releaseRunOwnership(client, fence) {
+	const released = await client.eval(RELEASE, [runClaimKey(TABLE_WORKFLOW_RUN_OWNERS, fence.runId)], [
+		String(fence.generation),
+		fence.ownerId,
+		String(Date.now()),
+		String(RUN_CLAIM_RETENTION_MS)
+	]);
+	return Number(released) === 1;
+}
+async function getRunOwnership(client, runId) {
+	return (await evalOwner(client, READ, runId, [], Date.now())).record;
+}
+async function raiseRunFence(client, fence) {
+	const raised = await client.eval(RAISE, [runClaimKey(TABLE_MEMORY_RUN_FENCES, fence.runId)], [String(fence.generation), fence.ownerId]);
+	return Number(raised) === 1;
+}
+async function retireRunFence(client, fence) {
+	const retired = await client.eval(RETIRE, [runClaimKey(TABLE_MEMORY_RUN_FENCES, fence.runId)], [
+		String(fence.generation),
+		fence.ownerId,
+		String(Date.now()),
+		String(RUN_CLAIM_RETENTION_MS)
+	]);
+	return Number(retired) === 1;
+}
+function serialize(value) {
+	return typeof value === "string" ? value : JSON.stringify(value);
+}
+/**
+* Collects writes and applies them in one script that first checks the run's
+* claim against the fence. A fence that is no longer current makes `exec()`
+* throw `RunFenceConflictError` without writing anything. Scripts run
+* atomically, so a takeover lands either before the batch (failing its check)
+* or after it.
+*/
+var FencedWriteBatch = class {
+	client;
+	check;
+	#keys = /* @__PURE__ */ new Set();
+	#args = [];
+	constructor(client, check) {
+		this.client = client;
+		this.check = check;
+	}
+	#add(key, ...command) {
+		this.#keys.add(key);
+		this.#args.push(String(command.length), ...command);
+		return this;
+	}
+	set(key, value) {
+		return this.#add(key, "SET", key, serialize(value));
+	}
+	del(key) {
+		return this.#add(key, "DEL", key);
+	}
+	zadd(key, { score, member }) {
+		return this.#add(key, "ZADD", key, String(score), member);
+	}
+	zrem(key, member) {
+		return this.#add(key, "ZREM", key, member);
+	}
+	async exec() {
+		await evalFenced(this.client, FENCED_WRITE, [...this.#keys], this.#args, this.check);
+	}
+};
+/** A pipeline, or a fenced batch when `check` is set. */
+function writeBatch(client, check) {
+	return check ? new FencedWriteBatch(client, check) : client.pipeline();
+}
+/**
+* Rejects a fenced write that turns out to have nothing to write when its
+* fence is no longer current, as the write itself would have been rejected.
+*/
+async function assertRunFence(client, check) {
+	if (check) await new FencedWriteBatch(client, check).exec();
+}
+//#endregion
 //#region src/storage/domains/memory/index.ts
 function getThreadMessagesKey(threadId) {
 	return `thread:${threadId}:messages`;
@@ -373,10 +606,46 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 		this.client = client;
 		this.#db = new UpstashDB({ client });
 	}
+	supportsRunFencing() {
+		return true;
+	}
+	async raiseRunFence(fence) {
+		try {
+			return await raiseRunFence(this.client, fence);
+		} catch (error) {
+			throw new MastraError({
+				id: createStorageErrorId("UPSTASH", "RAISE_RUN_FENCE", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { runId: fence.runId }
+			}, error);
+		}
+	}
+	async retireRunFence(fence) {
+		try {
+			return await retireRunFence(this.client, fence);
+		} catch (error) {
+			throw new MastraError({
+				id: createStorageErrorId("UPSTASH", "RETIRE_RUN_FENCE", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { runId: fence.runId }
+			}, error);
+		}
+	}
+	#runFenceCheck(fence, operation) {
+		const resolved = resolveRunFence(this, fence);
+		return resolved && {
+			claims: TABLE_MEMORY_RUN_FENCES,
+			fence: resolved,
+			operation
+		};
+	}
 	async dangerouslyClearAll() {
 		await this.#db.deleteData({ tableName: TABLE_THREADS });
 		await this.#db.deleteData({ tableName: TABLE_MESSAGES });
 		await this.#db.deleteData({ tableName: TABLE_RESOURCES });
+		await this.#db.scanAndDelete(`${runClaimKey(TABLE_MEMORY_RUN_FENCES, "")}*`);
 	}
 	async getThreadById({ threadId, resourceId }) {
 		try {
@@ -485,14 +754,16 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 			throw mastraError;
 		}
 	}
-	async saveThread({ thread }) {
+	async saveThread({ thread, fence }) {
+		const check = this.#runFenceCheck(fence, "saveThread");
 		try {
-			await this.#db.insert({
-				tableName: TABLE_THREADS,
-				record: thread
-			});
+			const { key, processedRecord } = processRecord(TABLE_THREADS, thread);
+			const batch = writeBatch(this.client, check);
+			batch.set(key, processedRecord);
+			await batch.exec();
 			return thread;
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			const mastraError = new MastraError({
 				id: createStorageErrorId("UPSTASH", "SAVE_THREAD", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -504,7 +775,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 			throw mastraError;
 		}
 	}
-	async updateThread({ id, title, metadata }) {
+	async updateThread({ id, title, metadata, fence }) {
 		const thread = await this.getThreadById({ threadId: id });
 		if (!thread) throw new MastraError({
 			id: createStorageErrorId("UPSTASH", "UPDATE_THREAD", "FAILED"),
@@ -524,9 +795,13 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 			updatedAt: now
 		};
 		try {
-			await this.saveThread({ thread: updatedThread });
+			await this.saveThread({
+				thread: updatedThread,
+				fence
+			});
 			return updatedThread;
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("UPSTASH", "UPDATE_THREAD", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -561,6 +836,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 	}
 	async saveMessages(args) {
 		const { messages } = args;
+		const check = this.#runFenceCheck(args.fence, "saveMessages");
 		if (messages.length === 0) return { messages: [] };
 		try {
 			for (const message of messages) {
@@ -615,7 +891,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 			}, /* @__PURE__ */ new Error(`Thread ${targetThreadId} not found`));
 			for (let i = 0; i < messagesWithIndex.length; i += batchSize) {
 				const batch = messagesWithIndex.slice(i, i + batchSize);
-				const pipeline = this.client.pipeline();
+				const pipeline = writeBatch(this.client, check);
 				const batchTouchedThreadIds = /* @__PURE__ */ new Set();
 				const batchExistingThreadIds = existingThreadIds.slice(i, i + batch.length);
 				for (const [batchIndex, message] of batch.entries()) {
@@ -930,6 +1206,9 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 		}
 	}
 	async saveResource({ resource }) {
+		return this.#saveResource(resource, void 0);
+	}
+	async #saveResource(resource, check) {
 		try {
 			const key = `${TABLE_RESOURCES}:${resource.id}`;
 			const serializedResource = {
@@ -938,14 +1217,17 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 				createdAt: resource.createdAt.toISOString(),
 				updatedAt: resource.updatedAt.toISOString()
 			};
-			await this.client.set(key, serializedResource);
+			const batch = writeBatch(this.client, check);
+			batch.set(key, serializedResource);
+			await batch.exec();
 			return resource;
 		} catch (error) {
-			this.logger.error("Error saving resource:", error);
+			if (!isRunFenceConflictError(error)) this.logger.error("Error saving resource:", error);
 			throw error;
 		}
 	}
-	async updateResource({ resourceId, workingMemory, metadata }) {
+	async updateResource({ resourceId, workingMemory, metadata, fence }) {
+		const check = this.#runFenceCheck(fence, "updateResource");
 		try {
 			const existingResource = await this.getResourceById({ resourceId });
 			if (!existingResource) {
@@ -956,7 +1238,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 					createdAt: /* @__PURE__ */ new Date(),
 					updatedAt: /* @__PURE__ */ new Date()
 				};
-				return this.saveResource({ resource: newResource });
+				return await this.#saveResource(newResource, check);
 			}
 			const updatedResource = {
 				...existingResource,
@@ -967,16 +1249,17 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 				},
 				updatedAt: /* @__PURE__ */ new Date()
 			};
-			await this.saveResource({ resource: updatedResource });
+			await this.#saveResource(updatedResource, check);
 			return updatedResource;
 		} catch (error) {
-			this.logger.error("Error updating resource:", error);
+			if (!isRunFenceConflictError(error)) this.logger.error("Error updating resource:", error);
 			throw error;
 		}
 	}
 	async updateMessages(args) {
 		const { messages } = args;
 		if (messages.length === 0) return [];
+		const check = this.#runFenceCheck(args.fence, "updateMessages");
 		try {
 			const messageIds = messages.map((m) => m.id);
 			const updatesById = new Map(messages.map((message) => [message.id, message]));
@@ -1029,7 +1312,10 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 				for (const [messageId, threadId] of Object.entries(backfillIndexValues)) backfillPipeline.set(getMessageIndexKey(messageId), threadId);
 				await backfillPipeline.exec();
 			}
-			if (existingMessages.length === 0) return [];
+			if (existingMessages.length === 0) {
+				await assertRunFence(this.client, check);
+				return [];
+			}
 			const threadIdsToUpdate = /* @__PURE__ */ new Set();
 			const destinationThreadIds = /* @__PURE__ */ new Set();
 			for (const existingMessage of existingMessages) {
@@ -1061,7 +1347,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 				domain: ErrorDomain.STORAGE,
 				category: ErrorCategory.USER
 			}, /* @__PURE__ */ new Error(`Thread ${destinationThreadId} not found`));
-			const pipeline = this.client.pipeline();
+			const pipeline = writeBatch(this.client, check);
 			for (const existingMessage of existingMessages) {
 				const updatePayload = updatesById.get(existingMessage.id);
 				if (!updatePayload) continue;
@@ -1131,8 +1417,9 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 			}, error);
 		}
 	}
-	async deleteMessages(messageIds) {
+	async deleteMessages(messageIds, options) {
 		if (!messageIds || messageIds.length === 0) return;
+		const check = this.#runFenceCheck(options?.fence, "deleteMessages");
 		try {
 			const threadIds = /* @__PURE__ */ new Set();
 			const messageKeys = [];
@@ -1168,8 +1455,11 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 					}
 				}
 			}
-			if (messageKeys.length === 0) return;
-			const pipeline = this.client.pipeline();
+			if (messageKeys.length === 0) {
+				await assertRunFence(this.client, check);
+				return;
+			}
+			const pipeline = writeBatch(this.client, check);
 			for (const key of messageKeys) pipeline.del(key);
 			for (const messageId of foundMessageIds) pipeline.del(getMessageIndexKey(messageId));
 			if (threadIds.size > 0) for (const threadId of threadIds) {
@@ -1185,6 +1475,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
 			}
 			await pipeline.exec();
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("UPSTASH", "DELETE_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -1612,8 +1903,56 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 			resourceId: row.resourceId
 		};
 	}
+	supportsRunFencing() {
+		return true;
+	}
 	async dangerouslyClearAll() {
 		await this.#db.deleteData({ tableName: TABLE_WORKFLOW_SNAPSHOT });
+		await this.#db.scanAndDelete(`${runClaimKey(TABLE_WORKFLOW_RUN_OWNERS, "")}*`);
+	}
+	#runFenceCheck(fence, runId, operation) {
+		const resolved = resolveRunFence(this, fence, runId);
+		return resolved && {
+			claims: TABLE_WORKFLOW_RUN_OWNERS,
+			fence: resolved,
+			operation
+		};
+	}
+	#ownershipError(operation, runId, error) {
+		return new MastraError({
+			id: createStorageErrorId("UPSTASH", operation, "FAILED"),
+			domain: ErrorDomain.STORAGE,
+			category: ErrorCategory.THIRD_PARTY,
+			details: { runId }
+		}, error);
+	}
+	async claimRunOwnership(args) {
+		try {
+			return await claimRunOwnership(this.client, args);
+		} catch (error) {
+			throw this.#ownershipError("CLAIM_RUN_OWNERSHIP", args.runId, error);
+		}
+	}
+	async renewRunOwnership(args) {
+		try {
+			return await renewRunOwnership(this.client, args);
+		} catch (error) {
+			throw this.#ownershipError("RENEW_RUN_OWNERSHIP", args.runId, error);
+		}
+	}
+	async releaseRunOwnership(fence) {
+		try {
+			return await releaseRunOwnership(this.client, fence);
+		} catch (error) {
+			throw this.#ownershipError("RELEASE_RUN_OWNERSHIP", fence.runId, error);
+		}
+	}
+	async getRunOwnership({ runId }) {
+		try {
+			return await getRunOwnership(this.client, runId);
+		} catch (error) {
+			throw this.#ownershipError("GET_RUN_OWNERSHIP", runId, error);
+		}
 	}
 	async getWorkflowRunRecord({ namespace, runId, resourceId, workflowName }) {
 		if (workflowName) {
@@ -1641,7 +1980,8 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 			return this.client.get(key);
 		}))).find((w) => w?.run_id === runId && (!workflowName || w.workflow_name === workflowName) && (!resourceId || w.resourceId === resourceId)) ?? null;
 	}
-	async updateWorkflowResults({ workflowName, runId, stepId, result, requestContext, state }) {
+	async updateWorkflowResults({ workflowName, runId, stepId, result, requestContext, state, fence }) {
+		const check = this.#runFenceCheck(fence, runId, "updateWorkflowResults");
 		try {
 			const key = getKey(TABLE_WORKFLOW_SNAPSHOT, {
 				namespace: "workflows",
@@ -1649,7 +1989,7 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 				run_id: runId
 			});
 			const now = (/* @__PURE__ */ new Date()).toISOString();
-			const resultJson = await this.client.eval(`
+			const resultJson = await evalFenced(this.client, `
         local key = KEYS[1]
         local stepId = ARGV[1]
         local resultJson = ARGV[2]
@@ -1739,7 +2079,7 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 				runId,
 				String(Date.now()),
 				state === void 0 ? "" : JSON.stringify(state)
-			]);
+			], check);
 			let data;
 			if (typeof resultJson === "string") data = JSON.parse(resultJson);
 			else data = resultJson;
@@ -1758,7 +2098,8 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 			}, error);
 		}
 	}
-	async updateWorkflowState({ workflowName, runId, opts }) {
+	async updateWorkflowState({ workflowName, runId, opts, fence }) {
+		const check = this.#runFenceCheck(fence, runId, "updateWorkflowState");
 		try {
 			const key = getKey(TABLE_WORKFLOW_SNAPSHOT, {
 				namespace: "workflows",
@@ -1823,11 +2164,11 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
       `;
 			const { expectedStatus, ...state } = opts;
 			const expectedStatusJson = expectedStatus === void 0 ? "" : JSON.stringify(Array.isArray(expectedStatus) ? expectedStatus : [expectedStatus]);
-			const resultJson = await this.client.eval(luaScript, [key], [
+			const resultJson = await evalFenced(this.client, luaScript, [key], [
 				JSON.stringify(state),
 				now,
 				expectedStatusJson
-			]);
+			], check);
 			if (!resultJson) return;
 			let data;
 			if (typeof resultJson === "string") data = JSON.parse(resultJson);
@@ -1848,6 +2189,7 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 	}
 	async persistWorkflowSnapshot(params) {
 		const { namespace = "workflows", workflowName, runId, resourceId, snapshot, createdAt, updatedAt } = params;
+		const check = this.#runFenceCheck(params.fence, runId, "persistWorkflowSnapshot");
 		try {
 			const now = /* @__PURE__ */ new Date();
 			const key = getKey(TABLE_WORKFLOW_SNAPSHOT, {
@@ -1865,7 +2207,7 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 				createdAt: (createdAt ?? now).toISOString(),
 				updatedAt: (updatedAt ?? now).toISOString()
 			};
-			await this.client.eval(`
+			await evalFenced(this.client, `
         local existing = redis.call("GET", KEYS[1])
         local next = cjson.decode(ARGV[1])
         if existing then
@@ -1876,8 +2218,9 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
         end
         redis.call("SET", KEYS[1], cjson.encode(next))
         return 1
-        `, [key], [JSON.stringify(record)]);
+        `, [key], [JSON.stringify(record)], check);
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("UPSTASH", "PERSIST_WORKFLOW_SNAPSHOT", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -1935,7 +2278,8 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 			}, error);
 		}
 	}
-	async deleteWorkflowRunById({ runId, workflowName }) {
+	async deleteWorkflowRunById({ runId, workflowName, fence }) {
+		const check = this.#runFenceCheck(fence, runId, "deleteWorkflowRunById");
 		try {
 			const record = await this.getWorkflowRunRecord({
 				namespace: "workflows",
@@ -1948,8 +2292,11 @@ var WorkflowsUpstash = class extends WorkflowsStorage {
 				run_id: runId,
 				...record?.resourceId ? { resourceId: record.resourceId } : {}
 			});
-			await this.client.del(key);
+			const batch = writeBatch(this.client, check);
+			batch.del(key);
+			await batch.exec();
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("UPSTASH", "DELETE_WORKFLOW_RUN_BY_ID", "FAILED"),
 				domain: ErrorDomain.STORAGE,