@mastra/spanner 1.9.3 → 1.9.4

package/dist/index.js

@@ -1,6 +1,6 @@
 import { Spanner } from "@google-cloud/spanner";
 import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
-import { AGENTS_SCHEMA, AGENT_VERSIONS_SCHEMA, AgentsStorage, BackgroundTasksStorage, BlobStore, ChannelsStorage, DatasetsStorage, ExperimentsStorage, FavoritesStorage, MCPClientsStorage, MCPServersStorage, MCP_CLIENTS_SCHEMA, MCP_CLIENT_VERSIONS_SCHEMA, MCP_SERVERS_SCHEMA, MCP_SERVER_VERSIONS_SCHEMA, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, PROMPT_BLOCKS_SCHEMA, PROMPT_BLOCK_VERSIONS_SCHEMA, PromptBlocksStorage, SCORER_DEFINITIONS_SCHEMA, SCORER_DEFINITION_VERSIONS_SCHEMA, SKILLS_SCHEMA, SKILL_BLOBS_SCHEMA, SKILL_VERSIONS_SCHEMA, SchedulesStorage, ScorerDefinitionsStorage, ScoresStorage, SkillsStorage, TABLE_AGENTS, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_CHANNEL_CONFIG, TABLE_CHANNEL_INSTALLATIONS, TABLE_CONFIGS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WORKFLOW_DEFINITIONS_SCHEMA, WORKSPACES_SCHEMA, WORKSPACE_VERSIONS_SCHEMA, WorkflowDefinitionsStorage, WorkflowsStorage, WorkspacesStorage, calculatePagination, createStorageErrorId, executeRetentionPrune, getDefaultValue, hasErrorCode, listMetricsArgsSchema, listTracesArgsSchema, matchesExpectedWorkflowStatus, normalizePerPage, normalizeScheduleTarget, resolveListOrderBy, resolveRetentionTargets, retentionCutoffMs, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
+import { AGENTS_SCHEMA, AGENT_VERSIONS_SCHEMA, AgentsStorage, BackgroundTasksStorage, BlobStore, ChannelsStorage, DatasetsStorage, ExperimentsStorage, FavoritesStorage, MCPClientsStorage, MCPServersStorage, MCP_CLIENTS_SCHEMA, MCP_CLIENT_VERSIONS_SCHEMA, MCP_SERVERS_SCHEMA, MCP_SERVER_VERSIONS_SCHEMA, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, PROMPT_BLOCKS_SCHEMA, PROMPT_BLOCK_VERSIONS_SCHEMA, PromptBlocksStorage, RUN_FENCING_TABLE_SCHEMAS, RunFenceConflictError, SCORER_DEFINITIONS_SCHEMA, SCORER_DEFINITION_VERSIONS_SCHEMA, SKILLS_SCHEMA, SKILL_BLOBS_SCHEMA, SKILL_VERSIONS_SCHEMA, SchedulesStorage, ScorerDefinitionsStorage, ScoresStorage, SkillsStorage, TABLE_AGENTS, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_CHANNEL_CONFIG, TABLE_CHANNEL_INSTALLATIONS, TABLE_CONFIGS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MEMORY_RUN_FENCES, TABLE_MESSAGES, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_RUN_OWNERS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WORKFLOW_DEFINITIONS_SCHEMA, WORKSPACES_SCHEMA, WORKSPACE_VERSIONS_SCHEMA, WorkflowDefinitionsStorage, WorkflowsStorage, WorkspacesStorage, calculatePagination, createStorageErrorId, executeRetentionPrune, getDefaultValue, hasErrorCode, isRunFenceConflictError, listMetricsArgsSchema, listTracesArgsSchema, matchesExpectedWorkflowStatus, matchesRunFence, normalizePerPage, normalizeScheduleTarget, resolveListOrderBy, resolveRetentionTargets, resolveRunFence, retentionCutoffMs, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
 import { MastraBase } from "@mastra/core/base";
 import { parseSqlIdentifier } from "@mastra/core/utils";
 import { MessageList } from "@mastra/core/agent";
@@ -6656,6 +6656,59 @@ var MCPServersSpanner = class MCPServersSpanner extends MCPServersStorage {
 	}
 };
 //#endregion
+//#region src/storage/db/run-fencing.ts
+/** Epoch milliseconds on the database clock, read when the statement runs. */
+const DB_NOW_MS = "UNIX_MILLIS(CURRENT_TIMESTAMP())";
+/**
+* The check half of a fenced write; call it first inside the write's
+* read-write transaction. Reads in a read-write transaction lock the rows they
+* return until commit, so a claim that updates the row either waits for the
+* write to commit or aborts it, and the retried write sees the new claim.
+*/
+async function assertRunFence(tx, claimsTable, fence, operation) {
+	const [rows] = await tx.run({
+		sql: `SELECT generation, ${quoteIdent("ownerId", "column name")} AS ownerId
+          FROM ${quoteIdent(claimsTable, "table name")}
+          WHERE ${quoteIdent("runId", "column name")} = @runId`,
+		params: { runId: fence.runId },
+		json: true
+	});
+	const row = rows[0];
+	if (!matchesRunFence(row ? {
+		generation: Number(row.generation),
+		ownerId: String(row.ownerId)
+	} : null, fence)) throw new RunFenceConflictError(fence, operation);
+}
+/**
+* Runs `fn` in a read-write transaction and commits it, retrying the whole
+* transaction when Spanner aborts it for conflicting with another one.
+*/
+function inTransaction(db, fn) {
+	return db.runWithAbortRetry(() => db.database.runTransactionAsync(async (tx) => {
+		try {
+			const result = await fn(tx);
+			await tx.commit();
+			return result;
+		} catch (err) {
+			await tx.rollback().catch((rollbackErr) => {
+				throw new AggregateError([err, rollbackErr], "Transaction and rollback both failed");
+			});
+			throw err;
+		}
+	}));
+}
+/**
+* Runs `write` directly when it carries no fence, or in a transaction behind
+* the fence check when it does.
+*/
+function withRunFence(db, claimsTable, fence, operation, write) {
+	if (!fence) return write(void 0);
+	return inTransaction(db, async (tx) => {
+		await assertRunFence(tx, claimsTable, fence, operation);
+		return write(tx);
+	});
+}
+//#endregion
 //#region src/storage/domains/memory/index.ts
 function buildSpannerMessageMetadataFilter(metadataFilter) {
 	if (!metadataFilter) return {
@@ -6719,7 +6772,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		this.skipDefaultIndexes = skipDefaultIndexes;
 		this.indexes = indexes?.filter((idx) => MemorySpanner.MANAGED_TABLES.includes(idx.table));
 	}
-	/** Creates the threads/messages/resources tables and any indexes. */
+	/** Creates the threads/messages/resources and run fence tables and any indexes. */
 	async init() {
 		await this.db.createTable({
 			tableName: TABLE_THREADS,
@@ -6733,6 +6786,10 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 			tableName: TABLE_RESOURCES,
 			schema: TABLE_SCHEMAS[TABLE_RESOURCES]
 		});
+		await this.db.createTable({
+			tableName: TABLE_MEMORY_RUN_FENCES,
+			schema: RUN_FENCING_TABLE_SCHEMAS[TABLE_MEMORY_RUN_FENCES]
+		});
 		await this.createDefaultIndexes();
 		await this.createCustomIndexes();
 	}
@@ -6758,11 +6815,92 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		if (!this.indexes || this.indexes.length === 0) return;
 		await this.db.createIndexes(this.indexes);
 	}
-	/** Removes every row from threads/messages/resources tables. Intended for tests. */
+	/** Removes every row from threads/messages/resources and run fence tables. Intended for tests. */
 	async dangerouslyClearAll() {
 		await this.db.clearTable({ tableName: TABLE_MESSAGES });
 		await this.db.clearTable({ tableName: TABLE_THREADS });
 		await this.db.clearTable({ tableName: TABLE_RESOURCES });
+		await this.db.clearTable({ tableName: TABLE_MEMORY_RUN_FENCES });
+	}
+	supportsRunFencing() {
+		return true;
+	}
+	async raiseRunFence(fence) {
+		const table = quoteIdent(TABLE_MEMORY_RUN_FENCES, "table name");
+		const runIdCol = quoteIdent("runId", "column name");
+		const ownerIdCol = quoteIdent("ownerId", "column name");
+		const retiredAtCol = quoteIdent("retiredAt", "column name");
+		try {
+			return await inTransaction(this.db, async (tx) => {
+				const [rows] = await tx.run({
+					sql: `SELECT generation, ${ownerIdCol} AS ownerId FROM ${table} WHERE ${runIdCol} = @runId`,
+					params: { runId: fence.runId },
+					json: true
+				});
+				const row = rows[0];
+				if (!row) {
+					await tx.runUpdate({
+						sql: `INSERT INTO ${table} (${runIdCol}, generation, ${ownerIdCol}) VALUES (@runId, @generation, @ownerId)`,
+						params: {
+							runId: fence.runId,
+							generation: fence.generation,
+							ownerId: fence.ownerId
+						}
+					});
+					return true;
+				}
+				const generation = Number(row.generation);
+				const raises = generation < fence.generation;
+				const reaffirms = generation === fence.generation && String(row.ownerId) === fence.ownerId;
+				if (!raises && !reaffirms) return false;
+				await tx.runUpdate({
+					sql: `UPDATE ${table} SET generation = @generation, ${ownerIdCol} = @ownerId, ${retiredAtCol} = NULL
+                WHERE ${runIdCol} = @runId`,
+					params: {
+						runId: fence.runId,
+						generation: fence.generation,
+						ownerId: fence.ownerId
+					}
+				});
+				return true;
+			});
+		} catch (error) {
+			throw new MastraError({
+				id: createStorageErrorId("SPANNER", "RAISE_RUN_FENCE", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { runId: fence.runId }
+			}, error);
+		}
+	}
+	async retireRunFence(fence) {
+		const table = quoteIdent(TABLE_MEMORY_RUN_FENCES, "table name");
+		try {
+			return await inTransaction(this.db, async (tx) => {
+				const [count] = await tx.runUpdate({
+					sql: `UPDATE ${table} SET ${quoteIdent("retiredAt", "column name")} = ${DB_NOW_MS}
+                WHERE ${quoteIdent("runId", "column name")} = @runId AND generation = @generation
+                  AND ${quoteIdent("ownerId", "column name")} = @ownerId`,
+					params: {
+						runId: fence.runId,
+						generation: fence.generation,
+						ownerId: fence.ownerId
+					}
+				});
+				return count === 1;
+			});
+		} catch (error) {
+			throw new MastraError({
+				id: createStorageErrorId("SPANNER", "RETIRE_RUN_FENCE", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { runId: fence.runId }
+			}, error);
+		}
+	}
+	/** The fence check for a write already running in a read-write transaction. */
+	async #assertFence(tx, fence, operation) {
+		if (fence) await assertRunFence(tx, TABLE_MEMORY_RUN_FENCES, fence, operation);
 	}
 	/**
 	* Fetches a thread row by id, or `null` when absent.
@@ -6931,9 +7069,9 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		}
 	}
 	/** Upserts a thread row by id (`INSERT OR UPDATE` semantics). */
-	async saveThread({ thread }) {
+	async saveThread({ thread, fence }) {
 		try {
-			await this.db.upsert({
+			await withRunFence(this.db, TABLE_MEMORY_RUN_FENCES, resolveRunFence(this, fence), "saveThread", (tx) => this.db.upsert({
 				tableName: TABLE_THREADS,
 				record: {
 					id: thread.id,
@@ -6942,10 +7080,12 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 					metadata: thread.metadata ?? {},
 					createdAt: thread.createdAt,
 					updatedAt: thread.updatedAt
-				}
-			});
+				},
+				transaction: tx
+			}));
 			return thread;
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("SPANNER", "SAVE_THREAD", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -6955,7 +7095,8 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		}
 	}
 	/** Updates a thread's title and merges the metadata payload. */
-	async updateThread({ id, title, metadata }) {
+	async updateThread({ id, title, metadata, fence: explicitFence }) {
+		const fence = resolveRunFence(this, explicitFence);
 		const tableThreads = quoteIdent(TABLE_THREADS, "table name");
 		const now = /* @__PURE__ */ new Date();
 		let merged = {};
@@ -6963,6 +7104,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		try {
 			await this.db.runWithAbortRetry(() => this.database.runTransactionAsync(async (tx) => {
 				try {
+					await this.#assertFence(tx, fence, "updateThread");
 					const [rows] = await tx.run({
 						sql: `SELECT * FROM ${tableThreads} WHERE id = @id LIMIT 1`,
 						params: { id },
@@ -7429,8 +7571,9 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		}
 	}
 	/** Upserts a batch of messages and bumps every touched thread's `updatedAt` in a single transaction. */
-	async saveMessages({ messages }) {
+	async saveMessages({ messages, fence: explicitFence }) {
 		if (messages.length === 0) return { messages: [] };
+		const fence = resolveRunFence(this, explicitFence);
 		const threadIds = /* @__PURE__ */ new Set();
 		for (const message of messages) {
 			if (!message.threadId) throw new MastraError({
@@ -7456,6 +7599,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		try {
 			await this.db.runWithAbortRetry(() => this.database.runTransactionAsync(async (tx) => {
 				try {
+					await this.#assertFence(tx, fence, "saveMessages");
 					for (const message of messages) {
 						if (!message.resourceId) throw new Error("Expected to find a resourceId for message");
 						const sql = `INSERT OR UPDATE INTO ${tableMessages} (id, ${quoteIdent("thread_id", "column name")}, content, ${quoteIdent("createdAt", "column name")}, role, type, ${quoteIdent("resourceId", "column name")})
@@ -7493,6 +7637,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 			}));
 			return { messages: new MessageList().add(messages, "memory").get.all.db() };
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("SPANNER", "SAVE_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -7502,8 +7647,9 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		}
 	}
 	/** Merges partial updates onto existing messages and bumps affected thread `updatedAt`. */
-	async updateMessages({ messages }) {
+	async updateMessages({ messages, fence: explicitFence }) {
 		if (!messages || messages.length === 0) return [];
+		const fence = resolveRunFence(this, explicitFence);
 		const messageIds = messages.map((m) => m.id);
 		const params = {};
 		const placeholders = messageIds.map((id, i) => {
@@ -7521,6 +7667,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		try {
 			await this.db.runWithAbortRetry(() => this.database.runTransactionAsync(async (tx) => {
 				try {
+					await this.#assertFence(tx, fence, "updateMessages");
 					const [existingRows] = await tx.run({
 						sql: selectSql,
 						params,
@@ -7596,6 +7743,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 				}
 			}));
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("SPANNER", "UPDATE_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -7619,8 +7767,9 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		});
 	}
 	/** Deletes a batch of messages and bumps the affected threads' `updatedAt`. */
-	async deleteMessages(messageIds) {
+	async deleteMessages(messageIds, options) {
 		if (!messageIds || messageIds.length === 0) return;
+		const fence = resolveRunFence(this, options?.fence);
 		try {
 			const messageTableName = quoteIdent(TABLE_MESSAGES, "table name");
 			const threadTableName = quoteIdent(TABLE_THREADS, "table name");
@@ -7638,6 +7787,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 			const threadIds = threadRows.map((r) => r.thread_id).filter(Boolean);
 			await this.db.runWithAbortRetry(() => this.database.runTransactionAsync(async (tx) => {
 				try {
+					await this.#assertFence(tx, fence, "deleteMessages");
 					await tx.runUpdate({
 						sql: `DELETE FROM ${messageTableName} WHERE id IN (${placeholders})`,
 						params
@@ -7655,6 +7805,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 				}
 			}));
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("SPANNER", "DELETE_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -7713,13 +7864,15 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 		return resource;
 	}
 	/** Updates a resource's `workingMemory` and merges its `metadata`. */
-	async updateResource({ resourceId, workingMemory, metadata }) {
+	async updateResource({ resourceId, workingMemory, metadata, fence: explicitFence }) {
+		const fence = resolveRunFence(this, explicitFence);
 		const tableResources = quoteIdent(TABLE_RESOURCES, "table name");
 		const now = /* @__PURE__ */ new Date();
 		let updated = null;
 		try {
 			await this.db.runWithAbortRetry(() => this.database.runTransactionAsync(async (tx) => {
 				try {
+					await this.#assertFence(tx, fence, "updateResource");
 					const [rows] = await tx.run({
 						sql: `SELECT * FROM ${tableResources} WHERE id = @id LIMIT 1`,
 						params: { id: resourceId },
@@ -7783,6 +7936,7 @@ var MemorySpanner = class MemorySpanner extends MemoryStorage {
 			}));
 			return updated;
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			const mastraError = new MastraError({
 				id: createStorageErrorId("SPANNER", "UPDATE_RESOURCE", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -12290,6 +12444,30 @@ var WorkflowDefinitionsSpanner = class WorkflowDefinitionsSpanner extends Workfl
 };
 //#endregion
 //#region src/storage/domains/workflows/index.ts
+const RUN_OWNERS_TABLE = quoteIdent(TABLE_WORKFLOW_RUN_OWNERS, "table name");
+const RUN_ID = quoteIdent("runId", "column name");
+const OWNER_ID = quoteIdent("ownerId", "column name");
+const LEASE_EXPIRES_AT = quoteIdent("leaseExpiresAt", "column name");
+const UPDATED_AT = quoteIdent("updatedAt", "column name");
+/** Reads a run's ownership record, judging the lease against the database clock. */
+async function readRunOwner(q, runId) {
+	const [rows] = await q.run({
+		sql: `SELECT generation, ${OWNER_ID} AS ownerId, ${LEASE_EXPIRES_AT} AS leaseExpiresAt, ${DB_NOW_MS} AS nowMs
+          FROM ${RUN_OWNERS_TABLE} WHERE ${RUN_ID} = @runId`,
+		params: { runId },
+		json: true
+	});
+	const row = rows[0];
+	if (!row) return null;
+	const leaseExpiresAt = row.leaseExpiresAt === null ? null : Number(row.leaseExpiresAt);
+	return {
+		runId,
+		generation: Number(row.generation),
+		ownerId: String(row.ownerId),
+		leaseExpiresAt: leaseExpiresAt === null ? null : new Date(leaseExpiresAt),
+		live: leaseExpiresAt !== null && leaseExpiresAt > Number(row.nowMs)
+	};
+}
 /**
 * Spanner-backed storage for workflow run snapshots, including persistence,
 * incremental updates, and listing of historical runs.
@@ -12370,12 +12548,19 @@ var WorkflowsSpanner = class WorkflowsSpanner extends WorkflowsStorage {
 		const indexes = this.getDefaultIndexDefinitions().filter((idx) => hasStatus || !idx.columns.some((c) => c.startsWith("snapshotStatus")));
 		await this.db.createIndexes(indexes);
 	}
-	/** Creates the workflow snapshot table, the snapshotStatus generated column, and indexes. */
+	/**
+	* Creates the workflow snapshot and run ownership tables, the snapshotStatus
+	* generated column, and indexes.
+	*/
 	async init() {
 		await this.db.createTable({
 			tableName: TABLE_WORKFLOW_SNAPSHOT,
 			schema: TABLE_SCHEMAS[TABLE_WORKFLOW_SNAPSHOT]
 		});
+		await this.db.createTable({
+			tableName: TABLE_WORKFLOW_RUN_OWNERS,
+			schema: RUN_FENCING_TABLE_SCHEMAS[TABLE_WORKFLOW_RUN_OWNERS]
+		});
 		await this.ensureStatusColumn();
 		await this.createDefaultIndexes();
 		await this.createCustomIndexes();
@@ -12424,9 +12609,121 @@ var WorkflowsSpanner = class WorkflowsSpanner extends WorkflowsStorage {
 		if (!this.indexes || this.indexes.length === 0) return;
 		await this.db.createIndexes(this.indexes);
 	}
-	/** Removes every workflow snapshot row. Intended for tests. */
+	/** Removes every workflow snapshot and run ownership row. Intended for tests. */
 	async dangerouslyClearAll() {
 		await this.db.clearTable({ tableName: TABLE_WORKFLOW_SNAPSHOT });
+		await this.db.clearTable({ tableName: TABLE_WORKFLOW_RUN_OWNERS });
+	}
+	supportsRunFencing() {
+		return true;
+	}
+	#ownershipError(operation, runId, error) {
+		return new MastraError({
+			id: createStorageErrorId("SPANNER", operation, "FAILED"),
+			domain: ErrorDomain.STORAGE,
+			category: ErrorCategory.THIRD_PARTY,
+			details: { runId }
+		}, error);
+	}
+	async claimRunOwnership({ runId, ownerId, leaseMs, force, expectedGeneration }) {
+		try {
+			return await inTransaction(this.db, async (tx) => {
+				const existing = await readRunOwner(tx, runId);
+				if (!existing) {
+					if (expectedGeneration !== void 0 && expectedGeneration !== 0) return {
+						acquired: false,
+						record: null
+					};
+					await tx.runUpdate({
+						sql: `INSERT INTO ${RUN_OWNERS_TABLE} (${RUN_ID}, generation, ${OWNER_ID}, ${LEASE_EXPIRES_AT}, ${UPDATED_AT})
+                  VALUES (@runId, ${DB_NOW_MS}, @ownerId, ${DB_NOW_MS} + @leaseMs, ${DB_NOW_MS})`,
+						params: {
+							runId,
+							ownerId,
+							leaseMs
+						}
+					});
+					return {
+						acquired: true,
+						record: await readRunOwner(tx, runId)
+					};
+				}
+				if (!(expectedGeneration === void 0 || existing.generation === expectedGeneration) || !force && existing.live) return {
+					acquired: false,
+					record: existing
+				};
+				await tx.runUpdate({
+					sql: `UPDATE ${RUN_OWNERS_TABLE}
+                SET generation = generation + 1, ${OWNER_ID} = @ownerId, ${LEASE_EXPIRES_AT} = ${DB_NOW_MS} + @leaseMs,
+                    ${UPDATED_AT} = ${DB_NOW_MS}
+                WHERE ${RUN_ID} = @runId`,
+					params: {
+						runId,
+						ownerId,
+						leaseMs
+					}
+				});
+				return {
+					acquired: true,
+					record: await readRunOwner(tx, runId)
+				};
+			});
+		} catch (error) {
+			throw this.#ownershipError("CLAIM_RUN_OWNERSHIP", runId, error);
+		}
+	}
+	async renewRunOwnership({ leaseMs, ...fence }) {
+		try {
+			return await inTransaction(this.db, async (tx) => {
+				const [count] = await tx.runUpdate({
+					sql: `UPDATE ${RUN_OWNERS_TABLE}
+                SET ${LEASE_EXPIRES_AT} = ${DB_NOW_MS} + @leaseMs, ${UPDATED_AT} = ${DB_NOW_MS}
+                WHERE ${RUN_ID} = @runId AND generation = @generation AND ${OWNER_ID} = @ownerId
+                  AND ${LEASE_EXPIRES_AT} IS NOT NULL`,
+					params: {
+						leaseMs,
+						runId: fence.runId,
+						generation: fence.generation,
+						ownerId: fence.ownerId
+					}
+				});
+				const record = await readRunOwner(tx, fence.runId);
+				return count === 1 && record ? {
+					renewed: true,
+					record
+				} : {
+					renewed: false,
+					record
+				};
+			});
+		} catch (error) {
+			throw this.#ownershipError("RENEW_RUN_OWNERSHIP", fence.runId, error);
+		}
+	}
+	async releaseRunOwnership(fence) {
+		try {
+			return await inTransaction(this.db, async (tx) => {
+				const [count] = await tx.runUpdate({
+					sql: `UPDATE ${RUN_OWNERS_TABLE} SET ${LEASE_EXPIRES_AT} = NULL, ${UPDATED_AT} = ${DB_NOW_MS}
+                WHERE ${RUN_ID} = @runId AND generation = @generation AND ${OWNER_ID} = @ownerId`,
+					params: {
+						runId: fence.runId,
+						generation: fence.generation,
+						ownerId: fence.ownerId
+					}
+				});
+				return count === 1;
+			});
+		} catch (error) {
+			throw this.#ownershipError("RELEASE_RUN_OWNERSHIP", fence.runId, error);
+		}
+	}
+	async getRunOwnership({ runId }) {
+		try {
+			return await readRunOwner(this.database, runId);
+		} catch (error) {
+			throw this.#ownershipError("GET_RUN_OWNERSHIP", runId, error);
+		}
 	}
 	/** Decodes a raw Spanner snapshot row into the public `WorkflowRun` shape. */
 	parseWorkflowRun(row) {
@@ -12450,12 +12747,14 @@ var WorkflowsSpanner = class WorkflowsSpanner extends WorkflowsStorage {
 		};
 	}
 	/** Upserts a workflow run snapshot, preserving the original `createdAt` on update. */
-	async persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot, createdAt, updatedAt }) {
+	async persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot, createdAt, updatedAt, fence: explicitFence }) {
+		const fence = resolveRunFence(this, explicitFence, runId);
 		const now = /* @__PURE__ */ new Date();
 		const table = quoteIdent(TABLE_WORKFLOW_SNAPSHOT, "table name");
 		try {
 			await this.db.runWithAbortRetry(() => this.database.runTransactionAsync(async (tx) => {
 				try {
+					if (fence) await assertRunFence(tx, TABLE_WORKFLOW_RUN_OWNERS, fence, "persistWorkflowSnapshot");
 					const [rows] = await tx.run({
 						sql: `SELECT ${quoteIdent("createdAt", "column name")}, ${quoteIdent("resourceId", "column name")} FROM ${table}
                     WHERE workflow_name = @workflow_name AND run_id = @run_id`,
@@ -12489,6 +12788,7 @@ var WorkflowsSpanner = class WorkflowsSpanner extends WorkflowsStorage {
 				}
 			}));
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("SPANNER", "PERSIST_WORKFLOW_SNAPSHOT", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -12525,12 +12825,14 @@ var WorkflowsSpanner = class WorkflowsSpanner extends WorkflowsStorage {
 		}
 	}
 	/** Merges a step result and request-context delta into a snapshot inside an RW transaction. */
-	async updateWorkflowResults({ workflowName, runId, stepId, result, requestContext, state }) {
+	async updateWorkflowResults({ workflowName, runId, stepId, result, requestContext, state, fence: explicitFence }) {
+		const fence = resolveRunFence(this, explicitFence, runId);
 		const table = quoteIdent(TABLE_WORKFLOW_SNAPSHOT, "table name");
 		try {
 			let mergedContext = {};
 			await this.db.runWithAbortRetry(() => this.database.runTransactionAsync(async (tx) => {
 				try {
+					if (fence) await assertRunFence(tx, TABLE_WORKFLOW_RUN_OWNERS, fence, "updateWorkflowResults");
 					const [rows] = await tx.run({
 						sql: `SELECT snapshot, ${quoteIdent("createdAt", "column name")} FROM ${table}
                     WHERE workflow_name = @workflow_name AND run_id = @run_id`,
@@ -12590,6 +12892,7 @@ var WorkflowsSpanner = class WorkflowsSpanner extends WorkflowsStorage {
 			}));
 			return mergedContext;
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("SPANNER", "UPDATE_WORKFLOW_RESULTS", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -12603,13 +12906,15 @@ var WorkflowsSpanner = class WorkflowsSpanner extends WorkflowsStorage {
 		}
 	}
 	/** Applies a top-level snapshot patch (e.g. status, value) inside an RW transaction. */
-	async updateWorkflowState({ workflowName, runId, opts }) {
+	async updateWorkflowState({ workflowName, runId, opts, fence: explicitFence }) {
+		const fence = resolveRunFence(this, explicitFence, runId);
 		const table = quoteIdent(TABLE_WORKFLOW_SNAPSHOT, "table name");
 		try {
 			let updated;
 			await this.db.runWithAbortRetry(() => this.database.runTransactionAsync(async (tx) => {
 				updated = void 0;
 				try {
+					if (fence) await assertRunFence(tx, TABLE_WORKFLOW_RUN_OWNERS, fence, "updateWorkflowState");
 					const [rows] = await tx.run({
 						sql: `SELECT snapshot FROM ${table} WHERE workflow_name = @workflow_name AND run_id = @run_id`,
 						params: {
@@ -12720,17 +13025,18 @@ var WorkflowsSpanner = class WorkflowsSpanner extends WorkflowsStorage {
 		}
 	}
 	/** Deletes the snapshot row for a `(workflowName, runId)` pair. */
-	async deleteWorkflowRunById({ runId, workflowName }) {
+	async deleteWorkflowRunById({ runId, workflowName, fence }) {
 		const table = quoteIdent(TABLE_WORKFLOW_SNAPSHOT, "table name");
 		try {
-			await this.db.runDml({
+			await withRunFence(this.db, TABLE_WORKFLOW_RUN_OWNERS, resolveRunFence(this, fence, runId), "deleteWorkflowRunById", (tx) => this.db.runDml({
 				sql: `DELETE FROM ${table} WHERE workflow_name = @workflow_name AND run_id = @run_id`,
 				params: {
 					workflow_name: workflowName,
 					run_id: runId
 				}
-			});
+			}, tx));
 		} catch (error) {
+			if (isRunFenceConflictError(error)) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("SPANNER", "DELETE_WORKFLOW_RUN_BY_ID", "FAILED"),
 				domain: ErrorDomain.STORAGE,