@mastra/pg 1.31.0-alpha.0 → 1.31.0-alpha.2

package/dist/index.js

@@ -2475,7 +2475,8 @@ async function loadSchemaSnapshot(client, schemaName) {
 	const schema = schemaName || "public";
 	const [tableRows, columnRows, indexRows] = await Promise.all([
 		client.manyOrNone(`SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = $1`, [schema]),
-		client.manyOrNone(`SELECT c.relname AS table_name, a.attname AS column_name, format_type(a.atttypid, a.atttypmod) AS data_type
+		client.manyOrNone(`SELECT c.relname AS table_name, a.attname AS column_name, format_type(a.atttypid, a.atttypmod) AS data_type,
+              a.attnotnull AS not_null
          FROM pg_catalog.pg_class c
          JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
          JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid
@@ -2483,7 +2484,7 @@ async function loadSchemaSnapshot(client, schemaName) {
           AND c.relkind IN ('r', 'p')
           AND a.attnum > 0
           AND NOT a.attisdropped`, [schema]),
-		client.manyOrNone(`SELECT c.relname AS indexname, i.indisreplident AS is_replica_identity, i.indisprimary AS is_primary
+		client.manyOrNone(`SELECT c.relname AS indexname, i.indisreplident AS is_replica_identity, i.indisprimary AS is_primary, i.indisvalid AS is_valid
          FROM pg_catalog.pg_index i
          JOIN pg_catalog.pg_class c ON c.oid = i.indexrelid
          JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
@@ -2491,6 +2492,7 @@ async function loadSchemaSnapshot(client, schemaName) {
 	]);
 	const columns = /* @__PURE__ */ new Map();
 	const columnTypes = /* @__PURE__ */ new Map();
+	const notNullColumns = /* @__PURE__ */ new Map();
 	for (const row of columnRows) {
 		let set = columns.get(row.table_name);
 		if (!set) {
@@ -2504,12 +2506,22 @@ async function loadSchemaSnapshot(client, schemaName) {
 			columnTypes.set(row.table_name, types);
 		}
 		types.set(row.column_name, row.data_type);
+		if (row.not_null) {
+			let notNull = notNullColumns.get(row.table_name);
+			if (!notNull) {
+				notNull = /* @__PURE__ */ new Set();
+				notNullColumns.set(row.table_name, notNull);
+			}
+			notNull.add(row.column_name);
+		}
 	}
 	const indexes = /* @__PURE__ */ new Set();
 	const replicaIdentityIndexes = /* @__PURE__ */ new Set();
 	const primaryKeyIndexes = /* @__PURE__ */ new Set();
+	const invalidIndexes = /* @__PURE__ */ new Set();
 	for (const row of indexRows) {
-		indexes.add(row.indexname);
+		if (row.is_valid) indexes.add(row.indexname);
+		else invalidIndexes.add(row.indexname);
 		if (row.is_replica_identity) replicaIdentityIndexes.add(row.indexname);
 		if (row.is_primary) primaryKeyIndexes.add(row.indexname.toLowerCase());
 	}
@@ -2518,9 +2530,11 @@ async function loadSchemaSnapshot(client, schemaName) {
 		tables: new Set(tableRows.map((r) => r.tablename)),
 		columns,
 		columnTypes,
+		notNullColumns,
 		indexes,
 		replicaIdentityIndexes,
-		primaryKeyIndexes
+		primaryKeyIndexes,
+		invalidIndexes
 	};
 }
 //#endregion
@@ -3595,6 +3609,33 @@ MIGRATION REQUIRED: Duplicate spans detected in ${duplicateInfo.tableName}\n====
 			this.columnTypeCache.delete(tableName);
 		}
 	}
+	/**
+	* Drops NOT NULL from `columns` on `tableName` where it is still set. Answered
+	* from the init snapshot when one is installed, so a converged schema issues
+	* no query and never takes the ACCESS EXCLUSIVE lock.
+	*/
+	async dropNotNull({ tableName, columns }) {
+		const parsedColumns = columns.map((c) => parseSqlIdentifier(c, "column name"));
+		const snapshot = this.schemaSnapshot;
+		let toAlter;
+		if (snapshot) {
+			const notNull = snapshot.notNullColumns.get(tableName);
+			toAlter = parsedColumns.filter((c) => notNull?.has(c));
+		} else toAlter = (await this.client.any(`SELECT column_name FROM information_schema.columns
+         WHERE table_schema = $1 AND table_name = $2 AND column_name = ANY($3) AND is_nullable = 'NO'`, [
+			this.schemaName || "public",
+			tableName,
+			parsedColumns
+		])).map((r) => r.column_name);
+		if (toAlter.length === 0) return;
+		const fullTableName = getTableName$6({
+			indexName: tableName,
+			schemaName: getSchemaName$6(this.schemaName)
+		});
+		await this.client.none(`ALTER TABLE ${fullTableName} ${toAlter.map((c) => `ALTER COLUMN "${c}" DROP NOT NULL`).join(", ")}`);
+		const notNull = snapshot?.notNullColumns.get(tableName);
+		for (const c of toAlter) notNull?.delete(c);
+	}
 	async load({ tableName, keys }) {
 		try {
 			const keyEntries = Object.entries(keys).map(([key, value]) => [parseSqlIdentifier(key, "column name"), value]);
@@ -3668,11 +3709,8 @@ MIGRATION REQUIRED: Duplicate spans detected in ${duplicateInfo.tableName}\n====
 				schemaName: getSchemaName$6(this.schemaName)
 			});
 			const snapshot = this.schemaSnapshot;
-			if (snapshot) {
-				if (snapshot.indexes.has(name)) return;
-			} else if (await this.client.oneOrNone(`SELECT 1 FROM pg_indexes
-         WHERE indexname = $1
-         AND schemaname = $2`, [name, schemaName])) return;
+			if (snapshot?.indexes.has(name)) return;
+			if (!await this.prepareIndexBuild(name, schemaName)) return;
 			const uniqueStr = unique ? "UNIQUE " : "";
 			const concurrentStr = concurrent ? "CONCURRENTLY " : "";
 			const methodStr = method !== "btree" ? `USING ${method} ` : "";
@@ -3691,16 +3729,9 @@ MIGRATION REQUIRED: Duplicate spans detected in ${duplicateInfo.tableName}\n====
 			let withStr = "";
 			if (storage && Object.keys(storage).length > 0) withStr = ` WITH (${Object.entries(storage).map(([key, value]) => `${key} = ${value}`).join(", ")})`;
 			const sql = `CREATE ${uniqueStr}INDEX ${concurrentStr}${`"${parseSqlIdentifier(name, "index name")}"`} ON ${fullTableName} ${methodStr}(${columnsStr})${withStr}${tablespaceStr}${whereStr}`;
-			await this.client.none(sql);
+			await this.timedIndexBuild(name, sql);
 			snapshot?.indexes.add(name);
 		} catch (error) {
-			if (error instanceof Error && error.message.includes("CONCURRENTLY")) {
-				const retryOptions = {
-					...options,
-					concurrent: false
-				};
-				return this.createIndex(retryOptions);
-			}
 			throw new MastraError({
 				id: createStorageErrorId("PG", "INDEX_CREATE", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -3724,9 +3755,43 @@ MIGRATION REQUIRED: Duplicate spans detected in ${duplicateInfo.tableName}\n====
 	async createIndexFromStatement(indexName, sql) {
 		const snapshot = this.schemaSnapshot;
 		if (snapshot?.indexes.has(indexName)) return;
-		await this.client.none(sql);
+		if (!await this.prepareIndexBuild(indexName, this.schemaName || "public")) return;
+		await this.timedIndexBuild(indexName, sql);
 		snapshot?.indexes.add(indexName);
 	}
+	/**
+	* Decides whether `indexName` still needs building. Returns false when a valid
+	* index already exists, or when another session is currently building it.
+	* An invalid index left by an interrupted `CREATE INDEX CONCURRENTLY` is
+	* dropped (concurrently) so the caller rebuilds it — `IF NOT EXISTS` would
+	* otherwise keep the broken index forever.
+	*/
+	async prepareIndexBuild(indexName, schemaName) {
+		const snapshot = this.schemaSnapshot;
+		if (snapshot && !snapshot.invalidIndexes.has(indexName)) return true;
+		const existing = await this.client.oneOrNone(`SELECT i.indisvalid AS is_valid,
+              EXISTS (SELECT 1 FROM pg_stat_progress_create_index p WHERE p.index_relid = c.oid) AS building
+         FROM pg_catalog.pg_index i
+         JOIN pg_catalog.pg_class c ON c.oid = i.indexrelid
+         JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relname = $1 AND n.nspname = $2`, [indexName, schemaName]);
+		if (!existing) return true;
+		if (existing.is_valid) return false;
+		if (existing.building) {
+			this.logger.warn(`Index ${indexName} is being built by another session; skipping`);
+			return false;
+		}
+		this.logger.warn(`Dropping invalid index ${indexName} left by an interrupted build; rebuilding it`);
+		const quotedIndexName = `"${parseSqlIdentifier(indexName, "index name")}"`;
+		await this.client.none(`DROP INDEX CONCURRENTLY IF EXISTS ${getSchemaName$6(this.schemaName)}.${quotedIndexName}`);
+		return true;
+	}
+	async timedIndexBuild(indexName, sql) {
+		const startedAt = Date.now();
+		this.logger.info(`Creating index ${indexName}`);
+		await this.client.none(sql);
+		this.logger.info(`Created index ${indexName} in ${Date.now() - startedAt}ms`);
+	}
 	async dropIndex(indexName) {
 		try {
 			const schemaName = this.schemaName || "public";
@@ -7061,6 +7126,10 @@ var ExperimentsPG = class ExperimentsPG extends ExperimentsStorage {
 				"scorerIds"
 			]
 		});
+		await this.#db.dropNotNull({
+			tableName: TABLE_EXPERIMENTS,
+			columns: ["targetType", "targetId"]
+		});
 		await this.#db.alterTable({
 			tableName: TABLE_EXPERIMENT_RESULTS,
 			schema: EXPERIMENT_RESULTS_SCHEMA,
@@ -10471,6 +10540,7 @@ function dedupeMessagesForSave(messages) {
 var MemoryPG = class MemoryPG extends MemoryStorage {
 	supportsPartialThreadUpdate = true;
 	supportsObservationalMemory = true;
+	supportsAtomicWorkingMemoryMerge = true;
 	supportsObservationalMemoryHistorySearch = true;
 	/**
 	* Retention-eligible tables. `threads`, `messages`, and `resources` all anchor
@@ -11817,6 +11887,57 @@ var MemoryPG = class MemoryPG extends MemoryStorage {
 			metadata: metadataJson ? JSON.parse(metadataJson) : resource.metadata
 		};
 	}
+	async mergeResourceWorkingMemory({ resourceId, merge }) {
+		const tableName = getTableName$3({
+			indexName: TABLE_RESOURCES,
+			schemaName: getSchemaName$3(this.#schema)
+		});
+		const callbackError = {};
+		try {
+			return await this.#db.client.tx(async (t) => {
+				const now = (/* @__PURE__ */ new Date()).toISOString();
+				await t.none(`INSERT INTO ${tableName} (id, metadata, "createdAt", "createdAtZ", "updatedAt", "updatedAtZ")
+         VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (id) DO NOTHING`, [
+					resourceId,
+					toPgJson({}),
+					now,
+					now,
+					now,
+					now
+				]);
+				await t.one(`SELECT id FROM ${tableName} WHERE id = $1 FOR UPDATE`, [resourceId]);
+				const existing = await this.#getResourceById(t, resourceId);
+				let workingMemory;
+				try {
+					workingMemory = merge(existing.workingMemory ?? void 0);
+				} catch (error) {
+					callbackError.error = error;
+					throw error;
+				}
+				const updatedAt = /* @__PURE__ */ new Date();
+				const updatedAtStr = updatedAt.toISOString();
+				await t.none(`UPDATE ${tableName} SET "workingMemory" = $1, "updatedAt" = $2, "updatedAtZ" = $3 WHERE id = $4`, [
+					workingMemory,
+					updatedAtStr,
+					updatedAtStr,
+					resourceId
+				]);
+				return {
+					...existing,
+					workingMemory,
+					updatedAt
+				};
+			});
+		} catch (error) {
+			if (error === callbackError.error || error instanceof MastraError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("PG", "MERGE_RESOURCE_WORKING_MEMORY", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { resourceId }
+			}, error);
+		}
+	}
 	async updateResource({ resourceId, workingMemory, metadata }) {
 		const existingResource = await this.#getResourceById(this.#db.client, resourceId);
 		if (!existingResource) {
@@ -23184,12 +23305,17 @@ function workflowSnapshotStatusIndexName(schemaName) {
 /**
 * Expression index on `(workflow_name, snapshot->>'status', "createdAt" DESC)` so
 * listWorkflowRuns() status filters can use an index instead of scanning every snapshot.
+*
+* `concurrently` is set by init, which may build the index on a large existing table while
+* other processes keep writing to it. The exported DDL stays plain because migration
+* tooling often wraps statements in a transaction, where CONCURRENTLY is rejected.
 */
-function workflowSnapshotStatusIndexSQL(indexName, schemaName) {
-	return `CREATE INDEX IF NOT EXISTS "${indexName}" ON ${getTableName({
+function workflowSnapshotStatusIndexSQL(indexName, schemaName, concurrently = false) {
+	const tableName = getTableName({
 		indexName: TABLE_WORKFLOW_SNAPSHOT,
 		schemaName: getSchemaName(schemaName)
-	})} (workflow_name, (snapshot ->> 'status'), "createdAt" DESC)`;
+	});
+	return `CREATE INDEX ${concurrently ? "CONCURRENTLY " : ""}IF NOT EXISTS "${indexName}" ON ${tableName} (workflow_name, (snapshot ->> 'status'), "createdAt" DESC)`;
 }
 /** Base name (before any schema prefix) of the expression index backing the threadId filter. */
 const WORKFLOW_SNAPSHOT_THREAD_ID_INDEX = "mastra_workflow_snapshot_threadid_idx";
@@ -23226,12 +23352,14 @@ const WORKFLOW_SNAPSHOT_THREAD_ID_EXPR = `COALESCE(jsonb_path_query_first(snapsh
 /**
 * Expression index on the snapshot-embedded thread id so listWorkflowRuns() threadId filters
 * (Agent.listSuspendedRuns) can use an index instead of detoasting every snapshot.
+* See {@link workflowSnapshotStatusIndexSQL} for `concurrently`.
 */
-function workflowSnapshotThreadIdIndexSQL(indexName, schemaName) {
-	return `CREATE INDEX IF NOT EXISTS "${indexName}" ON ${getTableName({
+function workflowSnapshotThreadIdIndexSQL(indexName, schemaName, concurrently = false) {
+	const tableName = getTableName({
 		indexName: TABLE_WORKFLOW_SNAPSHOT,
 		schemaName: getSchemaName(schemaName)
-	})} ((${WORKFLOW_SNAPSHOT_THREAD_ID_EXPR}))`;
+	});
+	return `CREATE INDEX ${concurrently ? "CONCURRENTLY " : ""}IF NOT EXISTS "${indexName}" ON ${tableName} ((${WORKFLOW_SNAPSHOT_THREAD_ID_EXPR}))`;
 }
 var WorkflowsPG = class WorkflowsPG extends WorkflowsStorage {
 	#db;
@@ -23328,13 +23456,13 @@ var WorkflowsPG = class WorkflowsPG extends WorkflowsStorage {
 		if (await this.#db.getColumnType(TABLE_WORKFLOW_SNAPSHOT, "snapshot") !== "jsonb") return;
 		const indexName = workflowSnapshotStatusIndexName(this.#schema);
 		try {
-			await this.#db.createIndexFromStatement(indexName, workflowSnapshotStatusIndexSQL(indexName, this.#schema));
+			await this.#db.createIndexFromStatement(indexName, workflowSnapshotStatusIndexSQL(indexName, this.#schema, true));
 		} catch (error) {
 			this.logger?.warn?.(`Failed to create index ${indexName}:`, error);
 		}
 		const threadIdIndexName = workflowSnapshotThreadIdIndexName(this.#schema);
 		try {
-			await this.#db.createIndexFromStatement(threadIdIndexName, workflowSnapshotThreadIdIndexSQL(threadIdIndexName, this.#schema));
+			await this.#db.createIndexFromStatement(threadIdIndexName, workflowSnapshotThreadIdIndexSQL(threadIdIndexName, this.#schema, true));
 		} catch (error) {
 			this.logger?.warn?.(`Failed to create index ${threadIdIndexName}:`, error);
 		}
@@ -23405,7 +23533,7 @@ var WorkflowsPG = class WorkflowsPG extends WorkflowsStorage {
 	async dangerouslyClearAll() {
 		await this.#db.clearTable({ tableName: TABLE_WORKFLOW_SNAPSHOT });
 	}
-	async updateWorkflowResults({ workflowName, runId, stepId, result, requestContext }) {
+	async updateWorkflowResults({ workflowName, runId, stepId, result, requestContext, state }) {
 		try {
 			return await this.#db.client.tx(async (t) => {
 				const tableName = getTableName({
@@ -23436,7 +23564,8 @@ var WorkflowsPG = class WorkflowsPG extends WorkflowsStorage {
 					snapshot,
 					stepId,
 					result,
-					requestContext
+					requestContext,
+					state
 				});
 				const now = /* @__PURE__ */ new Date();
 				const sanitizedSnapshot = toPgJson(snapshot);