@mastra/libsql 1.22.3 → 1.22.4-alpha.1

package/dist/index.js

@@ -3612,15 +3612,11 @@ var DatasetsLibSQL = class extends DatasetsStorage {
 		await this.#db.deleteData({ tableName: TABLE_DATASETS });
 	}
 	async experimentTablesExist() {
-		try {
-			const row = (await this.#client.execute({
-				sql: `SELECT COUNT(*) AS c FROM sqlite_master WHERE type = 'table' AND name IN (?, ?)`,
-				args: [TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS]
-			})).rows?.[0];
-			return Number(row?.c ?? 0) === 2;
-		} catch {
-			return false;
-		}
+		const row = (await this.#client.execute({
+			sql: `SELECT COUNT(*) AS c FROM sqlite_master WHERE type = 'table' AND name IN (?, ?)`,
+			args: [TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS]
+		})).rows?.[0];
+		return Number(row?.c ?? 0) === 2;
 	}
 	transformDatasetRow(row) {
 		return {
@@ -3645,6 +3641,8 @@ var DatasetsLibSQL = class extends DatasetsStorage {
 		};
 	}
 	transformItemRow(row) {
+		const metadata = row.metadata ? safelyParseJSON(row.metadata) : void 0;
+		const emptyValue = metadata?.__purged === true ? null : void 0;
 		return {
 			id: row.id,
 			datasetId: row.datasetId,
@@ -3653,19 +3651,21 @@ var DatasetsLibSQL = class extends DatasetsStorage {
 			organizationId: row.organizationId ?? null,
 			projectId: row.projectId ?? null,
 			input: safelyParseJSON(row.input),
-			groundTruth: row.groundTruth ? safelyParseJSON(row.groundTruth) : void 0,
-			expectedTrajectory: row.expectedTrajectory ? safelyParseJSON(row.expectedTrajectory) : void 0,
-			toolMocks: row.toolMocks ? safelyParseJSON(row.toolMocks) : void 0,
-			unmockedToolPolicy: row.unmockedToolPolicy ?? void 0,
-			scorerIds: row.scorerIds ? safelyParseJSON(row.scorerIds) : void 0,
-			requestContext: row.requestContext ? safelyParseJSON(row.requestContext) : void 0,
-			metadata: row.metadata ? safelyParseJSON(row.metadata) : void 0,
-			source: row.source ? safelyParseJSON(row.source) : void 0,
+			groundTruth: row.groundTruth ? safelyParseJSON(row.groundTruth) : emptyValue,
+			expectedTrajectory: row.expectedTrajectory ? safelyParseJSON(row.expectedTrajectory) : emptyValue,
+			toolMocks: row.toolMocks ? safelyParseJSON(row.toolMocks) : emptyValue,
+			unmockedToolPolicy: row.unmockedToolPolicy ?? emptyValue,
+			scorerIds: row.scorerIds ? safelyParseJSON(row.scorerIds) : emptyValue,
+			requestContext: row.requestContext ? safelyParseJSON(row.requestContext) : emptyValue,
+			metadata,
+			source: row.source ? safelyParseJSON(row.source) : emptyValue,
 			createdAt: ensureDate(row.createdAt),
 			updatedAt: ensureDate(row.updatedAt)
 		};
 	}
 	transformItemRowFull(row) {
+		const metadata = row.metadata ? safelyParseJSON(row.metadata) : void 0;
+		const emptyValue = metadata?.__purged === true ? null : void 0;
 		return {
 			id: row.id,
 			datasetId: row.datasetId,
@@ -3676,14 +3676,14 @@ var DatasetsLibSQL = class extends DatasetsStorage {
 			validTo: row.validTo,
 			isDeleted: Boolean(row.isDeleted),
 			input: safelyParseJSON(row.input),
-			groundTruth: row.groundTruth ? safelyParseJSON(row.groundTruth) : void 0,
-			expectedTrajectory: row.expectedTrajectory ? safelyParseJSON(row.expectedTrajectory) : void 0,
-			toolMocks: row.toolMocks ? safelyParseJSON(row.toolMocks) : void 0,
-			unmockedToolPolicy: row.unmockedToolPolicy ?? void 0,
-			scorerIds: row.scorerIds ? safelyParseJSON(row.scorerIds) : void 0,
-			requestContext: row.requestContext ? safelyParseJSON(row.requestContext) : void 0,
-			metadata: row.metadata ? safelyParseJSON(row.metadata) : void 0,
-			source: row.source ? safelyParseJSON(row.source) : void 0,
+			groundTruth: row.groundTruth ? safelyParseJSON(row.groundTruth) : emptyValue,
+			expectedTrajectory: row.expectedTrajectory ? safelyParseJSON(row.expectedTrajectory) : emptyValue,
+			toolMocks: row.toolMocks ? safelyParseJSON(row.toolMocks) : emptyValue,
+			unmockedToolPolicy: row.unmockedToolPolicy ?? emptyValue,
+			scorerIds: row.scorerIds ? safelyParseJSON(row.scorerIds) : emptyValue,
+			requestContext: row.requestContext ? safelyParseJSON(row.requestContext) : emptyValue,
+			metadata,
+			source: row.source ? safelyParseJSON(row.source) : emptyValue,
 			createdAt: ensureDate(row.createdAt),
 			updatedAt: ensureDate(row.updatedAt)
 		};
@@ -4219,6 +4219,57 @@ var DatasetsLibSQL = class extends DatasetsStorage {
 			}, error);
 		}
 	}
+	async _doPurgeItem({ id, datasetId }) {
+		try {
+			const experimentTablesExist = await this.experimentTablesExist();
+			await withClientWriteLock(this.#client, async () => {
+				const tx = await this.#client.transaction("write");
+				try {
+					if (!(await tx.execute({
+						sql: `SELECT id FROM ${TABLE_DATASET_ITEMS} WHERE id = ? AND datasetId = ? LIMIT 1`,
+						args: [id, datasetId]
+					})).rows[0]) {
+						await tx.commit();
+						return;
+					}
+					const purgedAt = (/* @__PURE__ */ new Date()).toISOString();
+					const purgedMetadata = JSON.stringify({
+						__purged: true,
+						purgedAt
+					});
+					await tx.execute({
+						sql: `UPDATE ${TABLE_DATASET_ITEMS} SET input = jsonb('null'), groundTruth = NULL, expectedTrajectory = NULL, toolMocks = NULL, unmockedToolPolicy = NULL, scorerIds = NULL, requestContext = NULL, metadata = jsonb(?), source = NULL WHERE id = ? AND datasetId = ?`,
+						args: [
+							purgedMetadata,
+							id,
+							datasetId
+						]
+					});
+					if (experimentTablesExist) await tx.execute({
+						sql: `UPDATE ${TABLE_EXPERIMENT_RESULTS} SET input = jsonb('null'), output = NULL, groundTruth = NULL, error = NULL, toolMockReport = NULL, tags = NULL, comment = NULL, metadata = jsonb(?) WHERE itemId = ? AND experimentId IN (SELECT id FROM ${TABLE_EXPERIMENTS} WHERE datasetId = ?)`,
+						args: [
+							purgedMetadata,
+							id,
+							datasetId
+						]
+					});
+					await tx.commit();
+				} catch (error) {
+					if (!tx.closed) await tx.rollback().catch((rollbackError) => {
+						throw new AggregateError([error, rollbackError], "Transaction and rollback both failed");
+					});
+					throw error;
+				}
+			});
+		} catch (error) {
+			if (error instanceof MastraError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("LIBSQL", "PURGE_ITEM", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY
+			}, error);
+		}
+	}
 	async getItemById(args) {
 		try {
 			let result;
@@ -4671,6 +4722,35 @@ var ExperimentsLibSQL = class extends ExperimentsStorage {
 			initialBackoffMs: config.initialBackoffMs
 		});
 	}
+	async #withPurgeBarrier(experimentId, itemId, fn) {
+		return withClientWriteLock(this.#client, async () => {
+			const tx = await this.#client.transaction("write");
+			try {
+				const row = (await tx.execute({
+					sql: `SELECT json_extract(i."metadata", '$.__purged') AS "purged",
+                       json_extract(i."metadata", '$.purgedAt') AS "purgedAt"
+                FROM ${TABLE_EXPERIMENTS} e
+                JOIN ${TABLE_DATASET_ITEMS} i ON i."datasetId" = e."datasetId" AND i."id" = ?
+                WHERE e."id" = ? AND json_extract(i."metadata", '$.__purged') = 1
+                LIMIT 1`,
+					args: [itemId, experimentId]
+				})).rows[0];
+				const result = await fn(tx, row?.purged === 1 ? {
+					__purged: true,
+					purgedAt: row.purgedAt
+				} : null);
+				await tx.commit();
+				return result;
+			} catch (error) {
+				if (!tx.closed) await tx.rollback().catch((rollbackError) => {
+					throw new AggregateError([error, rollbackError], "Transaction and rollback both failed");
+				});
+				throw error;
+			} finally {
+				tx.close();
+			}
+		});
+	}
 	async init() {
 		await this.#db.createTable({
 			tableName: TABLE_EXPERIMENTS,
@@ -5142,30 +5222,37 @@ var ExperimentsLibSQL = class extends ExperimentsStorage {
 			const id = input.id ?? crypto.randomUUID();
 			const now = /* @__PURE__ */ new Date();
 			const nowIso = now.toISOString();
-			await this.#db.insert({
-				tableName: TABLE_EXPERIMENT_RESULTS,
-				record: {
-					id,
-					experimentId: input.experimentId,
-					itemId: input.itemId,
-					itemDatasetVersion: input.itemDatasetVersion ?? null,
-					organizationId: input.organizationId ?? null,
-					projectId: input.projectId ?? null,
-					input: input.input,
-					output: input.output,
-					groundTruth: input.groundTruth,
-					metadata: input.metadata ?? null,
-					error: input.error ?? null,
-					startedAt: input.startedAt.toISOString(),
-					completedAt: input.completedAt.toISOString(),
-					retryCount: input.retryCount,
-					attempt: input.attempt ?? 0,
-					traceId: input.traceId ?? null,
-					status: input.status ?? null,
-					tags: input.tags !== void 0 && input.tags !== null ? JSON.stringify(input.tags) : null,
-					toolMockReport: input.toolMockReport ?? null,
-					createdAt: nowIso
-				}
+			const purgeMetadata = await this.#withPurgeBarrier(input.experimentId, input.itemId, async (tx, marker) => {
+				await tx.execute({
+					sql: `INSERT INTO ${TABLE_EXPERIMENT_RESULTS} (
+            "id", "experimentId", "itemId", "itemDatasetVersion", "organizationId", "projectId",
+            "input", "output", "groundTruth", "metadata", "error", "startedAt", "completedAt",
+            "retryCount", "attempt", "traceId", "status", "tags", "toolMockReport", "createdAt"
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+					args: [
+						id,
+						input.experimentId,
+						input.itemId,
+						input.itemDatasetVersion ?? null,
+						input.organizationId ?? null,
+						input.projectId ?? null,
+						JSON.stringify(marker ? null : input.input),
+						JSON.stringify(marker ? null : input.output ?? null),
+						JSON.stringify(marker ? null : input.groundTruth ?? null),
+						JSON.stringify(marker ?? input.metadata ?? null),
+						JSON.stringify(marker ? null : input.error ?? null),
+						input.startedAt.toISOString(),
+						input.completedAt.toISOString(),
+						input.retryCount,
+						input.attempt ?? 0,
+						input.traceId ?? null,
+						input.status ?? null,
+						JSON.stringify(marker ? null : input.tags ?? null),
+						JSON.stringify(marker ? null : input.toolMockReport ?? null),
+						nowIso
+					]
+				});
+				return marker;
 			});
 			return {
 				id,
@@ -5174,19 +5261,19 @@ var ExperimentsLibSQL = class extends ExperimentsStorage {
 				itemDatasetVersion: input.itemDatasetVersion,
 				organizationId: input.organizationId ?? null,
 				projectId: input.projectId ?? null,
-				input: input.input,
-				output: input.output,
-				groundTruth: input.groundTruth,
-				metadata: input.metadata ?? null,
-				error: input.error,
+				input: purgeMetadata ? null : input.input,
+				output: purgeMetadata ? null : input.output,
+				groundTruth: purgeMetadata ? null : input.groundTruth,
+				metadata: purgeMetadata ?? input.metadata ?? null,
+				error: purgeMetadata ? null : input.error,
 				startedAt: input.startedAt,
 				completedAt: input.completedAt,
 				retryCount: input.retryCount,
 				attempt: input.attempt ?? 0,
 				traceId: input.traceId ?? null,
 				status: input.status ?? null,
-				tags: input.tags ?? null,
-				toolMockReport: input.toolMockReport ?? null,
+				tags: purgeMetadata ? null : input.tags ?? null,
+				toolMockReport: purgeMetadata ? null : input.toolMockReport ?? null,
 				createdAt: now
 			};
 		} catch (error) {
@@ -5200,70 +5287,61 @@ var ExperimentsLibSQL = class extends ExperimentsStorage {
 	async upsertExperimentResult(input) {
 		try {
 			const attempt = input.attempt ?? 0;
-			let existingId = (await this.#client.execute({
-				sql: `SELECT "id" FROM ${TABLE_EXPERIMENT_RESULTS} WHERE "experimentId" = ? AND "itemId" = ? AND COALESCE("attempt", 0) = ?`,
-				args: [
-					input.experimentId,
-					input.itemId,
-					attempt
-				]
-			})).rows[0]?.id;
-			if (!existingId) try {
-				return await this.addExperimentResult({
-					...input,
-					attempt
+			return await this.#withPurgeBarrier(input.experimentId, input.itemId, async (tx, marker) => {
+				const id = crypto.randomUUID();
+				await tx.execute({
+					sql: `INSERT INTO ${TABLE_EXPERIMENT_RESULTS} (
+            "id", "experimentId", "itemId", "itemDatasetVersion", "organizationId", "projectId",
+            "input", "output", "groundTruth", "metadata", "error", "startedAt", "completedAt",
+            "retryCount", "attempt", "traceId", "status", "tags", "toolMockReport", "createdAt"
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+          ON CONFLICT("experimentId", "itemId", "attempt") DO UPDATE SET
+            "itemDatasetVersion" = excluded."itemDatasetVersion", "organizationId" = excluded."organizationId",
+            "projectId" = excluded."projectId", "input" = excluded."input", "output" = excluded."output",
+            "groundTruth" = excluded."groundTruth", "metadata" = excluded."metadata", "error" = excluded."error",
+            "startedAt" = excluded."startedAt", "completedAt" = excluded."completedAt",
+            "retryCount" = excluded."retryCount", "attempt" = excluded."attempt", "traceId" = excluded."traceId",
+            "status" = excluded."status", "tags" = excluded."tags", "toolMockReport" = excluded."toolMockReport",
+            "comment" = CASE WHEN ? THEN NULL ELSE "comment" END`,
+					args: [
+						id,
+						input.experimentId,
+						input.itemId,
+						input.itemDatasetVersion ?? null,
+						input.organizationId ?? null,
+						input.projectId ?? null,
+						JSON.stringify(marker ? null : input.input),
+						JSON.stringify(marker ? null : input.output ?? null),
+						JSON.stringify(marker ? null : input.groundTruth ?? null),
+						JSON.stringify(marker ?? input.metadata ?? null),
+						JSON.stringify(marker ? null : input.error ?? null),
+						input.startedAt.toISOString(),
+						input.completedAt.toISOString(),
+						input.retryCount,
+						attempt,
+						input.traceId ?? null,
+						input.status ?? null,
+						JSON.stringify(marker ? null : input.tags ?? null),
+						JSON.stringify(marker ? null : input.toolMockReport ?? null),
+						(/* @__PURE__ */ new Date()).toISOString(),
+						Boolean(marker)
+					]
 				});
-			} catch (insertError) {
-				if (!hasErrorCode(insertError, /* @__PURE__ */ new Set([
-					"SQLITE_CONSTRAINT",
-					"SQLITE_CONSTRAINT_PRIMARYKEY",
-					"SQLITE_CONSTRAINT_UNIQUE"
-				]))) throw insertError;
-				existingId = (await this.#client.execute({
-					sql: `SELECT "id" FROM ${TABLE_EXPERIMENT_RESULTS} WHERE "experimentId" = ? AND "itemId" = ? AND COALESCE("attempt", 0) = ?`,
+				const row = (await tx.execute({
+					sql: `SELECT * FROM ${TABLE_EXPERIMENT_RESULTS} WHERE "experimentId" = ? AND "itemId" = ? AND COALESCE("attempt", 0) = ?`,
 					args: [
 						input.experimentId,
 						input.itemId,
 						attempt
 					]
-				})).rows[0]?.id;
-				if (!existingId) throw insertError;
-			}
-			await this.#client.execute({
-				sql: `UPDATE ${TABLE_EXPERIMENT_RESULTS} SET
-          "itemDatasetVersion" = ?, "organizationId" = ?, "projectId" = ?,
-          "input" = ?, "output" = ?, "groundTruth" = ?, "metadata" = ?, "error" = ?,
-          "startedAt" = ?, "completedAt" = ?, "retryCount" = ?, "attempt" = ?,
-          "traceId" = ?, "status" = ?, "tags" = ?, "toolMockReport" = ?
-        WHERE "id" = ?`,
-				args: [
-					input.itemDatasetVersion ?? null,
-					input.organizationId ?? null,
-					input.projectId ?? null,
-					JSON.stringify(input.input),
-					input.output != null ? JSON.stringify(input.output) : null,
-					input.groundTruth != null ? JSON.stringify(input.groundTruth) : null,
-					input.metadata != null ? JSON.stringify(input.metadata) : null,
-					input.error != null ? JSON.stringify(input.error) : null,
-					input.startedAt.toISOString(),
-					input.completedAt.toISOString(),
-					input.retryCount,
-					attempt,
-					input.traceId ?? null,
-					input.status ?? null,
-					input.tags != null ? JSON.stringify(input.tags) : null,
-					input.toolMockReport != null ? JSON.stringify(input.toolMockReport) : null,
-					existingId
-				]
-			});
-			const result = await this.getExperimentResultById({ id: existingId });
-			if (!result) throw new MastraError({
-				id: createStorageErrorId("LIBSQL", "UPSERT_EXPERIMENT_RESULT", "NOT_FOUND"),
-				domain: ErrorDomain.STORAGE,
-				category: ErrorCategory.USER,
-				details: { resultId: existingId }
+				})).rows[0];
+				if (!row) throw new MastraError({
+					id: createStorageErrorId("LIBSQL", "UPSERT_EXPERIMENT_RESULT", "NOT_FOUND"),
+					domain: ErrorDomain.STORAGE,
+					category: ErrorCategory.USER
+				});
+				return this.transformExperimentResultRow(row);
 			});
-			return result;
 		} catch (error) {
 			if (error instanceof MastraError) throw error;
 			throw new MastraError({
@@ -5275,56 +5353,47 @@ var ExperimentsLibSQL = class extends ExperimentsStorage {
 	}
 	async updateExperimentResult(input) {
 		try {
-			const setClauses = [];
-			const values = [];
-			if (input.status !== void 0) {
-				setClauses.push(`"status" = ?`);
-				values.push(input.status);
-			}
-			if (input.tags !== void 0) {
-				setClauses.push(`"tags" = ?`);
-				values.push(JSON.stringify(input.tags));
-			}
-			if (input.comment !== void 0) {
-				setClauses.push(`"comment" = ?`);
-				values.push(input.comment);
-			}
-			if (setClauses.length === 0) {
-				const existing = await this.getExperimentResultById({ id: input.id });
-				if (!existing) throw new MastraError({
+			const ownerRow = (await this.#client.execute({
+				sql: `SELECT "experimentId", "itemId" FROM ${TABLE_EXPERIMENT_RESULTS} WHERE "id" = ?${input.experimentId ? " AND \"experimentId\" = ?" : ""}`,
+				args: input.experimentId ? [input.id, input.experimentId] : [input.id]
+			})).rows[0];
+			if (!ownerRow) throw new MastraError({
+				id: createStorageErrorId("LIBSQL", "UPDATE_EXPERIMENT_RESULT", "NOT_FOUND"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.USER,
+				details: { resultId: input.id }
+			});
+			return await this.#withPurgeBarrier(String(ownerRow.experimentId), String(ownerRow.itemId), async (tx, marker) => {
+				if ((await tx.execute({
+					sql: `UPDATE ${TABLE_EXPERIMENT_RESULTS}
+                SET "status" = CASE WHEN ? THEN ? ELSE "status" END,
+                    "tags" = CASE WHEN ? THEN NULL WHEN ? THEN ? ELSE "tags" END,
+                    "comment" = CASE WHEN ? THEN NULL WHEN ? THEN ? ELSE "comment" END
+                WHERE "id" = ?${input.experimentId ? " AND \"experimentId\" = ?" : ""}`,
+					args: [
+						input.status !== void 0,
+						input.status ?? null,
+						Boolean(marker),
+						input.tags !== void 0,
+						JSON.stringify(input.tags ?? null),
+						Boolean(marker),
+						input.comment !== void 0,
+						input.comment ?? null,
+						input.id,
+						...input.experimentId ? [input.experimentId] : []
+					]
+				})).rowsAffected === 0) throw new MastraError({
 					id: createStorageErrorId("LIBSQL", "UPDATE_EXPERIMENT_RESULT", "NOT_FOUND"),
 					domain: ErrorDomain.STORAGE,
 					category: ErrorCategory.USER,
 					details: { resultId: input.id }
 				});
-				return existing;
-			}
-			values.push(input.id);
-			let whereClause = `"id" = ?`;
-			if (input.experimentId) {
-				values.push(input.experimentId);
-				whereClause += ` AND "experimentId" = ?`;
-			}
-			if ((await this.#client.execute({
-				sql: `UPDATE ${TABLE_EXPERIMENT_RESULTS} SET ${setClauses.join(", ")} WHERE ${whereClause}`,
-				args: values
-			})).rowsAffected === 0) throw new MastraError({
-				id: createStorageErrorId("LIBSQL", "UPDATE_EXPERIMENT_RESULT", "NOT_FOUND"),
-				domain: ErrorDomain.STORAGE,
-				category: ErrorCategory.USER,
-				details: {
-					resultId: input.id,
-					...input.experimentId ? { experimentId: input.experimentId } : {}
-				}
-			});
-			const result = await this.getExperimentResultById({ id: input.id });
-			if (!result) throw new MastraError({
-				id: createStorageErrorId("LIBSQL", "UPDATE_EXPERIMENT_RESULT", "NOT_FOUND"),
-				domain: ErrorDomain.STORAGE,
-				category: ErrorCategory.USER,
-				details: { resultId: input.id }
+				const result = await tx.execute({
+					sql: `SELECT ${buildSelectColumns(TABLE_EXPERIMENT_RESULTS)} FROM ${TABLE_EXPERIMENT_RESULTS} WHERE "id" = ?`,
+					args: [input.id]
+				});
+				return this.transformExperimentResultRow(result.rows[0]);
 			});
-			return result;
 		} catch (error) {
 			if (error instanceof MastraError) throw error;
 			throw new MastraError({
@@ -9352,6 +9421,10 @@ var MemoryLibSQL = class MemoryLibSQL extends MemoryStorage {
 					} catch {
 						existingChunks = [];
 					}
+					if (existingChunks.some((existing) => existing.cycleId === input.chunk.cycleId)) {
+						await tx.commit();
+						return;
+					}
 					const newChunk = {
 						id: `ombuf-${randomUUID()}`,
 						cycleId: input.chunk.cycleId,
@@ -10458,6 +10531,7 @@ var ObservabilityLibSQL = class ObservabilityLibSQL extends ObservabilityStorage
 		}
 	}
 	async batchDeleteTraces(args) {
+		this.assertUnscopedBatchDeleteTraces(args);
 		try {
 			const keys = args.traceIds.map((traceId) => ({ traceId }));
 			return this.#db.batchDelete({