@mastra/libsql 1.15.1 → 1.16.0-alpha.1

package/dist/index.cjs

@@ -545,6 +545,31 @@ var LibSQLVector = class extends vector.MastraVector {
     }
     this.vectorIndexes = this.isMemoryDb ? Promise.resolve(/* @__PURE__ */ new Set()) : this.discoverVectorIndexes();
   }
+  /**
+   * Closes the underlying libsql client, releasing all OS file handles.
+   *
+   * For local file databases, first runs PRAGMA wal_checkpoint(TRUNCATE) and
+   * switches back to journal_mode=DELETE so the -wal and -shm sidecar files
+   * are released promptly (mirrors LibSQLStore.close()).
+   *
+   * Remote (Turso) databases skip the WAL pragmas and just close the client.
+   *
+   * Safe to call more than once; subsequent calls are no-ops.
+   */
+  async close() {
+    if (this.turso.closed) {
+      return;
+    }
+    if (this.turso.protocol === "file" && !this.isMemoryDb) {
+      try {
+        await this.turso.execute("PRAGMA wal_checkpoint(TRUNCATE);");
+        await this.turso.execute("PRAGMA journal_mode=DELETE;");
+      } catch (err) {
+        this.logger.warn("LibSQLVector: Failed to checkpoint WAL before close.", err);
+      }
+    }
+    this.turso.close();
+  }
   async discoverVectorIndexes() {
     try {
       const result = await this.turso.execute({
@@ -1572,6 +1597,19 @@ var LibSQLDB = class extends base.MastraBase {
   insert(args) {
     return this.executeWriteOperationWithRetry(() => this.doInsert(args), `insert into table ${args.tableName}`);
   }
+  /** Inserts a record without replacing an existing primary key. */
+  insertOnly(args) {
+    return this.executeWriteOperationWithRetry(async () => {
+      const filteredRecord = await this.filterRecordToKnownColumns(args.tableName, args.record);
+      if (Object.keys(filteredRecord).length === 0) return;
+      const statement = prepareStatement({ tableName: args.tableName, record: filteredRecord });
+      if (!statement.sql.startsWith("INSERT OR REPLACE")) {
+        throw new Error(`Unexpected insert statement generated for table ${args.tableName}`);
+      }
+      statement.sql = statement.sql.replace("INSERT OR REPLACE", "INSERT");
+      await withClientWriteLock(this.client, () => this.client.execute(statement));
+    }, `insert into table ${args.tableName}`);
+  }
   /**
    * Internal update implementation without retry logic.
    */
@@ -3881,6 +3919,7 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
     await this.#addColumnIfNotExists(storage.TABLE_DATASET_ITEMS, "organizationId", "TEXT");
     await this.#addColumnIfNotExists(storage.TABLE_DATASET_ITEMS, "projectId", "TEXT");
     await this.#addColumnIfNotExists(storage.TABLE_DATASET_ITEMS, "toolMocks", "TEXT");
+    await this.#addColumnIfNotExists(storage.TABLE_DATASET_ITEMS, "externalId", "TEXT");
     await this.#client.batch(
       [
         {
@@ -3891,6 +3930,10 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
           sql: `CREATE INDEX IF NOT EXISTS idx_dataset_items_dataset_version ON "${storage.TABLE_DATASET_ITEMS}" ("datasetId", "datasetVersion")`,
           args: []
         },
+        {
+          sql: `CREATE INDEX IF NOT EXISTS idx_dataset_items_dataset_externalid_version ON "${storage.TABLE_DATASET_ITEMS}" ("datasetId", "externalId", "datasetVersion")`,
+          args: []
+        },
         {
           sql: `CREATE INDEX IF NOT EXISTS idx_dataset_items_dataset_validto_deleted ON "${storage.TABLE_DATASET_ITEMS}" ("datasetId", "validTo", "isDeleted")`,
           args: []
@@ -3970,6 +4013,7 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
       id: row.id,
       datasetId: row.datasetId,
       datasetVersion: row.datasetVersion,
+      externalId: row.externalId ?? null,
       organizationId: row.organizationId ?? null,
       projectId: row.projectId ?? null,
       input: storage.safelyParseJSON(row.input),
@@ -3988,6 +4032,7 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
       id: row.id,
       datasetId: row.datasetId,
       datasetVersion: row.datasetVersion,
+      externalId: row.externalId ?? null,
       organizationId: row.organizationId ?? null,
       projectId: row.projectId ?? null,
       validTo: row.validTo,
@@ -4014,10 +4059,11 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
   // --- Dataset CRUD ---
   async createDataset(input) {
     try {
-      const id = crypto.randomUUID();
+      const id = input.id ?? crypto.randomUUID();
+      if (input.id !== void 0) this.validateCallerDefinedDatasetId(input.id);
       const now = /* @__PURE__ */ new Date();
       const nowIso = now.toISOString();
-      await this.#db.insert({
+      await this.#db.insertOnly({
         tableName: storage.TABLE_DATASETS,
         record: {
           id,
@@ -4059,6 +4105,11 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
         updatedAt: now
       };
     } catch (error$1) {
+      if (input.id !== void 0 && storage.hasErrorCode(error$1, /* @__PURE__ */ new Set(["SQLITE_CONSTRAINT", "SQLITE_CONSTRAINT_PRIMARYKEY", "SQLITE_CONSTRAINT_UNIQUE"]))) {
+        const existing = await this.getDatasetById({ id: input.id });
+        if (existing) return this.resolveExistingDataset(existing, { ...input, id: input.id });
+      }
+      if (error$1 instanceof error.MastraError) throw error$1;
       throw new error.MastraError(
         {
           id: storage.createStorageErrorId("LIBSQL", "CREATE_DATASET", "FAILED"),
@@ -4304,11 +4355,12 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
             args: [args.datasetId]
           },
           {
-            sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT organizationId FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT projectId FROM ${storage.TABLE_DATASETS} WHERE id = ?), NULL, 0, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
+            sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, externalId, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), ?, (SELECT organizationId FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT projectId FROM ${storage.TABLE_DATASETS} WHERE id = ?), NULL, 0, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
             args: [
               id,
               args.datasetId,
               args.datasetId,
+              args.externalId ?? null,
               args.datasetId,
               args.datasetId,
               jsonbArg(args.input),
@@ -4334,6 +4386,7 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
         id,
         datasetId: args.datasetId,
         datasetVersion: newVersion,
+        externalId: args.externalId ?? null,
         organizationId: dataset?.organizationId ?? null,
         projectId: dataset?.projectId ?? null,
         input: args.input,
@@ -4399,11 +4452,12 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
             args: [args.datasetId, args.id]
           },
           {
-            sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT organizationId FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT projectId FROM ${storage.TABLE_DATASETS} WHERE id = ?), NULL, 0, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
+            sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, externalId, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), ?, (SELECT organizationId FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT projectId FROM ${storage.TABLE_DATASETS} WHERE id = ?), NULL, 0, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
             args: [
               args.id,
               args.datasetId,
               args.datasetId,
+              existing.externalId ?? null,
               args.datasetId,
               args.datasetId,
               jsonbArg(mergedInput),
@@ -4476,11 +4530,12 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
             args: [datasetId, id]
           },
           {
-            sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT organizationId FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT projectId FROM ${storage.TABLE_DATASETS} WHERE id = ?), NULL, 1, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
+            sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, externalId, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), ?, (SELECT organizationId FROM ${storage.TABLE_DATASETS} WHERE id = ?), (SELECT projectId FROM ${storage.TABLE_DATASETS} WHERE id = ?), NULL, 1, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
             args: [
               id,
               datasetId,
               datasetId,
+              existing.externalId ?? null,
               datasetId,
               datasetId,
               jsonbArg(existing.input),
@@ -4762,70 +4817,94 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
   // --- Bulk operations (SCD-2 internally) ---
   async _doBatchInsertItems(input) {
     try {
-      const dataset = await this.getDatasetById({ id: input.datasetId });
-      if (!dataset) {
-        throw new error.MastraError({
-          id: storage.createStorageErrorId("LIBSQL", "BULK_ADD_ITEMS", "DATASET_NOT_FOUND"),
-          domain: error.ErrorDomain.STORAGE,
-          category: error.ErrorCategory.USER,
-          details: { datasetId: input.datasetId }
-        });
-      }
-      const now = /* @__PURE__ */ new Date();
-      const nowIso = now.toISOString();
-      const versionId = crypto.randomUUID();
-      const statements = [
-        {
-          sql: `UPDATE ${storage.TABLE_DATASETS} SET version = version + 1 WHERE id = ? RETURNING version`,
-          args: [input.datasetId]
+      if (input.items.length === 0) return [];
+      return await withClientWriteLock(this.#client, async () => {
+        const tx = await this.#client.transaction("write");
+        try {
+          const datasetResult = await tx.execute({
+            sql: `SELECT version, organizationId, projectId FROM ${storage.TABLE_DATASETS} WHERE id = ?`,
+            args: [input.datasetId]
+          });
+          const dataset = datasetResult.rows[0];
+          if (!dataset)
+            throw new error.MastraError({
+              id: storage.createStorageErrorId("LIBSQL", "BULK_ADD_ITEMS", "DATASET_NOT_FOUND"),
+              domain: error.ErrorDomain.STORAGE,
+              category: error.ErrorCategory.USER,
+              details: { datasetId: input.datasetId }
+            });
+          const externalIds = [...new Set(input.items.flatMap((item) => item.externalId ? [item.externalId] : []))];
+          const historyResult = externalIds.length ? await tx.execute({
+            sql: `SELECT id, datasetId, datasetVersion, externalId, organizationId, projectId, validTo, isDeleted, json(input) AS input, json(groundTruth) AS groundTruth, json(expectedTrajectory) AS expectedTrajectory, json(toolMocks) AS toolMocks, json(requestContext) AS requestContext, json(metadata) AS metadata, json(source) AS source, createdAt, updatedAt FROM ${storage.TABLE_DATASET_ITEMS} WHERE datasetId = ? AND externalId IN (${externalIds.map(() => "?").join(",")}) ORDER BY datasetVersion`,
+            args: [input.datasetId, ...externalIds]
+          }) : { rows: [] };
+          const plan = this.planDatasetItemBatch(
+            input.items,
+            historyResult.rows.map((row) => this.transformItemRowFull(row)),
+            () => crypto.randomUUID()
+          );
+          const resolved = new Map(
+            [...plan.existingCurrentItems].map(([id, row]) => [id, this.datasetItemFromRow(row)])
+          );
+          if (plan.inserts.length > 0) {
+            const newVersion = Number(dataset.version) + 1;
+            const now = /* @__PURE__ */ new Date();
+            const nowIso = now.toISOString();
+            await tx.execute({
+              sql: `UPDATE ${storage.TABLE_DATASETS} SET version = ? WHERE id = ?`,
+              args: [newVersion, input.datasetId]
+            });
+            for (const { id, item } of plan.inserts) {
+              await tx.execute({
+                sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, externalId, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?, NULL, 0, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
+                args: [
+                  id,
+                  input.datasetId,
+                  newVersion,
+                  item.externalId ?? null,
+                  dataset.organizationId ?? null,
+                  dataset.projectId ?? null,
+                  jsonbArg(item.input),
+                  jsonbArg(item.groundTruth),
+                  jsonbArg(item.expectedTrajectory),
+                  jsonbArg(item.toolMocks),
+                  jsonbArg(item.requestContext),
+                  jsonbArg(item.metadata),
+                  jsonbArg(item.source),
+                  nowIso,
+                  nowIso
+                ]
+              });
+              resolved.set(id, {
+                id,
+                datasetId: input.datasetId,
+                datasetVersion: newVersion,
+                externalId: item.externalId ?? null,
+                organizationId: dataset.organizationId,
+                projectId: dataset.projectId,
+                input: item.input,
+                groundTruth: item.groundTruth,
+                expectedTrajectory: item.expectedTrajectory,
+                toolMocks: item.toolMocks,
+                requestContext: item.requestContext,
+                metadata: item.metadata,
+                source: item.source,
+                createdAt: now,
+                updatedAt: now
+              });
+            }
+            await tx.execute({
+              sql: `INSERT INTO ${storage.TABLE_DATASET_VERSIONS} (id, datasetId, version, createdAt) VALUES (?, ?, ?, ?)`,
+              args: [crypto.randomUUID(), input.datasetId, newVersion, nowIso]
+            });
+          }
+          await tx.commit();
+          return plan.resolvedIds.map((id) => resolved.get(id));
+        } catch (error) {
+          if (!tx.closed) await tx.rollback();
+          throw error;
         }
-      ];
-      const items = [];
-      for (const itemInput of input.items) {
-        const id = crypto.randomUUID();
-        items.push({ id, input: itemInput });
-        statements.push({
-          sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), ?, ?, NULL, 0, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
-          args: [
-            id,
-            input.datasetId,
-            input.datasetId,
-            dataset.organizationId ?? null,
-            dataset.projectId ?? null,
-            jsonbArg(itemInput.input),
-            jsonbArg(itemInput.groundTruth),
-            jsonbArg(itemInput.expectedTrajectory),
-            jsonbArg(itemInput.toolMocks),
-            jsonbArg(itemInput.requestContext),
-            jsonbArg(itemInput.metadata),
-            jsonbArg(itemInput.source),
-            nowIso,
-            nowIso
-          ]
-        });
-      }
-      statements.push({
-        sql: `INSERT INTO ${storage.TABLE_DATASET_VERSIONS} (id, datasetId, version, createdAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), ?)`,
-        args: [versionId, input.datasetId, input.datasetId, nowIso]
       });
-      const results = await this.#client.batch(statements, "write");
-      const newVersion = Number(results[0].rows[0].version);
-      return items.map(({ id, input: itemInput }) => ({
-        id,
-        datasetId: input.datasetId,
-        datasetVersion: newVersion,
-        organizationId: dataset.organizationId ?? null,
-        projectId: dataset.projectId ?? null,
-        input: itemInput.input,
-        groundTruth: itemInput.groundTruth,
-        expectedTrajectory: itemInput.expectedTrajectory,
-        toolMocks: itemInput.toolMocks,
-        requestContext: itemInput.requestContext,
-        metadata: itemInput.metadata,
-        source: itemInput.source,
-        createdAt: now,
-        updatedAt: now
-      }));
     } catch (error$1) {
       if (error$1 instanceof error.MastraError) throw error$1;
       throw new error.MastraError(
@@ -4871,11 +4950,12 @@ var DatasetsLibSQL = class extends storage.DatasetsStorage {
           args: [input.datasetId, item.id]
         });
         statements.push({
-          sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), ?, ?, NULL, 1, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
+          sql: `INSERT INTO ${storage.TABLE_DATASET_ITEMS} (id, datasetId, datasetVersion, externalId, organizationId, projectId, validTo, isDeleted, input, groundTruth, expectedTrajectory, toolMocks, requestContext, metadata, source, createdAt, updatedAt) VALUES (?, ?, (SELECT version FROM ${storage.TABLE_DATASETS} WHERE id = ?), ?, ?, ?, NULL, 1, jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), jsonb(?), ?, ?)`,
           args: [
             item.id,
             input.datasetId,
             input.datasetId,
+            item.externalId ?? null,
             dataset.organizationId ?? null,
             dataset.projectId ?? null,
             jsonbArg(item.input),