@mastra/libsql 1.0.0-beta.13 → 1.0.0-beta.14

package/dist/index.js

@@ -1,8 +1,8 @@
 import { createClient } from '@libsql/client';
 import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
-import { createVectorErrorId, AgentsStorage, AGENTS_SCHEMA, TABLE_AGENTS, createStorageErrorId, normalizePerPage, calculatePagination, MemoryStorage, TABLE_SCHEMAS, TABLE_THREADS, TABLE_MESSAGES, TABLE_RESOURCES, ObservabilityStorage, SPAN_SCHEMA, TABLE_SPANS, listTracesArgsSchema, ScoresStorage, SCORERS_SCHEMA, TABLE_SCORERS, transformScoreRow, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, MastraStorage, TraceStatus, getSqlType, safelyParseJSON } from '@mastra/core/storage';
+import { createVectorErrorId, AgentsStorage, AGENTS_SCHEMA, TABLE_AGENTS, createStorageErrorId, normalizePerPage, calculatePagination, MemoryStorage, TABLE_SCHEMAS, TABLE_THREADS, TABLE_MESSAGES, TABLE_RESOURCES, ObservabilityStorage, SPAN_SCHEMA, TABLE_SPANS, listTracesArgsSchema, ScoresStorage, SCORERS_SCHEMA, TABLE_SCORERS, transformScoreRow, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, MastraCompositeStore, TraceStatus, getSqlType, safelyParseJSON } from '@mastra/core/storage';
 import { parseSqlIdentifier, parseFieldKey } from '@mastra/core/utils';
-import { MastraVector } from '@mastra/core/vector';
+import { MastraVector, validateTopK, validateUpsertInput } from '@mastra/core/vector';
 import { BaseFilterTranslator } from '@mastra/core/vector/filter';
 import { MastraBase } from '@mastra/core/base';
 import { MessageList } from '@mastra/core/agent';
@@ -242,10 +242,10 @@ var FILTER_OPERATORS = {
     };
   },
   // Element Operators
-  $exists: (key) => {
+  $exists: (key, value) => {
     const jsonPath = getJsonPath(key);
     return {
-      sql: `json_extract(metadata, ${jsonPath}) IS NOT NULL`,
+      sql: value === false ? `json_extract(metadata, ${jsonPath}) IS NULL` : `json_extract(metadata, ${jsonPath}) IS NOT NULL`,
       needsValue: false
     };
   },
@@ -538,7 +538,7 @@ var LibSQLVector = class extends MastraVector {
       try {
         return await operation();
       } catch (error) {
-        if (error.code === "SQLITE_BUSY" || error.message && error.message.toLowerCase().includes("database is locked")) {
+        if (error.code === "SQLITE_BUSY" || error.code === "SQLITE_LOCKED" || error.code === "SQLITE_LOCKED_SHAREDCACHE" || error.message && error.message.toLowerCase().includes("database is locked") || error.message && error.message.toLowerCase().includes("database table is locked")) {
           attempts++;
           if (attempts >= this.maxRetries) {
             this.logger.error(
@@ -572,22 +572,14 @@ var LibSQLVector = class extends MastraVector {
     minScore = -1
     // Default to -1 to include all results (cosine similarity ranges from -1 to 1)
   }) {
-    try {
-      if (!Number.isInteger(topK) || topK <= 0) {
-        throw new Error("topK must be a positive integer");
-      }
-      if (!Array.isArray(queryVector) || !queryVector.every((x) => typeof x === "number" && Number.isFinite(x))) {
-        throw new Error("queryVector must be an array of finite numbers");
-      }
-    } catch (error) {
-      throw new MastraError(
-        {
-          id: createVectorErrorId("LIBSQL", "QUERY", "INVALID_ARGS"),
-          domain: ErrorDomain.STORAGE,
-          category: ErrorCategory.USER
-        },
-        error
-      );
+    validateTopK("LIBSQL", topK);
+    if (!Array.isArray(queryVector) || !queryVector.every((x) => typeof x === "number" && Number.isFinite(x))) {
+      throw new MastraError({
+        id: createVectorErrorId("LIBSQL", "QUERY", "INVALID_ARGS"),
+        domain: ErrorDomain.STORAGE,
+        category: ErrorCategory.USER,
+        details: { message: "queryVector must be an array of finite numbers" }
+      });
     }
     try {
       const parsedIndexName = parseSqlIdentifier(indexName, "index name");
@@ -647,6 +639,7 @@ var LibSQLVector = class extends MastraVector {
     }
   }
   async doUpsert({ indexName, vectors, metadata, ids }) {
+    validateUpsertInput("LIBSQL", vectors, metadata, ids);
     const tx = await this.turso.transaction("write");
     try {
       const parsedIndexName = parseSqlIdentifier(indexName, "index name");
@@ -1710,6 +1703,9 @@ var LibSQLDB = class extends MastraBase {
       if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
         tableConstraints.push("UNIQUE (workflow_name, run_id)");
       }
+      if (tableName === TABLE_SPANS) {
+        tableConstraints.push("UNIQUE (spanId, traceId)");
+      }
       const allDefinitions = [...columnDefinitions, ...tableConstraints].join(",\n  ");
       const sql = `CREATE TABLE IF NOT EXISTS ${parsedTableName} (
   ${allDefinitions}
@@ -1720,6 +1716,9 @@ var LibSQLDB = class extends MastraBase {
         await this.migrateSpansTable();
       }
     } catch (error) {
+      if (error instanceof MastraError) {
+        throw error;
+      }
       throw new MastraError(
         {
           id: createStorageErrorId("LIBSQL", "CREATE_TABLE", "FAILED"),
@@ -1733,7 +1732,7 @@ var LibSQLDB = class extends MastraBase {
   }
   /**
    * Migrates the spans table schema from OLD_SPAN_SCHEMA to current SPAN_SCHEMA.
-   * This adds new columns that don't exist in old schema.
+   * This adds new columns that don't exist in old schema and ensures required indexes exist.
    */
   async migrateSpansTable() {
     const schema = TABLE_SCHEMAS[TABLE_SPANS];
@@ -1747,11 +1746,205 @@ var LibSQLDB = class extends MastraBase {
           this.logger.debug(`LibSQLDB: Added column '${columnName}' to ${TABLE_SPANS}`);
         }
       }
+      const indexExists = await this.spansUniqueIndexExists();
+      if (!indexExists) {
+        const duplicateInfo = await this.checkForDuplicateSpans();
+        if (duplicateInfo.hasDuplicates) {
+          const errorMessage = `
+===========================================================================
+MIGRATION REQUIRED: Duplicate spans detected in ${TABLE_SPANS}
+===========================================================================
+
+Found ${duplicateInfo.duplicateCount} duplicate (traceId, spanId) combinations.
+
+The spans table requires a unique constraint on (traceId, spanId), but your
+database contains duplicate entries that must be resolved first.
+
+To fix this, run the manual migration command:
+
+  npx mastra migrate
+
+This command will:
+  1. Remove duplicate spans (keeping the most complete/recent version)
+  2. Add the required unique constraint
+
+Note: This migration may take some time for large tables.
+===========================================================================
+`;
+          throw new MastraError({
+            id: createStorageErrorId("LIBSQL", "MIGRATION_REQUIRED", "DUPLICATE_SPANS"),
+            domain: ErrorDomain.STORAGE,
+            category: ErrorCategory.USER,
+            text: errorMessage
+          });
+        } else {
+          await this.client.execute(
+            `CREATE UNIQUE INDEX IF NOT EXISTS "mastra_ai_spans_spanid_traceid_idx" ON "${TABLE_SPANS}" ("spanId", "traceId")`
+          );
+          this.logger.debug(`LibSQLDB: Created unique index on (spanId, traceId) for ${TABLE_SPANS}`);
+        }
+      }
       this.logger.info(`LibSQLDB: Migration completed for ${TABLE_SPANS}`);
     } catch (error) {
+      if (error instanceof MastraError) {
+        throw error;
+      }
       this.logger.warn(`LibSQLDB: Failed to migrate spans table ${TABLE_SPANS}:`, error);
     }
   }
+  /**
+   * Checks if the unique index on (spanId, traceId) already exists on the spans table.
+   * Used to skip deduplication when the index already exists (migration already complete).
+   */
+  async spansUniqueIndexExists() {
+    try {
+      const result = await this.client.execute(
+        `SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = 'mastra_ai_spans_spanid_traceid_idx'`
+      );
+      return (result.rows?.length ?? 0) > 0;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Checks for duplicate (traceId, spanId) combinations in the spans table.
+   * Returns information about duplicates for logging/CLI purposes.
+   */
+  async checkForDuplicateSpans() {
+    try {
+      const result = await this.client.execute(`
+        SELECT COUNT(*) as duplicate_count FROM (
+          SELECT "spanId", "traceId"
+          FROM "${TABLE_SPANS}"
+          GROUP BY "spanId", "traceId"
+          HAVING COUNT(*) > 1
+        )
+      `);
+      const duplicateCount = Number(result.rows?.[0]?.duplicate_count ?? 0);
+      return {
+        hasDuplicates: duplicateCount > 0,
+        duplicateCount
+      };
+    } catch (error) {
+      this.logger.debug(`LibSQLDB: Could not check for duplicates: ${error}`);
+      return { hasDuplicates: false, duplicateCount: 0 };
+    }
+  }
+  /**
+   * Manually run the spans migration to deduplicate and add the unique constraint.
+   * This is intended to be called from the CLI when duplicates are detected.
+   *
+   * @returns Migration result with status and details
+   */
+  async migrateSpans() {
+    const indexExists = await this.spansUniqueIndexExists();
+    if (indexExists) {
+      return {
+        success: true,
+        alreadyMigrated: true,
+        duplicatesRemoved: 0,
+        message: `Migration already complete. Unique index exists on ${TABLE_SPANS}.`
+      };
+    }
+    const duplicateInfo = await this.checkForDuplicateSpans();
+    if (duplicateInfo.hasDuplicates) {
+      this.logger.info(
+        `Found ${duplicateInfo.duplicateCount} duplicate (traceId, spanId) combinations. Starting deduplication...`
+      );
+      await this.deduplicateSpans();
+    } else {
+      this.logger.info(`No duplicate spans found.`);
+    }
+    await this.client.execute(
+      `CREATE UNIQUE INDEX IF NOT EXISTS "mastra_ai_spans_spanid_traceid_idx" ON "${TABLE_SPANS}" ("spanId", "traceId")`
+    );
+    return {
+      success: true,
+      alreadyMigrated: false,
+      duplicatesRemoved: duplicateInfo.duplicateCount,
+      message: duplicateInfo.hasDuplicates ? `Migration complete. Removed duplicates and added unique index to ${TABLE_SPANS}.` : `Migration complete. Added unique index to ${TABLE_SPANS}.`
+    };
+  }
+  /**
+   * Check migration status for the spans table.
+   * Returns information about whether migration is needed.
+   */
+  async checkSpansMigrationStatus() {
+    const indexExists = await this.spansUniqueIndexExists();
+    if (indexExists) {
+      return {
+        needsMigration: false,
+        hasDuplicates: false,
+        duplicateCount: 0,
+        constraintExists: true,
+        tableName: TABLE_SPANS
+      };
+    }
+    const duplicateInfo = await this.checkForDuplicateSpans();
+    return {
+      needsMigration: true,
+      hasDuplicates: duplicateInfo.hasDuplicates,
+      duplicateCount: duplicateInfo.duplicateCount,
+      constraintExists: false,
+      tableName: TABLE_SPANS
+    };
+  }
+  /**
+   * Deduplicates spans table by removing duplicate (spanId, traceId) combinations.
+   * Keeps the "best" record for each duplicate group based on:
+   * 1. Completed spans (endedAt IS NOT NULL) over incomplete ones
+   * 2. Most recently updated (updatedAt DESC)
+   * 3. Most recently created (createdAt DESC) as tiebreaker
+   */
+  async deduplicateSpans() {
+    try {
+      const duplicateCheck = await this.client.execute(`
+        SELECT COUNT(*) as duplicate_count FROM (
+          SELECT "spanId", "traceId"
+          FROM "${TABLE_SPANS}"
+          GROUP BY "spanId", "traceId"
+          HAVING COUNT(*) > 1
+        )
+      `);
+      const duplicateCount = Number(duplicateCheck.rows?.[0]?.duplicate_count ?? 0);
+      if (duplicateCount === 0) {
+        this.logger.debug(`LibSQLDB: No duplicate spans found, skipping deduplication`);
+        return;
+      }
+      this.logger.warn(`LibSQLDB: Found ${duplicateCount} duplicate (spanId, traceId) combinations, deduplicating...`);
+      const deleteResult = await this.client.execute(`
+        DELETE FROM "${TABLE_SPANS}"
+        WHERE rowid NOT IN (
+          SELECT MIN(best_rowid) FROM (
+            SELECT
+              rowid as best_rowid,
+              "spanId",
+              "traceId",
+              ROW_NUMBER() OVER (
+                PARTITION BY "spanId", "traceId"
+                ORDER BY
+                  CASE WHEN "endedAt" IS NOT NULL THEN 0 ELSE 1 END,
+                  "updatedAt" DESC,
+                  "createdAt" DESC
+              ) as rn
+            FROM "${TABLE_SPANS}"
+          ) ranked
+          WHERE rn = 1
+          GROUP BY "spanId", "traceId"
+        )
+        AND ("spanId", "traceId") IN (
+          SELECT "spanId", "traceId"
+          FROM "${TABLE_SPANS}"
+          GROUP BY "spanId", "traceId"
+          HAVING COUNT(*) > 1
+        )
+      `);
+      const deletedCount = deleteResult.rowsAffected ?? 0;
+      this.logger.warn(`LibSQLDB: Deleted ${deletedCount} duplicate span records`);
+    } catch (error) {
+      this.logger.warn(`LibSQLDB: Failed to deduplicate spans:`, error);
+    }
+  }
   /**
    * Gets a default value for a column type (used when adding NOT NULL columns).
    */
@@ -3019,6 +3212,22 @@ var ObservabilityLibSQL = class extends ObservabilityStorage {
   async dangerouslyClearAll() {
     await this.#db.deleteData({ tableName: TABLE_SPANS });
   }
+  /**
+   * Manually run the spans migration to deduplicate and add the unique constraint.
+   * This is intended to be called from the CLI when duplicates are detected.
+   *
+   * @returns Migration result with status and details
+   */
+  async migrateSpans() {
+    return this.#db.migrateSpans();
+  }
+  /**
+   * Check migration status for the spans table.
+   * Returns information about whether migration is needed.
+   */
+  async checkSpansMigrationStatus() {
+    return this.#db.checkSpansMigrationStatus();
+  }
   get tracingStrategy() {
     return {
       preferred: "batch-with-updates",
@@ -4054,7 +4263,7 @@ var WorkflowsLibSQL = class extends WorkflowsStorage {
 };
 
 // src/storage/index.ts
-var LibSQLStore = class extends MastraStorage {
+var LibSQLStore = class extends MastraCompositeStore {
   client;
   maxRetries;
   initialBackoffMs;