@mastra/duckdb 1.1.2 → 1.2.0-alpha.1

package/dist/{observability-EMLEGDF7.js → observability-YJBOVLPV.js}

@@ -1,5 +1,6 @@
 import { DuckDBConnection } from './chunk-37GBWD4M.js';
-import { ObservabilityStorage, toTraceSpans } from '@mastra/core/storage';
+import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
+import { ObservabilityStorage, createStorageErrorId, toTraceSpans } from '@mastra/core/storage';
 import { EntityType } from '@mastra/core/observability';
 import { parseFieldKey } from '@mastra/core/utils';
 
@@ -57,6 +58,7 @@ CREATE TABLE IF NOT EXISTS metric_events (
   timestamp TIMESTAMP NOT NULL,
 
   -- IDs
+  metricId VARCHAR NOT NULL PRIMARY KEY,
   traceId VARCHAR,
   spanId VARCHAR,
   experimentId VARCHAR,
@@ -108,6 +110,7 @@ CREATE TABLE IF NOT EXISTS log_events (
   timestamp TIMESTAMP NOT NULL,
 
   -- IDs
+  logId VARCHAR NOT NULL PRIMARY KEY,
   traceId VARCHAR,
   spanId VARCHAR,
   experimentId VARCHAR,
@@ -154,6 +157,7 @@ CREATE TABLE IF NOT EXISTS score_events (
   timestamp TIMESTAMP NOT NULL,
 
   -- IDs
+  scoreId VARCHAR NOT NULL PRIMARY KEY,
   traceId VARCHAR,
   spanId VARCHAR,
   experimentId VARCHAR,
@@ -204,6 +208,7 @@ CREATE TABLE IF NOT EXISTS feedback_events (
   timestamp TIMESTAMP NOT NULL,
 
   -- IDs
+  feedbackId VARCHAR NOT NULL PRIMARY KEY,
   traceId VARCHAR,
   spanId VARCHAR,
   experimentId VARCHAR,
@@ -717,6 +722,7 @@ function rowToFeedbackRecord(row) {
   const numValue = Number(rawValue);
   if (!isNaN(numValue)) value = numValue;
   return {
+    feedbackId: row.feedbackId,
     timestamp: toDate(row.timestamp),
     traceId: row.traceId ?? null,
     spanId: row.spanId ?? null,
@@ -788,12 +794,13 @@ async function createFeedback(db, args) {
   const feedbackUserId = f.feedbackUserId ?? f.userId ?? null;
   await db.execute(
     `INSERT INTO feedback_events (
-      timestamp, traceId, spanId, experimentId,
+      feedbackId, timestamp, traceId, spanId, experimentId,
       entityType, entityId, entityName, entityVersionId, parentEntityVersionId, parentEntityType, parentEntityId, parentEntityName, rootEntityVersionId, rootEntityType, rootEntityId, rootEntityName,
       userId, organizationId, resourceId, runId, sessionId, threadId, requestId, environment, executionSource, serviceName,
       feedbackUserId, sourceId, feedbackSource, feedbackType, value, comment, tags, metadata, scope
     )
      VALUES (${[
+      v(f.feedbackId),
       v(f.timestamp),
       v(f.traceId),
       v(f.spanId ?? null),
@@ -829,7 +836,8 @@ async function createFeedback(db, args) {
       jsonV(f.tags ?? null),
       jsonV(f.metadata),
       jsonV(f.scope ?? null)
-    ].join(", ")})`
+    ].join(", ")})
+     ON CONFLICT DO NOTHING`
   );
 }
 async function batchCreateFeedback(db, args) {
@@ -839,6 +847,7 @@ async function batchCreateFeedback(db, args) {
     const feedbackSource = legacyFeedback.feedbackSource ?? legacyFeedback.source ?? "";
     const feedbackUserId = legacyFeedback.feedbackUserId ?? legacyFeedback.userId ?? null;
     return `(${[
+      v(legacyFeedback.feedbackId),
       v(legacyFeedback.timestamp),
       v(legacyFeedback.traceId),
       v(legacyFeedback.spanId ?? null),
@@ -878,12 +887,13 @@ async function batchCreateFeedback(db, args) {
   });
   await db.execute(
     `INSERT INTO feedback_events (
-      timestamp, traceId, spanId, experimentId,
+      feedbackId, timestamp, traceId, spanId, experimentId,
       entityType, entityId, entityName, entityVersionId, parentEntityVersionId, parentEntityType, parentEntityId, parentEntityName, rootEntityVersionId, rootEntityType, rootEntityId, rootEntityName,
       userId, organizationId, resourceId, runId, sessionId, threadId, requestId, environment, executionSource, serviceName,
       feedbackUserId, sourceId, feedbackSource, feedbackType, value, comment, tags, metadata, scope
     )
-     VALUES ${tuples.join(",\n       ")}`
+     VALUES ${tuples.join(",\n       ")}
+     ON CONFLICT DO NOTHING`
   );
 }
 async function listFeedback(db, args) {
@@ -1040,6 +1050,7 @@ async function getFeedbackPercentiles(db, args) {
 
 // src/storage/domains/observability/logs.ts
 var COLUMNS = [
+  "logId",
   "timestamp",
   "level",
   "message",
@@ -1076,6 +1087,7 @@ var COLUMNS = [
 var COLUMNS_SQL = COLUMNS.join(", ");
 function rowToLogRecord(row) {
   return {
+    logId: row.logId,
     timestamp: toDate(row.timestamp),
     level: row.level,
     message: row.message,
@@ -1114,6 +1126,7 @@ async function batchCreateLogs(db, args) {
   if (args.logs.length === 0) return;
   const tuples = args.logs.map((log) => {
     return `(${[
+      v(log.logId),
       v(log.timestamp),
       v(log.level),
       v(log.message),
@@ -1148,7 +1161,7 @@ async function batchCreateLogs(db, args) {
       jsonV(log.scope)
     ].join(", ")})`;
   });
-  await db.execute(`INSERT INTO log_events (${COLUMNS_SQL}) VALUES ${tuples.join(",\n")}`);
+  await db.execute(`INSERT INTO log_events (${COLUMNS_SQL}) VALUES ${tuples.join(",\n")} ON CONFLICT DO NOTHING`);
 }
 async function listLogs(db, args) {
   const filters = args.filters ?? {};
@@ -1215,6 +1228,7 @@ function buildMetricNameFilter(name) {
   return { clause: `name = ?`, params: [name] };
 }
 var METRIC_COLUMNS = [
+  "metricId",
   "timestamp",
   "name",
   "value",
@@ -1318,6 +1332,7 @@ function resolveGroupBy(groupBy) {
 }
 function rowToMetricRecord(row) {
   return {
+    metricId: row.metricId,
     timestamp: toDate(row.timestamp),
     name: row.name,
     value: Number(row.value),
@@ -1361,6 +1376,7 @@ async function batchCreateMetrics(db, args) {
   if (args.metrics.length === 0) return;
   const tuples = args.metrics.map((m) => {
     return `(${[
+      v(m.metricId),
       v(m.timestamp),
       v(m.name),
       v(m.value),
@@ -1400,7 +1416,9 @@ async function batchCreateMetrics(db, args) {
       jsonV(m.scope ?? null)
     ].join(", ")})`;
   });
-  await db.execute(`INSERT INTO metric_events (${METRIC_COLUMNS_SQL}) VALUES ${tuples.join(",\n")}`);
+  await db.execute(
+    `INSERT INTO metric_events (${METRIC_COLUMNS_SQL}) VALUES ${tuples.join(",\n")} ON CONFLICT DO NOTHING`
+  );
 }
 async function listMetrics(db, args) {
   const filters = args.filters ?? {};
@@ -1699,6 +1717,122 @@ async function getMetricLabelValues(db, args) {
   );
   return { values: rows.map((r) => r.val) };
 }
+var SIGNAL_MIGRATIONS = [
+  { table: "metric_events", createDDL: METRIC_EVENTS_DDL, idColumn: "metricId" },
+  { table: "log_events", createDDL: LOG_EVENTS_DDL, idColumn: "logId" },
+  { table: "score_events", createDDL: SCORE_EVENTS_DDL, idColumn: "scoreId" },
+  { table: "feedback_events", createDDL: FEEDBACK_EVENTS_DDL, idColumn: "feedbackId" }
+];
+async function tableExists(db, table) {
+  const rows = await db.query(
+    `SELECT table_name FROM information_schema.tables WHERE table_name = ?`,
+    [table]
+  );
+  return rows.length > 0;
+}
+async function hasPrimaryKey(db, table) {
+  const rows = await db.query(
+    `SELECT constraint_type FROM information_schema.table_constraints
+     WHERE table_name = ? AND constraint_type = 'PRIMARY KEY'`,
+    [table]
+  );
+  return rows.length > 0;
+}
+async function getColumns(db, table) {
+  const rows = await db.query(
+    `SELECT column_name FROM information_schema.columns WHERE table_name = ?`,
+    [table]
+  );
+  return rows.map((r) => r.column_name);
+}
+function buildTemporaryTableDDL(createDDL, table, tempTable) {
+  return createDDL.replace(`CREATE TABLE IF NOT EXISTS ${table}`, `CREATE TABLE ${tempTable}`);
+}
+async function dropTableIfExists(db, table) {
+  if (await tableExists(db, table)) {
+    await db.execute(`DROP TABLE ${table}`);
+  }
+}
+function createMigrationError(args, error) {
+  return new MastraError(
+    {
+      id: createStorageErrorId("DUCKDB", "MIGRATE_SIGNAL_TABLES", "FAILED"),
+      domain: ErrorDomain.STORAGE,
+      category: ErrorCategory.THIRD_PARTY,
+      details: args
+    },
+    error
+  );
+}
+async function checkSignalTablesMigrationStatus(db) {
+  const tables = [];
+  for (const { table, idColumn } of SIGNAL_MIGRATIONS) {
+    if (!await tableExists(db, table)) {
+      continue;
+    }
+    if (await hasPrimaryKey(db, table)) {
+      continue;
+    }
+    tables.push({ table, idColumn });
+  }
+  return {
+    needsMigration: tables.length > 0,
+    tables
+  };
+}
+async function migrateSignalTables(db, logger) {
+  for (const { table, createDDL, idColumn } of SIGNAL_MIGRATIONS) {
+    if (!await tableExists(db, table)) continue;
+    if (await hasPrimaryKey(db, table)) continue;
+    logger?.info?.(`Migrating ${table} to schema with ${idColumn} PRIMARY KEY`);
+    const temp = `${table}_migrating_${Date.now()}`;
+    const backup = `${table}_backup_${Date.now()}`;
+    let originalRenamed = false;
+    let swapCompleted = false;
+    try {
+      await db.execute(buildTemporaryTableDDL(createDDL, table, temp));
+      const newColumns = await getColumns(db, temp);
+      const currentColumns = new Set(await getColumns(db, table));
+      const columnList = newColumns.map((c) => `"${c}"`).join(", ");
+      const selectExprs = newColumns.map((c) => {
+        if (c === idColumn) {
+          return currentColumns.has(c) ? `COALESCE(NULLIF("${c}", ''), CAST(uuid() AS VARCHAR)) AS "${c}"` : `CAST(uuid() AS VARCHAR) AS "${c}"`;
+        }
+        return currentColumns.has(c) ? `"${c}"` : `NULL AS "${c}"`;
+      }).join(", ");
+      await db.execute(`INSERT INTO ${temp} (${columnList}) SELECT ${selectExprs} FROM ${table}`);
+      await db.execute(`ALTER TABLE ${table} RENAME TO ${backup}`);
+      originalRenamed = true;
+      await db.execute(`ALTER TABLE ${temp} RENAME TO ${table}`);
+      swapCompleted = true;
+      try {
+        await db.execute(`DROP TABLE ${backup}`);
+      } catch (cleanupError) {
+        logger?.warn?.(
+          `Migration of ${table} completed, but failed to drop backup ${backup}: ${cleanupError.message}`
+        );
+      }
+      logger?.info?.(`Successfully migrated ${table}`);
+    } catch (error) {
+      logger?.error?.(`Migration of ${table} failed: ${error.message}`);
+      try {
+        await dropTableIfExists(db, temp);
+      } catch (restoreError) {
+        logger?.error?.(`Failed to clean up temporary table ${temp}: ${restoreError.message}`);
+      }
+      if (originalRenamed && !swapCompleted) {
+        try {
+          await db.execute(`ALTER TABLE ${backup} RENAME TO ${table}`);
+        } catch (restoreError) {
+          logger?.error?.(
+            `Failed to restore original table ${table} from backup ${backup}: ${restoreError.message}`
+          );
+        }
+      }
+      throw createMigrationError({ table, idColumn }, error);
+    }
+  }
+}
 var SCORE_GROUP_BY_COLUMNS = /* @__PURE__ */ new Set([
   "timestamp",
   "traceId",
@@ -1814,6 +1948,7 @@ function toSeriesName2(values) {
 }
 function rowToScoreRecord(row) {
   return {
+    scoreId: row.scoreId,
     timestamp: toDate(row.timestamp),
     traceId: row.traceId ?? null,
     spanId: row.spanId ?? null,
@@ -1884,12 +2019,13 @@ async function createScore(db, args) {
   const scoreSource = s.scoreSource ?? s.source ?? null;
   await db.execute(
     `INSERT INTO score_events (
-      timestamp, traceId, spanId, experimentId, scoreTraceId,
+      scoreId, timestamp, traceId, spanId, experimentId, scoreTraceId,
       entityType, entityId, entityName, entityVersionId, parentEntityVersionId, parentEntityType, parentEntityId, parentEntityName, rootEntityVersionId, rootEntityType, rootEntityId, rootEntityName,
       userId, organizationId, resourceId, runId, sessionId, threadId, requestId, environment, executionSource, serviceName,
       scorerId, scorerVersion, scoreSource, score, reason, tags, metadata, scope
     )
      VALUES (${[
+      v(s.scoreId),
       v(s.timestamp),
       v(s.traceId),
       v(s.spanId ?? null),
@@ -1925,7 +2061,8 @@ async function createScore(db, args) {
       jsonV(s.tags ?? null),
       jsonV(s.metadata),
       jsonV(s.scope ?? null)
-    ].join(", ")})`
+    ].join(", ")})
+     ON CONFLICT DO NOTHING`
   );
 }
 async function batchCreateScores(db, args) {
@@ -1934,6 +2071,7 @@ async function batchCreateScores(db, args) {
     const legacyScore = s;
     const scoreSource = legacyScore.scoreSource ?? legacyScore.source ?? null;
     return `(${[
+      v(legacyScore.scoreId),
       v(legacyScore.timestamp),
       v(legacyScore.traceId),
       v(legacyScore.spanId ?? null),
@@ -1973,12 +2111,13 @@ async function batchCreateScores(db, args) {
   });
   await db.execute(
     `INSERT INTO score_events (
-      timestamp, traceId, spanId, experimentId, scoreTraceId,
+      scoreId, timestamp, traceId, spanId, experimentId, scoreTraceId,
       entityType, entityId, entityName, entityVersionId, parentEntityVersionId, parentEntityType, parentEntityId, parentEntityName, rootEntityVersionId, rootEntityType, rootEntityId, rootEntityName,
       userId, organizationId, resourceId, runId, sessionId, threadId, requestId, environment, executionSource, serviceName,
       scorerId, scorerVersion, scoreSource, score, reason, tags, metadata, scope
     )
-     VALUES ${tuples.join(",\n       ")}`
+     VALUES ${tuples.join(",\n       ")}
+     ON CONFLICT DO NOTHING`
   );
 }
 async function listScores(db, args) {
@@ -2202,6 +2341,40 @@ var SPAN_RECONSTRUCT_SELECT = `
     ${argMaxNonNull("requestContext")}
   FROM span_events
 `;
+var SPAN_RECONSTRUCT_SELECT_LIGHT = `
+  SELECT
+    traceId, spanId,
+    ${argMaxNonNull("name")},
+    ${argMaxNonNull("spanType")},
+    ${argMaxNonNull("parentSpanId")},
+    ${argMaxNonNull("isEvent")},
+    coalesce(min(timestamp) FILTER (WHERE eventType = 'start'), min(timestamp)) as startedAt,
+    ${argMaxNonNull("endedAt")},
+    ${argMaxNonNull("entityType")},
+    ${argMaxNonNull("entityId")},
+    ${argMaxNonNull("entityName")},
+    ${argMaxNonNull("error")}
+  FROM span_events
+`;
+function rowToLightSpanRecord(row) {
+  return {
+    traceId: row.traceId,
+    spanId: row.spanId,
+    name: row.name,
+    spanType: row.spanType,
+    parentSpanId: row.parentSpanId ?? null,
+    isEvent: row.isEvent,
+    startedAt: toDate(row.startedAt),
+    endedAt: toDateOrNull(row.endedAt),
+    entityType: row.entityType ?? null,
+    entityId: row.entityId ?? null,
+    entityName: row.entityName ?? null,
+    error: parseJson(row.error),
+    createdAt: toDate(row.startedAt),
+    // DuckDB event-sourced — use startedAt as proxy
+    updatedAt: toDateOrNull(row.endedAt)
+  };
+}
 function rowToSpanRecord(row) {
   return {
     traceId: row.traceId,
@@ -2408,6 +2581,16 @@ async function getTrace(db, args) {
     spans: rows.map((row) => rowToSpanRecord(row))
   };
 }
+async function getTraceLight(db, args) {
+  const rows = await db.query(`${SPAN_RECONSTRUCT_SELECT_LIGHT} WHERE traceId = ? GROUP BY traceId, spanId`, [
+    args.traceId
+  ]);
+  if (rows.length === 0) return null;
+  return {
+    traceId: args.traceId,
+    spans: rows.map((row) => rowToLightSpanRecord(row))
+  };
+}
 async function listTraces(db, args) {
   const filters = args.filters ?? {};
   const page = Number(args.pagination?.page ?? 0);
@@ -2456,6 +2639,32 @@ async function listTraces(db, args) {
 }
 
 // src/storage/domains/observability/index.ts
+function buildSignalMigrationRequiredMessage(args) {
+  const tableList = args.tables.map((table) => `  - ${table.table}`).join("\n");
+  return `
+===========================================================================
+MIGRATION REQUIRED: DuckDB observability signal tables need signal IDs
+===========================================================================
+
+The following signal tables still use the legacy schema and must be migrated
+before observability storage can initialize:
+
+${tableList}
+
+To fix this, run the manual migration command:
+
+  npx mastra migrate
+
+This command will:
+  1. Create replacement signal tables with signal-ID primary keys
+  2. Backfill missing signal IDs for legacy rows
+  3. Swap the migrated tables into place
+
+WARNING: This migration recreates the signal tables and may take significant
+time for large databases. Please ensure you have a backup before proceeding.
+===========================================================================
+`;
+}
 var ObservabilityStorageDuckDB = class extends ObservabilityStorage {
   db;
   constructor(config) {
@@ -2464,6 +2673,17 @@ var ObservabilityStorageDuckDB = class extends ObservabilityStorage {
   }
   /** Create all observability tables if they don't exist. */
   async init() {
+    const migrationStatus = await checkSignalTablesMigrationStatus(this.db);
+    if (migrationStatus.needsMigration) {
+      throw new MastraError({
+        id: createStorageErrorId("DUCKDB", "MIGRATION_REQUIRED", "SIGNAL_TABLES"),
+        domain: ErrorDomain.STORAGE,
+        category: ErrorCategory.USER,
+        text: buildSignalMigrationRequiredMessage({
+          tables: migrationStatus.tables.map(({ table }) => ({ table }))
+        })
+      });
+    }
     for (const ddl of ALL_DDL) {
       await this.db.execute(ddl);
     }
@@ -2471,6 +2691,29 @@ var ObservabilityStorageDuckDB = class extends ObservabilityStorage {
       await this.db.execute(migration);
     }
   }
+  /**
+   * Manually migrate legacy signal tables to the signal-ID primary-key schema.
+   * The public method name is historical; the CLI still calls `migrateSpans()`
+   * for observability migrations even though this now also migrates signal tables.
+   */
+  async migrateSpans() {
+    const migrationStatus = await checkSignalTablesMigrationStatus(this.db);
+    if (!migrationStatus.needsMigration) {
+      return {
+        success: true,
+        alreadyMigrated: true,
+        duplicatesRemoved: 0,
+        message: "Migration already complete. Signal tables already use signal-ID primary keys."
+      };
+    }
+    await migrateSignalTables(this.db, this.logger);
+    return {
+      success: true,
+      alreadyMigrated: false,
+      duplicatesRemoved: 0,
+      message: `Migration complete. Migrated signal tables: ${migrationStatus.tables.map((t) => t.table).join(", ")}.`
+    };
+  }
   /** Delete all rows from every observability table. Use with caution. */
   async dangerouslyClearAll() {
     for (const table of ["span_events", "metric_events", "log_events", "score_events", "feedback_events"]) {
@@ -2502,6 +2745,9 @@ var ObservabilityStorageDuckDB = class extends ObservabilityStorage {
   async getTrace(args) {
     return getTrace(this.db, args);
   }
+  async getTraceLight(args) {
+    return getTraceLight(this.db, args);
+  }
   async listTraces(args) {
     return listTraces(this.db, args);
   }
@@ -2604,5 +2850,5 @@ var ObservabilityStorageDuckDB = class extends ObservabilityStorage {
 };
 
 export { ObservabilityStorageDuckDB };
-//# sourceMappingURL=observability-EMLEGDF7.js.map
-//# sourceMappingURL=observability-EMLEGDF7.js.map
+//# sourceMappingURL=observability-YJBOVLPV.js.map
+//# sourceMappingURL=observability-YJBOVLPV.js.map