@mastra/clickhouse 0.0.0-vnext-20251104230439 → 0.0.0-vnext-20251119160359

package/dist/index.js

@@ -1,6 +1,6 @@
 import { createClient } from '@clickhouse/client';
 import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
-import { TABLE_AI_SPANS, TABLE_RESOURCES, TABLE_SCORERS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, StoreOperations, TABLE_SCHEMAS, WorkflowsStorage, normalizePerPage, ScoresStorage, safelyParseJSON, calculatePagination, MemoryStorage, resolveMessageLimit } from '@mastra/core/storage';
+import { TABLE_SPANS, TABLE_RESOURCES, TABLE_SCORERS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, StoreOperations, TABLE_SCHEMAS, WorkflowsStorage, normalizePerPage, ScoresStorage, safelyParseJSON, calculatePagination, MemoryStorage } from '@mastra/core/storage';
 import { MessageList } from '@mastra/core/agent';
 import { saveScorePayloadSchema } from '@mastra/core/evals';
 
@@ -12,8 +12,8 @@ var TABLE_ENGINES = {
   [TABLE_THREADS]: `ReplacingMergeTree()`,
   [TABLE_SCORERS]: `MergeTree()`,
   [TABLE_RESOURCES]: `ReplacingMergeTree()`,
-  // TODO: verify this is the correct engine for ai spans when implementing clickhouse storage
-  [TABLE_AI_SPANS]: `ReplacingMergeTree()`
+  // TODO: verify this is the correct engine for Spans when implementing clickhouse storage
+  [TABLE_SPANS]: `ReplacingMergeTree()`
 };
 var COLUMN_TYPES = {
   text: "String",
@@ -45,6 +45,24 @@ function transformRows(rows) {
 }
 
 // src/storage/domains/memory/index.ts
+function serializeMetadata(metadata) {
+  if (!metadata || Object.keys(metadata).length === 0) {
+    return "{}";
+  }
+  return JSON.stringify(metadata);
+}
+function parseMetadata(metadata) {
+  if (!metadata) return {};
+  if (typeof metadata === "object") return metadata;
+  if (typeof metadata !== "string") return {};
+  const trimmed = metadata.trim();
+  if (trimmed === "" || trimmed === "null") return {};
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    return {};
+  }
+}
 var MemoryStorageClickhouse = class extends MemoryStorage {
   client;
   operations;
@@ -53,135 +71,6 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
     this.client = client;
     this.operations = operations;
   }
-  async getMessages({
-    threadId,
-    resourceId,
-    selectBy
-  }) {
-    try {
-      if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
-      const messages = [];
-      const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
-      const include = selectBy?.include || [];
-      if (include.length) {
-        const unionQueries = [];
-        const params = [];
-        let paramIdx = 1;
-        for (const inc of include) {
-          const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
-          const searchId = inc.threadId || threadId;
-          unionQueries.push(`
-            SELECT * FROM (
-              WITH numbered_messages AS (
-                SELECT
-                  id, content, role, type, "createdAt", thread_id, "resourceId",
-                  ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
-                FROM "${TABLE_MESSAGES}"
-                WHERE thread_id = {var_thread_id_${paramIdx}:String}
-              ),
-              target_positions AS (
-                SELECT row_num as target_pos
-                FROM numbered_messages
-                WHERE id = {var_include_id_${paramIdx}:String}
-              )
-              SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
-              FROM numbered_messages m
-              CROSS JOIN target_positions t
-              WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
-            ) AS query_${paramIdx}
-          `);
-          params.push(
-            { [`var_thread_id_${paramIdx}`]: searchId },
-            { [`var_include_id_${paramIdx}`]: id },
-            { [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
-            { [`var_withNextMessages_${paramIdx}`]: withNextMessages }
-          );
-          paramIdx++;
-        }
-        const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
-        const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
-        const includeResult = await this.client.query({
-          query: finalQuery,
-          query_params: mergedParams,
-          clickhouse_settings: {
-            date_time_input_format: "best_effort",
-            date_time_output_format: "iso",
-            use_client_time_zone: 1,
-            output_format_json_quote_64bit_integers: 0
-          }
-        });
-        const rows2 = await includeResult.json();
-        const includedMessages = transformRows(rows2.data);
-        const seen = /* @__PURE__ */ new Set();
-        const dedupedMessages = includedMessages.filter((message) => {
-          if (seen.has(message.id)) return false;
-          seen.add(message.id);
-          return true;
-        });
-        messages.push(...dedupedMessages);
-      }
-      let whereClause = "WHERE thread_id = {threadId:String}";
-      const queryParams = {
-        threadId,
-        exclude: messages.map((m) => m.id),
-        limit
-      };
-      if (resourceId) {
-        whereClause += ' AND "resourceId" = {resourceId:String}';
-        queryParams.resourceId = resourceId;
-      }
-      const result = await this.client.query({
-        query: `
-        SELECT 
-            id, 
-            content, 
-            role, 
-            type,
-            toDateTime64(createdAt, 3) as createdAt,
-            thread_id AS "threadId"
-        FROM "${TABLE_MESSAGES}"
-        ${whereClause}
-        AND id NOT IN ({exclude:Array(String)})
-        ORDER BY "createdAt" DESC
-        LIMIT {limit:Int64}
-        `,
-        query_params: queryParams,
-        clickhouse_settings: {
-          // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
-          date_time_input_format: "best_effort",
-          date_time_output_format: "iso",
-          use_client_time_zone: 1,
-          output_format_json_quote_64bit_integers: 0
-        }
-      });
-      const rows = await result.json();
-      messages.push(...transformRows(rows.data));
-      messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
-      messages.forEach((message) => {
-        if (typeof message.content === "string") {
-          try {
-            message.content = JSON.parse(message.content);
-          } catch {
-          }
-        }
-      });
-      const list = new MessageList({ threadId, resourceId }).add(
-        messages,
-        "memory"
-      );
-      return { messages: list.get.all.db() };
-    } catch (error) {
-      throw new MastraError(
-        {
-          id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
-          domain: ErrorDomain.STORAGE,
-          category: ErrorCategory.THIRD_PARTY,
-          details: { threadId, resourceId: resourceId ?? "" }
-        },
-        error
-      );
-    }
-  }
   async listMessagesById({ messageIds }) {
     if (messageIds.length === 0) return { messages: [] };
     try {
@@ -288,7 +177,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
         dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
         dataParams.toDate = endDate;
       }
-      const { field, direction } = this.parseOrderBy(orderBy);
+      const { field, direction } = this.parseOrderBy(orderBy, "ASC");
       dataQuery += ` ORDER BY "${field}" ${direction}`;
       if (perPageForResponse === false) ; else {
         dataQuery += ` LIMIT {limit:Int64} OFFSET {offset:Int64}`;
@@ -577,7 +466,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
             id: thread.id,
             resourceId: thread.resourceId,
             title: thread.title,
-            metadata: thread.metadata,
+            metadata: serializeMetadata(thread.metadata),
             createdAt: thread.createdAt,
             updatedAt: (/* @__PURE__ */ new Date()).toISOString()
           })),
@@ -612,8 +501,9 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
           toDateTime64(createdAt, 3) as createdAt,
           toDateTime64(updatedAt, 3) as updatedAt
         FROM "${TABLE_THREADS}"
-        FINAL
-        WHERE id = {var_id:String}`,
+        WHERE id = {var_id:String}
+        ORDER BY updatedAt DESC
+        LIMIT 1`,
         query_params: { var_id: threadId },
         clickhouse_settings: {
           // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
@@ -630,7 +520,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
       }
       return {
         ...thread,
-        metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
+        metadata: parseMetadata(thread.metadata),
         createdAt: thread.createdAt,
         updatedAt: thread.updatedAt
       };
@@ -653,6 +543,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
         values: [
           {
             ...thread,
+            metadata: serializeMetadata(thread.metadata),
             createdAt: thread.createdAt.toISOString(),
             updatedAt: thread.updatedAt.toISOString()
           }
@@ -706,7 +597,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
             id: updatedThread.id,
             resourceId: updatedThread.resourceId,
             title: updatedThread.title,
-            metadata: updatedThread.metadata,
+            metadata: serializeMetadata(updatedThread.metadata),
             createdAt: updatedThread.createdAt,
             updatedAt: updatedThread.updatedAt.toISOString()
           }
@@ -776,7 +667,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
     const { field, direction } = this.parseOrderBy(orderBy);
     try {
       const countResult = await this.client.query({
-        query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
+        query: `SELECT count(DISTINCT id) as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
         query_params: { resourceId },
         clickhouse_settings: {
           date_time_input_format: "best_effort",
@@ -798,15 +689,27 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
       }
       const dataResult = await this.client.query({
         query: `
+              WITH ranked_threads AS (
+                SELECT
+                  id,
+                  resourceId,
+                  title,
+                  metadata,
+                  toDateTime64(createdAt, 3) as createdAt,
+                  toDateTime64(updatedAt, 3) as updatedAt,
+                  ROW_NUMBER() OVER (PARTITION BY id ORDER BY updatedAt DESC) as row_num
+                FROM ${TABLE_THREADS}
+                WHERE resourceId = {resourceId:String}
+              )
               SELECT
                 id,
                 resourceId,
                 title,
                 metadata,
-                toDateTime64(createdAt, 3) as createdAt,
-                toDateTime64(updatedAt, 3) as updatedAt
-              FROM ${TABLE_THREADS}
-              WHERE resourceId = {resourceId:String}
+                createdAt,
+                updatedAt
+              FROM ranked_threads
+              WHERE row_num = 1
               ORDER BY "${field}" ${direction === "DESC" ? "DESC" : "ASC"}
               LIMIT {perPage:Int64} OFFSET {offset:Int64}
             `,
@@ -823,7 +726,10 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
         }
       });
       const rows = await dataResult.json();
-      const threads = transformRows(rows.data);
+      const threads = transformRows(rows.data).map((thread) => ({
+        ...thread,
+        metadata: parseMetadata(thread.metadata)
+      }));
       return {
         threads,
         total,
@@ -1040,7 +946,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
         const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
         const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
           const threadResult = await this.client.query({
-            query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
+            query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String} ORDER BY updatedAt DESC LIMIT 1`,
             query_params: { threadId },
             clickhouse_settings: {
               date_time_input_format: "best_effort",
@@ -1069,7 +975,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
                   id: existingThread.id,
                   resourceId: existingThread.resourceId,
                   title: existingThread.title,
-                  metadata: existingThread.metadata,
+                  metadata: typeof existingThread.metadata === "string" ? existingThread.metadata : serializeMetadata(existingThread.metadata),
                   createdAt: existingThread.createdAt,
                   updatedAt: now
                 }
@@ -1128,7 +1034,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
   async getResourceById({ resourceId }) {
     try {
       const result = await this.client.query({
-        query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
+        query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String} ORDER BY updatedAt DESC LIMIT 1`,
         query_params: { resourceId },
         clickhouse_settings: {
           date_time_input_format: "best_effort",
@@ -1300,6 +1206,9 @@ var StoreOperationsClickhouse = class extends StoreOperations {
       const columns = Object.entries(schema).map(([name, def]) => {
         const constraints = [];
         if (!def.nullable) constraints.push("NOT NULL");
+        if (name === "metadata" && def.type === "text" && def.nullable) {
+          constraints.push("DEFAULT '{}'");
+        }
         const columnTtl = this.ttl?.[tableName]?.columns?.[name];
         return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
       }).join(",\n");
@@ -2071,7 +1980,8 @@ var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
     toDate,
     page,
     perPage,
-    resourceId
+    resourceId,
+    status
   } = {}) {
     try {
       const conditions = [];
@@ -2080,6 +1990,10 @@ var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
         conditions.push(`workflow_name = {var_workflow_name:String}`);
         values.var_workflow_name = workflowName;
       }
+      if (status) {
+        conditions.push(`JSONExtractString(snapshot, 'status') = {var_status:String}`);
+        values.var_status = status;
+      }
       if (resourceId) {
         const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
         if (hasResourceId) {
@@ -2206,7 +2120,7 @@ var ClickhouseStore = class extends MastraStorage {
   ttl = {};
   stores;
   constructor(config) {
-    super({ name: "ClickhouseStore" });
+    super({ id: config.id, name: "ClickhouseStore" });
     this.db = createClient({
       url: config.url,
       username: config.username,
@@ -2333,15 +2247,8 @@ var ClickhouseStore = class extends MastraStorage {
   }) {
     return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
   }
-  async listWorkflowRuns({
-    workflowName,
-    fromDate,
-    toDate,
-    perPage,
-    page,
-    resourceId
-  } = {}) {
-    return this.stores.workflows.listWorkflowRuns({ workflowName, fromDate, toDate, perPage, page, resourceId });
+  async listWorkflowRuns(args = {}) {
+    return this.stores.workflows.listWorkflowRuns(args);
   }
   async getWorkflowRunById({
     runId,
@@ -2365,13 +2272,6 @@ var ClickhouseStore = class extends MastraStorage {
   async deleteThread({ threadId }) {
     return this.stores.memory.deleteThread({ threadId });
   }
-  async getMessages({
-    threadId,
-    resourceId,
-    selectBy
-  }) {
-    return this.stores.memory.getMessages({ threadId, resourceId, selectBy });
-  }
   async saveMessages(args) {
     return this.stores.memory.saveMessages(args);
   }