@mastra/clickhouse 0.2.7-alpha.1 → 0.2.7-alpha.3

package/.turbo/turbo-build.log

@@ -1,23 +1,23 @@
 
-> @mastra/clickhouse@0.2.7-alpha.1 build /home/runner/work/mastra/mastra/stores/clickhouse
+> @mastra/clickhouse@0.2.7-alpha.3 build /home/runner/work/mastra/mastra/stores/clickhouse
 > tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
 
 CLI Building entry: src/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.4.0
 TSC Build start
-TSC ⚡️ Build success in 8308ms
+TSC ⚡️ Build success in 7488ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/stores/clickhouse/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/stores/clickhouse/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 10391ms
+DTS ⚡️ Build success in 11085ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-ESM dist/index.js 19.85 KB
-ESM ⚡️ Build success in 754ms
-CJS dist/index.cjs 19.95 KB
-CJS ⚡️ Build success in 752ms
+ESM dist/index.js 25.49 KB
+ESM ⚡️ Build success in 1089ms
+CJS dist/index.cjs 25.66 KB
+CJS ⚡️ Build success in 1091ms

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @mastra/clickhouse
 
+## 0.2.7-alpha.3
+
+### Patch Changes
+
+- Updated dependencies [d7e08e8]
+  - @mastra/core@0.8.0-alpha.4
+
+## 0.2.7-alpha.2
+
+### Patch Changes
+
+- 88fa727: Added getWorkflowRuns for libsql, pg, clickhouse and upstash as well as added route getWorkflowRunsHandler
+- 4d67826: Fix eval writes, remove id column
+- Updated dependencies [5ae0180]
+- Updated dependencies [9bfa12b]
+- Updated dependencies [515ebfb]
+- Updated dependencies [88fa727]
+- Updated dependencies [f37f535]
+- Updated dependencies [4d67826]
+- Updated dependencies [6330967]
+- Updated dependencies [8393832]
+- Updated dependencies [6330967]
+  - @mastra/core@0.8.0-alpha.3
+
 ## 0.2.7-alpha.1
 
 ### Patch Changes

package/dist/_tsup-dts-rollup.d.cts

@@ -5,31 +5,57 @@ import type { StorageColumn } from '@mastra/core/storage';
 import type { StorageGetMessagesArg } from '@mastra/core/storage';
 import type { StorageThreadType } from '@mastra/core/memory';
 import type { TABLE_NAMES } from '@mastra/core/storage';
+import { TABLE_SCHEMAS } from '@mastra/core/storage';
 import type { WorkflowRunState } from '@mastra/core/workflows';
 
 declare type ClickhouseConfig = {
     url: string;
     username: string;
     password: string;
+    ttl?: {
+        [TableKey in TABLE_NAMES]?: {
+            row?: {
+                interval: number;
+                unit: IntervalUnit;
+                ttlKey?: string;
+            };
+            columns?: Partial<{
+                [ColumnKey in keyof (typeof TABLE_SCHEMAS)[TableKey]]: {
+                    interval: number;
+                    unit: IntervalUnit;
+                    ttlKey?: string;
+                };
+            }>;
+        };
+    };
 };
 export { ClickhouseConfig }
 export { ClickhouseConfig as ClickhouseConfig_alias_1 }
 
 declare class ClickhouseStore extends MastraStorage {
     private db;
+    private ttl;
     constructor(config: ClickhouseConfig);
-    getEvalsByAgentName(_agentName: string, _type?: 'test' | 'live'): Promise<EvalRow[]>;
+    private transformEvalRow;
+    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
     batchInsert({ tableName, records }: {
         tableName: TABLE_NAMES;
         records: Record<string, any>[];
     }): Promise<void>;
-    getTraces({ name, scope, page, perPage, attributes, }: {
+    getTraces({ name, scope, page, perPage, attributes, filters, }: {
         name?: string;
         scope?: string;
         page: number;
         perPage: number;
         attributes?: Record<string, string>;
+        filters?: Record<string, any>;
     }): Promise<any[]>;
+    optimizeTable({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    materializeTtl({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
     createTable({ tableName, schema, }: {
         tableName: TABLE_NAMES;
         schema: Record<string, StorageColumn>;
@@ -75,9 +101,27 @@ declare class ClickhouseStore extends MastraStorage {
         workflowName: string;
         runId: string;
     }): Promise<WorkflowRunState | null>;
+    getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, }?: {
+        workflowName?: string;
+        fromDate?: Date;
+        toDate?: Date;
+        limit?: number;
+        offset?: number;
+    }): Promise<{
+        runs: Array<{
+            workflowName: string;
+            runId: string;
+            snapshot: WorkflowRunState | string;
+            createdAt: Date;
+            updatedAt: Date;
+        }>;
+        total: number;
+    }>;
     close(): Promise<void>;
 }
 export { ClickhouseStore }
 export { ClickhouseStore as ClickhouseStore_alias_1 }
 
+declare type IntervalUnit = 'NANOSECOND' | 'MICROSECOND' | 'MILLISECOND' | 'SECOND' | 'MINUTE' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR';
+
 export { }

package/dist/_tsup-dts-rollup.d.ts

@@ -5,31 +5,57 @@ import type { StorageColumn } from '@mastra/core/storage';
 import type { StorageGetMessagesArg } from '@mastra/core/storage';
 import type { StorageThreadType } from '@mastra/core/memory';
 import type { TABLE_NAMES } from '@mastra/core/storage';
+import { TABLE_SCHEMAS } from '@mastra/core/storage';
 import type { WorkflowRunState } from '@mastra/core/workflows';
 
 declare type ClickhouseConfig = {
     url: string;
     username: string;
     password: string;
+    ttl?: {
+        [TableKey in TABLE_NAMES]?: {
+            row?: {
+                interval: number;
+                unit: IntervalUnit;
+                ttlKey?: string;
+            };
+            columns?: Partial<{
+                [ColumnKey in keyof (typeof TABLE_SCHEMAS)[TableKey]]: {
+                    interval: number;
+                    unit: IntervalUnit;
+                    ttlKey?: string;
+                };
+            }>;
+        };
+    };
 };
 export { ClickhouseConfig }
 export { ClickhouseConfig as ClickhouseConfig_alias_1 }
 
 declare class ClickhouseStore extends MastraStorage {
     private db;
+    private ttl;
     constructor(config: ClickhouseConfig);
-    getEvalsByAgentName(_agentName: string, _type?: 'test' | 'live'): Promise<EvalRow[]>;
+    private transformEvalRow;
+    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
     batchInsert({ tableName, records }: {
         tableName: TABLE_NAMES;
         records: Record<string, any>[];
     }): Promise<void>;
-    getTraces({ name, scope, page, perPage, attributes, }: {
+    getTraces({ name, scope, page, perPage, attributes, filters, }: {
         name?: string;
         scope?: string;
         page: number;
         perPage: number;
         attributes?: Record<string, string>;
+        filters?: Record<string, any>;
     }): Promise<any[]>;
+    optimizeTable({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    materializeTtl({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
     createTable({ tableName, schema, }: {
         tableName: TABLE_NAMES;
         schema: Record<string, StorageColumn>;
@@ -75,9 +101,27 @@ declare class ClickhouseStore extends MastraStorage {
         workflowName: string;
         runId: string;
     }): Promise<WorkflowRunState | null>;
+    getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, }?: {
+        workflowName?: string;
+        fromDate?: Date;
+        toDate?: Date;
+        limit?: number;
+        offset?: number;
+    }): Promise<{
+        runs: Array<{
+            workflowName: string;
+            runId: string;
+            snapshot: WorkflowRunState | string;
+            createdAt: Date;
+            updatedAt: Date;
+        }>;
+        total: number;
+    }>;
     close(): Promise<void>;
 }
 export { ClickhouseStore }
 export { ClickhouseStore as ClickhouseStore_alias_1 }
 
+declare type IntervalUnit = 'NANOSECOND' | 'MICROSECOND' | 'MILLISECOND' | 'SECOND' | 'MINUTE' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR';
+
 export { }

package/dist/index.cjs

@@ -1,7 +1,7 @@
 'use strict';
 
-var storage = require('@mastra/core/storage');
 var client = require('@clickhouse/client');
+var storage = require('@mastra/core/storage');
 
 // src/storage/index.ts
 function safelyParseJSON(jsonString) {
@@ -43,6 +43,7 @@ function transformRow(row) {
 }
 var ClickhouseStore = class extends storage.MastraStorage {
   db;
+  ttl = {};
   constructor(config) {
     super({ name: "ClickhouseStore" });
     this.db = client.createClient({
@@ -57,9 +58,54 @@ var ClickhouseStore = class extends storage.MastraStorage {
         output_format_json_quote_64bit_integers: 0
       }
     });
+    this.ttl = config.ttl;
+  }
+  transformEvalRow(row) {
+    row = transformRow(row);
+    const resultValue = JSON.parse(row.result);
+    const testInfoValue = row.test_info ? JSON.parse(row.test_info) : void 0;
+    if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
+      throw new Error(`Invalid MetricResult format: ${JSON.stringify(resultValue)}`);
+    }
+    return {
+      input: row.input,
+      output: row.output,
+      result: resultValue,
+      agentName: row.agent_name,
+      metricName: row.metric_name,
+      instructions: row.instructions,
+      testInfo: testInfoValue,
+      globalRunId: row.global_run_id,
+      runId: row.run_id,
+      createdAt: row.created_at
+    };
   }
-  getEvalsByAgentName(_agentName, _type) {
-    throw new Error("Method not implemented.");
+  async getEvalsByAgentName(agentName, type) {
+    try {
+      const baseQuery = `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
+      const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
+      const result = await this.db.query({
+        query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
+        query_params: { var_agent_name: agentName },
+        clickhouse_settings: {
+          date_time_input_format: "best_effort",
+          date_time_output_format: "iso",
+          use_client_time_zone: 1,
+          output_format_json_quote_64bit_integers: 0
+        }
+      });
+      if (!result) {
+        return [];
+      }
+      const rows = await result.json();
+      return rows.data.map((row) => this.transformEvalRow(row));
+    } catch (error) {
+      if (error instanceof Error && error.message.includes("no such table")) {
+        return [];
+      }
+      this.logger.error("Failed to get evals for the specified agent: " + error?.message);
+      throw error;
+    }
   }
   async batchInsert({ tableName, records }) {
     try {
@@ -91,7 +137,8 @@ var ClickhouseStore = class extends storage.MastraStorage {
     scope,
     page,
     perPage,
-    attributes
+    attributes,
+    filters
   }) {
     const limit = perPage;
     const offset = page * perPage;
@@ -107,8 +154,16 @@ var ClickhouseStore = class extends storage.MastraStorage {
     }
     if (attributes) {
       Object.entries(attributes).forEach(([key, value]) => {
-        conditions.push(`JSONExtractString(attributes, '${key}') = {var_${key}:String}`);
-        args[`var_${key}`] = value;
+        conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
+        args[`var_attr_${key}`] = value;
+      });
+    }
+    if (filters) {
+      Object.entries(filters).forEach(([key, value]) => {
+        conditions.push(
+          `${key} = {var_col_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"]}}`
+        );
+        args[`var_col_${key}`] = value;
       });
     }
     const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
@@ -145,6 +200,16 @@ var ClickhouseStore = class extends storage.MastraStorage {
       createdAt: row.createdAt
     }));
   }
+  async optimizeTable({ tableName }) {
+    await this.db.command({
+      query: `OPTIMIZE TABLE ${tableName} FINAL`
+    });
+  }
+  async materializeTtl({ tableName }) {
+    await this.db.command({
+      query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
+    });
+  }
   async createTable({
     tableName,
     schema
@@ -153,8 +218,10 @@ var ClickhouseStore = class extends storage.MastraStorage {
       const columns = Object.entries(schema).map(([name, def]) => {
         const constraints = [];
         if (!def.nullable) constraints.push("NOT NULL");
-        return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")}`;
+        const columnTtl = this.ttl?.[tableName]?.columns?.[name];
+        return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
       }).join(",\n");
+      const rowTtl = this.ttl?.[tableName]?.row;
       const sql = tableName === storage.TABLE_WORKFLOW_SNAPSHOT ? `
         CREATE TABLE IF NOT EXISTS ${tableName} (
           ${["id String"].concat(columns)}
@@ -163,16 +230,18 @@ var ClickhouseStore = class extends storage.MastraStorage {
         PARTITION BY "createdAt"
         PRIMARY KEY (createdAt, run_id, workflow_name)
         ORDER BY (createdAt, run_id, workflow_name)
-        SETTINGS index_granularity = 8192;
+        ${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
+        SETTINGS index_granularity = 8192
           ` : `
         CREATE TABLE IF NOT EXISTS ${tableName} (
           ${columns}
         )
         ENGINE = ${TABLE_ENGINES[tableName]}
         PARTITION BY "createdAt"
-        PRIMARY KEY (createdAt, id)
-        ORDER BY (createdAt, id)
-        SETTINGS index_granularity = 8192;
+        PRIMARY KEY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
+        ORDER BY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
+        ${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
+        SETTINGS index_granularity = 8192
       `;
       await this.db.query({
         query: sql,
@@ -627,6 +696,83 @@ var ClickhouseStore = class extends storage.MastraStorage {
       throw error;
     }
   }
+  async getWorkflowRuns({
+    workflowName,
+    fromDate,
+    toDate,
+    limit,
+    offset
+  } = {}) {
+    try {
+      const conditions = [];
+      const values = {};
+      if (workflowName) {
+        conditions.push(`workflow_name = {var_workflow_name:String}`);
+        values.var_workflow_name = workflowName;
+      }
+      if (fromDate) {
+        conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
+        values.var_from_date = fromDate.getTime() / 1e3;
+      }
+      if (toDate) {
+        conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
+        values.var_to_date = toDate.getTime() / 1e3;
+      }
+      const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+      const limitClause = limit !== void 0 ? `LIMIT ${limit}` : "";
+      const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
+      let total = 0;
+      if (limit !== void 0 && offset !== void 0) {
+        const countResult = await this.db.query({
+          query: `SELECT COUNT(*) as count FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
+          query_params: values,
+          format: "JSONEachRow"
+        });
+        const countRows = await countResult.json();
+        total = Number(countRows[0]?.count ?? 0);
+      }
+      const result = await this.db.query({
+        query: `
+          SELECT 
+            workflow_name,
+            run_id,
+            snapshot,
+            toDateTime64(createdAt, 3) as createdAt,
+            toDateTime64(updatedAt, 3) as updatedAt
+          FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
+          ${whereClause}
+          ORDER BY createdAt DESC
+          ${limitClause}
+          ${offsetClause}
+        `,
+        query_params: values,
+        format: "JSONEachRow"
+      });
+      const resultJson = await result.json();
+      const rows = resultJson;
+      const runs = rows.map((row) => {
+        let parsedSnapshot = row.snapshot;
+        if (typeof parsedSnapshot === "string") {
+          try {
+            parsedSnapshot = JSON.parse(row.snapshot);
+          } catch (e) {
+            console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
+          }
+        }
+        return {
+          workflowName: row.workflow_name,
+          runId: row.run_id,
+          snapshot: parsedSnapshot,
+          createdAt: new Date(row.createdAt),
+          updatedAt: new Date(row.updatedAt)
+        };
+      });
+      return { runs, total: total || runs.length };
+    } catch (error) {
+      console.error("Error getting workflow runs:", error);
+      throw error;
+    }
+  }
   async close() {
     await this.db.close();
   }