@mastra/clickhouse 0.0.0-a2a-20250421213654

package/CHANGELOG.md

@@ -0,0 +1,275 @@
+# @mastra/clickhouse
+
+## 0.0.0-a2a-20250421213654
+
+### Minor Changes
+
+- fe3ae4d: Remove \_\_ functions in storage and move to storage proxy to make sure init is called
+
+### Patch Changes
+
+- dd8d0cb: make Clickhouse storage extendable
+- Updated dependencies [08bb78e]
+- Updated dependencies [ed2f549]
+- Updated dependencies [7e92011]
+- Updated dependencies [9ee4293]
+- Updated dependencies [03f3cd0]
+- Updated dependencies [c0f22b4]
+- Updated dependencies [71d9444]
+- Updated dependencies [157c741]
+- Updated dependencies [8a8a73b]
+- Updated dependencies [fe3ae4d]
+- Updated dependencies [6f92295]
+  - @mastra/core@0.0.0-a2a-20250421213654
+
+## 0.3.0-alpha.7
+
+### Patch Changes
+
+- Updated dependencies [71d9444]
+  - @mastra/core@0.9.0-alpha.7
+
+## 0.3.0-alpha.6
+
+### Patch Changes
+
+- dd8d0cb: make Clickhouse storage extendable
+- Updated dependencies [157c741]
+  - @mastra/core@0.9.0-alpha.6
+
+## 0.3.0-alpha.5
+
+### Patch Changes
+
+- Updated dependencies [08bb78e]
+  - @mastra/core@0.9.0-alpha.5
+
+## 0.3.0-alpha.4
+
+### Patch Changes
+
+- Updated dependencies [7e92011]
+  - @mastra/core@0.9.0-alpha.4
+
+## 0.3.0-alpha.3
+
+### Minor Changes
+
+- fe3ae4d: Remove \_\_ functions in storage and move to storage proxy to make sure init is called
+
+### Patch Changes
+
+- Updated dependencies [fe3ae4d]
+  - @mastra/core@0.9.0-alpha.3
+
+## 0.2.11-alpha.2
+
+### Patch Changes
+
+- Updated dependencies [9ee4293]
+  - @mastra/core@0.8.4-alpha.2
+
+## 0.2.11-alpha.1
+
+### Patch Changes
+
+- Updated dependencies [8a8a73b]
+- Updated dependencies [6f92295]
+  - @mastra/core@0.8.4-alpha.1
+
+## 0.2.11-alpha.0
+
+### Patch Changes
+
+- Updated dependencies [03f3cd0]
+  - @mastra/core@0.8.4-alpha.0
+
+## 0.2.10
+
+### Patch Changes
+
+- ec3cbf9: add MIT license for stores/vector packages
+- 37bb612: Add Elastic-2.0 licensing for packages
+- Updated dependencies [d72318f]
+- Updated dependencies [0bcc862]
+- Updated dependencies [10a8caf]
+- Updated dependencies [359b089]
+- Updated dependencies [32e7b71]
+- Updated dependencies [37bb612]
+- Updated dependencies [7f1b291]
+  - @mastra/core@0.8.3
+
+## 0.2.10-alpha.5
+
+### Patch Changes
+
+- ec3cbf9: add MIT license for stores/vector packages
+- Updated dependencies [d72318f]
+  - @mastra/core@0.8.3-alpha.5
+
+## 0.2.10-alpha.4
+
+### Patch Changes
+
+- Updated dependencies [7f1b291]
+  - @mastra/core@0.8.3-alpha.4
+
+## 0.2.10-alpha.3
+
+### Patch Changes
+
+- Updated dependencies [10a8caf]
+  - @mastra/core@0.8.3-alpha.3
+
+## 0.2.10-alpha.2
+
+### Patch Changes
+
+- Updated dependencies [0bcc862]
+  - @mastra/core@0.8.3-alpha.2
+
+## 0.2.10-alpha.1
+
+### Patch Changes
+
+- 37bb612: Add Elastic-2.0 licensing for packages
+- Updated dependencies [32e7b71]
+- Updated dependencies [37bb612]
+  - @mastra/core@0.8.3-alpha.1
+
+## 0.2.10-alpha.0
+
+### Patch Changes
+
+- Updated dependencies [359b089]
+  - @mastra/core@0.8.3-alpha.0
+
+## 0.2.9
+
+### Patch Changes
+
+- Updated dependencies [a06aadc]
+  - @mastra/core@0.8.2
+
+## 0.2.9-alpha.0
+
+### Patch Changes
+
+- Updated dependencies [a06aadc]
+  - @mastra/core@0.8.2-alpha.0
+
+## 0.2.8
+
+### Patch Changes
+
+- Updated dependencies [99e2998]
+- Updated dependencies [8fdb414]
+  - @mastra/core@0.8.1
+
+## 0.2.8-alpha.0
+
+### Patch Changes
+
+- Updated dependencies [99e2998]
+- Updated dependencies [8fdb414]
+  - @mastra/core@0.8.1-alpha.0
+
+## 0.2.7
+
+### Patch Changes
+
+- dbbbf80: Added clickhouse storage
+- 88fa727: Added getWorkflowRuns for libsql, pg, clickhouse and upstash as well as added route getWorkflowRunsHandler
+- 4d67826: Fix eval writes, remove id column
+- Updated dependencies [56c31b7]
+- Updated dependencies [619c39d]
+- Updated dependencies [5ae0180]
+- Updated dependencies [fe56be0]
+- Updated dependencies [93875ed]
+- Updated dependencies [107bcfe]
+- Updated dependencies [9bfa12b]
+- Updated dependencies [515ebfb]
+- Updated dependencies [5b4e19f]
+- Updated dependencies [dbbbf80]
+- Updated dependencies [a0967a0]
+- Updated dependencies [fca3b21]
+- Updated dependencies [88fa727]
+- Updated dependencies [f37f535]
+- Updated dependencies [a3f0e90]
+- Updated dependencies [4d67826]
+- Updated dependencies [6330967]
+- Updated dependencies [8393832]
+- Updated dependencies [6330967]
+- Updated dependencies [99d43b9]
+- Updated dependencies [d7e08e8]
+- Updated dependencies [febc8a6]
+- Updated dependencies [7599d77]
+- Updated dependencies [0118361]
+- Updated dependencies [619c39d]
+- Updated dependencies [cafae83]
+- Updated dependencies [8076ecf]
+- Updated dependencies [8df4a77]
+- Updated dependencies [304397c]
+  - @mastra/core@0.8.0
+
+## 0.2.7-alpha.7
+
+### Patch Changes
+
+- Updated dependencies [8df4a77]
+  - @mastra/core@0.8.0-alpha.8
+
+## 0.2.7-alpha.6
+
+### Patch Changes
+
+- Updated dependencies [febc8a6]
+  - @mastra/core@0.8.0-alpha.7
+
+## 0.2.7-alpha.5
+
+### Patch Changes
+
+- Updated dependencies [a3f0e90]
+  - @mastra/core@0.8.0-alpha.6
+
+## 0.2.7-alpha.4
+
+### Patch Changes
+
+- Updated dependencies [93875ed]
+  - @mastra/core@0.8.0-alpha.5
+
+## 0.2.7-alpha.3
+
+### Patch Changes
+
+- Updated dependencies [d7e08e8]
+  - @mastra/core@0.8.0-alpha.4
+
+## 0.2.7-alpha.2
+
+### Patch Changes
+
+- 88fa727: Added getWorkflowRuns for libsql, pg, clickhouse and upstash as well as added route getWorkflowRunsHandler
+- 4d67826: Fix eval writes, remove id column
+- Updated dependencies [5ae0180]
+- Updated dependencies [9bfa12b]
+- Updated dependencies [515ebfb]
+- Updated dependencies [88fa727]
+- Updated dependencies [f37f535]
+- Updated dependencies [4d67826]
+- Updated dependencies [6330967]
+- Updated dependencies [8393832]
+- Updated dependencies [6330967]
+  - @mastra/core@0.8.0-alpha.3
+
+## 0.2.7-alpha.1
+
+### Patch Changes
+
+- dbbbf80: Added clickhouse storage
+- Updated dependencies [56c31b7]
+- Updated dependencies [dbbbf80]
+- Updated dependencies [99d43b9]
+  - @mastra/core@0.8.0-alpha.2

package/LICENSE.md

@@ -0,0 +1,7 @@
+Copyright 2025 Mastra AI, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,122 @@
+# @mastra/clickhouse
+
+Clickhouse implementation for Mastra, providing efficient storage capabilities with support for threads, messages, and workflow snapshots.
+
+## Installation
+
+```bash
+npm install @mastra/clickhouse
+```
+
+## Prerequisites
+
+- Clickhouse server (version 21.8 or higher recommended)
+- Node.js 16 or higher
+
+## Usage
+
+```typescript
+import { ClickhouseStore } from '@mastra/clickhouse';
+
+const store = new ClickhouseStore({
+  url: 'http://localhost:8123',
+  username: 'default',
+  password: 'password',
+});
+
+// Create a thread
+await store.saveThread({
+  id: 'thread-123',
+  resourceId: 'resource-456',
+  title: 'My Thread',
+  metadata: { key: 'value' },
+  createdAt: new Date(),
+  updatedAt: new Date(),
+});
+
+// Add messages to thread
+await store.saveMessages([
+  {
+    id: 'msg-789',
+    threadId: 'thread-123',
+    role: 'user',
+    type: 'text',
+    content: [{ type: 'text', text: 'Hello' }],
+    createdAt: new Date(),
+  },
+]);
+
+// Query threads and messages
+const savedThread = await store.getThreadById({ threadId: 'thread-123' });
+const messages = await store.getMessages({ threadId: 'thread-123' });
+
+// Clean up
+await store.close();
+```
+
+## Configuration
+
+The Clickhouse store can be initialized with the following configuration:
+
+```typescript
+type ClickhouseConfig = {
+  url: string; // Clickhouse HTTP interface URL
+  username: string; // Database username
+  password: string; // Database password
+};
+```
+
+## Features
+
+### Storage Features
+
+- Thread and message storage with JSON support
+- Efficient batch operations
+- Rich metadata support
+- Timestamp tracking
+- Workflow snapshot persistence
+- Optimized for high-volume data ingestion
+- Uses Clickhouse's MergeTree and ReplacingMergeTree engines for optimal performance
+
+### Table Engines
+
+The store uses different table engines for different types of data:
+
+- `MergeTree()`: Used for messages, traces, and evals
+- `ReplacingMergeTree()`: Used for threads and workflow snapshots
+
+## Storage Methods
+
+### Thread Operations
+
+- `saveThread(thread)`: Create or update a thread
+- `getThreadById({ threadId })`: Get a thread by ID
+- `updateThread({ id, title, metadata })`: Update thread title and metadata
+- `deleteThread({ threadId })`: Delete a thread and its messages
+
+### Message Operations
+
+- `saveMessages(messages)`: Save multiple messages
+- `getMessages({ threadId, selectBy? })`: Get messages for a thread with optional filtering
+- `deleteMessages(messageIds)`: Delete specific messages
+
+### Workflow Operations
+
+- `persistWorkflowSnapshot({ workflowName, runId, snapshot })`: Save workflow state
+- `loadWorkflowSnapshot({ workflowName, runId })`: Load workflow state
+
+## Data Types
+
+The store supports the following data types:
+
+- `text`: String
+- `timestamp`: DateTime64(3)
+- `uuid`: String
+- `jsonb`: String (JSON serialized)
+- `integer`: Int64
+- `bigint`: Int64
+
+## Related Links
+
+- [Clickhouse Documentation](https://clickhouse.com/docs)
+- [Clickhouse Node.js Client](https://github.com/clickhouse/clickhouse-js)

package/dist/_tsup-dts-rollup.d.cts

@@ -0,0 +1,136 @@
+import type { ClickHouseClient } from '@clickhouse/client';
+import type { EvalRow } from '@mastra/core/storage';
+import { MastraStorage } from '@mastra/core/storage';
+import type { MessageType } from '@mastra/core/memory';
+import type { StorageColumn } from '@mastra/core/storage';
+import type { StorageGetMessagesArg } from '@mastra/core/storage';
+import type { StorageThreadType } from '@mastra/core/memory';
+import type { TABLE_NAMES } from '@mastra/core/storage';
+import { TABLE_SCHEMAS } from '@mastra/core/storage';
+import type { WorkflowRunState } from '@mastra/core/workflows';
+
+declare type ClickhouseConfig = {
+    url: string;
+    username: string;
+    password: string;
+    ttl?: {
+        [TableKey in TABLE_NAMES]?: {
+            row?: {
+                interval: number;
+                unit: IntervalUnit;
+                ttlKey?: string;
+            };
+            columns?: Partial<{
+                [ColumnKey in keyof (typeof TABLE_SCHEMAS)[TableKey]]: {
+                    interval: number;
+                    unit: IntervalUnit;
+                    ttlKey?: string;
+                };
+            }>;
+        };
+    };
+};
+export { ClickhouseConfig }
+export { ClickhouseConfig as ClickhouseConfig_alias_1 }
+
+declare class ClickhouseStore extends MastraStorage {
+    protected db: ClickHouseClient;
+    protected ttl: ClickhouseConfig['ttl'];
+    constructor(config: ClickhouseConfig);
+    private transformEvalRow;
+    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
+    batchInsert({ tableName, records }: {
+        tableName: TABLE_NAMES;
+        records: Record<string, any>[];
+    }): Promise<void>;
+    getTraces({ name, scope, page, perPage, attributes, filters, }: {
+        name?: string;
+        scope?: string;
+        page: number;
+        perPage: number;
+        attributes?: Record<string, string>;
+        filters?: Record<string, any>;
+    }): Promise<any[]>;
+    optimizeTable({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    materializeTtl({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    createTable({ tableName, schema, }: {
+        tableName: TABLE_NAMES;
+        schema: Record<string, StorageColumn>;
+    }): Promise<void>;
+    clearTable({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    insert({ tableName, record }: {
+        tableName: TABLE_NAMES;
+        record: Record<string, any>;
+    }): Promise<void>;
+    load<R>({ tableName, keys }: {
+        tableName: TABLE_NAMES;
+        keys: Record<string, string>;
+    }): Promise<R | null>;
+    getThreadById({ threadId }: {
+        threadId: string;
+    }): Promise<StorageThreadType | null>;
+    getThreadsByResourceId({ resourceId }: {
+        resourceId: string;
+    }): Promise<StorageThreadType[]>;
+    saveThread({ thread }: {
+        thread: StorageThreadType;
+    }): Promise<StorageThreadType>;
+    updateThread({ id, title, metadata, }: {
+        id: string;
+        title: string;
+        metadata: Record<string, unknown>;
+    }): Promise<StorageThreadType>;
+    deleteThread({ threadId }: {
+        threadId: string;
+    }): Promise<void>;
+    getMessages<T = unknown>({ threadId, selectBy }: StorageGetMessagesArg): Promise<T>;
+    saveMessages({ messages }: {
+        messages: MessageType[];
+    }): Promise<MessageType[]>;
+    persistWorkflowSnapshot({ workflowName, runId, snapshot, }: {
+        workflowName: string;
+        runId: string;
+        snapshot: WorkflowRunState;
+    }): Promise<void>;
+    loadWorkflowSnapshot({ workflowName, runId, }: {
+        workflowName: string;
+        runId: string;
+    }): Promise<WorkflowRunState | null>;
+    getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, }?: {
+        workflowName?: string;
+        fromDate?: Date;
+        toDate?: Date;
+        limit?: number;
+        offset?: number;
+    }): Promise<{
+        runs: Array<{
+            workflowName: string;
+            runId: string;
+            snapshot: WorkflowRunState | string;
+            createdAt: Date;
+            updatedAt: Date;
+        }>;
+        total: number;
+    }>;
+    close(): Promise<void>;
+}
+export { ClickhouseStore }
+export { ClickhouseStore as ClickhouseStore_alias_1 }
+
+declare const COLUMN_TYPES: Record<StorageColumn['type'], string>;
+export { COLUMN_TYPES }
+export { COLUMN_TYPES as COLUMN_TYPES_alias_1 }
+
+declare type IntervalUnit = 'NANOSECOND' | 'MICROSECOND' | 'MILLISECOND' | 'SECOND' | 'MINUTE' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR';
+
+declare const TABLE_ENGINES: Record<TABLE_NAMES, string>;
+export { TABLE_ENGINES }
+export { TABLE_ENGINES as TABLE_ENGINES_alias_1 }
+
+export { }

package/dist/_tsup-dts-rollup.d.ts

@@ -0,0 +1,136 @@
+import type { ClickHouseClient } from '@clickhouse/client';
+import type { EvalRow } from '@mastra/core/storage';
+import { MastraStorage } from '@mastra/core/storage';
+import type { MessageType } from '@mastra/core/memory';
+import type { StorageColumn } from '@mastra/core/storage';
+import type { StorageGetMessagesArg } from '@mastra/core/storage';
+import type { StorageThreadType } from '@mastra/core/memory';
+import type { TABLE_NAMES } from '@mastra/core/storage';
+import { TABLE_SCHEMAS } from '@mastra/core/storage';
+import type { WorkflowRunState } from '@mastra/core/workflows';
+
+declare type ClickhouseConfig = {
+    url: string;
+    username: string;
+    password: string;
+    ttl?: {
+        [TableKey in TABLE_NAMES]?: {
+            row?: {
+                interval: number;
+                unit: IntervalUnit;
+                ttlKey?: string;
+            };
+            columns?: Partial<{
+                [ColumnKey in keyof (typeof TABLE_SCHEMAS)[TableKey]]: {
+                    interval: number;
+                    unit: IntervalUnit;
+                    ttlKey?: string;
+                };
+            }>;
+        };
+    };
+};
+export { ClickhouseConfig }
+export { ClickhouseConfig as ClickhouseConfig_alias_1 }
+
+declare class ClickhouseStore extends MastraStorage {
+    protected db: ClickHouseClient;
+    protected ttl: ClickhouseConfig['ttl'];
+    constructor(config: ClickhouseConfig);
+    private transformEvalRow;
+    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
+    batchInsert({ tableName, records }: {
+        tableName: TABLE_NAMES;
+        records: Record<string, any>[];
+    }): Promise<void>;
+    getTraces({ name, scope, page, perPage, attributes, filters, }: {
+        name?: string;
+        scope?: string;
+        page: number;
+        perPage: number;
+        attributes?: Record<string, string>;
+        filters?: Record<string, any>;
+    }): Promise<any[]>;
+    optimizeTable({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    materializeTtl({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    createTable({ tableName, schema, }: {
+        tableName: TABLE_NAMES;
+        schema: Record<string, StorageColumn>;
+    }): Promise<void>;
+    clearTable({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    insert({ tableName, record }: {
+        tableName: TABLE_NAMES;
+        record: Record<string, any>;
+    }): Promise<void>;
+    load<R>({ tableName, keys }: {
+        tableName: TABLE_NAMES;
+        keys: Record<string, string>;
+    }): Promise<R | null>;
+    getThreadById({ threadId }: {
+        threadId: string;
+    }): Promise<StorageThreadType | null>;
+    getThreadsByResourceId({ resourceId }: {
+        resourceId: string;
+    }): Promise<StorageThreadType[]>;
+    saveThread({ thread }: {
+        thread: StorageThreadType;
+    }): Promise<StorageThreadType>;
+    updateThread({ id, title, metadata, }: {
+        id: string;
+        title: string;
+        metadata: Record<string, unknown>;
+    }): Promise<StorageThreadType>;
+    deleteThread({ threadId }: {
+        threadId: string;
+    }): Promise<void>;
+    getMessages<T = unknown>({ threadId, selectBy }: StorageGetMessagesArg): Promise<T>;
+    saveMessages({ messages }: {
+        messages: MessageType[];
+    }): Promise<MessageType[]>;
+    persistWorkflowSnapshot({ workflowName, runId, snapshot, }: {
+        workflowName: string;
+        runId: string;
+        snapshot: WorkflowRunState;
+    }): Promise<void>;
+    loadWorkflowSnapshot({ workflowName, runId, }: {
+        workflowName: string;
+        runId: string;
+    }): Promise<WorkflowRunState | null>;
+    getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, }?: {
+        workflowName?: string;
+        fromDate?: Date;
+        toDate?: Date;
+        limit?: number;
+        offset?: number;
+    }): Promise<{
+        runs: Array<{
+            workflowName: string;
+            runId: string;
+            snapshot: WorkflowRunState | string;
+            createdAt: Date;
+            updatedAt: Date;
+        }>;
+        total: number;
+    }>;
+    close(): Promise<void>;
+}
+export { ClickhouseStore }
+export { ClickhouseStore as ClickhouseStore_alias_1 }
+
+declare const COLUMN_TYPES: Record<StorageColumn['type'], string>;
+export { COLUMN_TYPES }
+export { COLUMN_TYPES as COLUMN_TYPES_alias_1 }
+
+declare type IntervalUnit = 'NANOSECOND' | 'MICROSECOND' | 'MILLISECOND' | 'SECOND' | 'MINUTE' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR';
+
+declare const TABLE_ENGINES: Record<TABLE_NAMES, string>;
+export { TABLE_ENGINES }
+export { TABLE_ENGINES as TABLE_ENGINES_alias_1 }
+
+export { }