@mastra/pg 0.0.0-a2a-20250421213654

package/LICENSE.md

@@ -0,0 +1,7 @@
+Copyright 2025 Mastra AI, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,166 @@
+# @mastra/pg
+
+PostgreSQL implementation for Mastra, providing both vector similarity search (using pgvector) and general storage capabilities with connection pooling and transaction support.
+
+## Installation
+
+```bash
+npm install @mastra/pg
+```
+
+## Prerequisites
+
+- PostgreSQL server with pgvector extension installed (if using vector store)
+- PostgreSQL 11 or higher
+
+## Usage
+
+### Vector Store
+
+```typescript
+import { PgVector } from '@mastra/pg';
+
+const vectorStore = new PgVector('postgresql://user:pass@localhost:5432/db');
+
+// Create a new table with vector support
+await vectorStore.createIndex({
+  indexName: 'my_vectors',
+  dimension: 1536,
+  metric: 'cosine',
+});
+
+// Add vectors
+const ids = await vectorStore.upsert({
+  indexName: 'my_vectors',
+  vectors: [[0.1, 0.2, ...], [0.3, 0.4, ...]],
+  metadata: [{ text: 'doc1' }, { text: 'doc2' }],
+});
+
+// Query vectors
+const results = await vectorStore.query({
+  indexName: 'my_vectors',
+  queryVector: [0.1, 0.2, ...],
+  topK: 10, // topK
+  filter: { text: 'doc1' }, // filter
+  includeVector: false, // includeVector
+  minScore: 0.5, // minScore
+});
+
+// Clean up
+await vectorStore.disconnect();
+```
+
+### Storage
+
+```typescript
+import { PostgresStore } from '@mastra/pg';
+
+const store = new PostgresStore({
+  host: 'localhost',
+  port: 5432,
+  database: 'mastra',
+  user: 'postgres',
+  password: 'postgres',
+});
+
+// Create a thread
+await store.saveThread({
+  id: 'thread-123',
+  resourceId: 'resource-456',
+  title: 'My Thread',
+  metadata: { key: 'value' },
+});
+
+// Add messages to thread
+await store.saveMessages([
+  {
+    id: 'msg-789',
+    threadId: 'thread-123',
+    role: 'user',
+    type: 'text',
+    content: [{ type: 'text', text: 'Hello' }],
+  },
+]);
+
+// Query threads and messages
+const savedThread = await store.getThread('thread-123');
+const messages = await store.getMessages('thread-123');
+```
+
+## Configuration
+
+The PostgreSQL store can be initialized with either:
+
+- `connectionString`: PostgreSQL connection string (for vector store)
+- Configuration object with host, port, database, user, and password (for storage)
+
+Connection pool settings:
+
+- Maximum connections: 20
+- Idle timeout: 30 seconds
+- Connection timeout: 2 seconds
+
+## Features
+
+### Vector Store Features
+
+- Vector similarity search with cosine, euclidean, and dot product metrics
+- Advanced metadata filtering with MongoDB-like query syntax
+- Minimum score threshold for queries
+- Automatic UUID generation for vectors
+- Table management (create, list, describe, delete, truncate)
+- Uses pgvector's IVFFLAT indexing with 100 lists by default
+- Supports HNSW indexing with configurable parameters
+- Supports flat indexing
+
+### Storage Features
+
+- Thread and message storage with JSON support
+- Atomic transactions for data consistency
+- Efficient batch operations
+- Rich metadata support
+- Timestamp tracking
+- Cascading deletes
+
+## Supported Filter Operators
+
+The following filter operators are supported for metadata queries:
+
+- Comparison: `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`
+- Logical: `$and`, `$or`
+- Array: `$in`, `$nin`
+- Text: `$regex`, `$like`
+
+Example filter:
+
+```typescript
+{
+  $and: [{ age: { $gt: 25 } }, { tags: { $in: ['tag1', 'tag2'] } }];
+}
+```
+
+## Vector Store Methods
+
+- `createIndex({indexName, dimension, metric?, indexConfig?, defineIndex?})`: Create a new table with vector support
+- `upsert({indexName, vectors, metadata?, ids?})`: Add or update vectors
+- `query({indexName, queryVector, topK?, filter?, includeVector?, minScore?})`: Search for similar vectors
+- `defineIndex({indexName, metric?, indexConfig?})`: Define an index
+- `listIndexes()`: List all vector-enabled tables
+- `describeIndex(indexName)`: Get table statistics
+- `deleteIndex(indexName)`: Delete a table
+- `truncateIndex(indexName)`: Remove all data from a table
+- `disconnect()`: Close all database connections
+
+## Storage Methods
+
+- `saveThread(thread)`: Create or update a thread
+- `getThread(threadId)`: Get a thread by ID
+- `deleteThread(threadId)`: Delete a thread and its messages
+- `saveMessages(messages)`: Save multiple messages in a transaction
+- `getMessages(threadId)`: Get all messages for a thread
+- `deleteMessages(messageIds)`: Delete specific messages
+
+## Related Links
+
+- [pgvector Documentation](https://github.com/pgvector/pgvector)
+- [PostgreSQL Documentation](https://www.postgresql.org/docs/)

package/dist/_tsup-dts-rollup.d.cts

@@ -0,0 +1,392 @@
+import type { ArrayOperator } from '@mastra/core/vector/filter';
+import { BaseFilterTranslator } from '@mastra/core/vector/filter';
+import type { BasicOperator } from '@mastra/core/vector/filter';
+import type { CreateIndexArgs } from '@mastra/core/vector';
+import type { CreateIndexParams } from '@mastra/core/vector';
+import type { ElementOperator } from '@mastra/core/vector/filter';
+import type { EvalRow } from '@mastra/core/storage';
+import type { IndexStats } from '@mastra/core/vector';
+import type { ISSLConfig } from 'pg-promise/typescript/pg-subset';
+import type { LogicalOperator } from '@mastra/core/vector/filter';
+import { MastraStorage } from '@mastra/core/storage';
+import { MastraVector } from '@mastra/core/vector';
+import type { MessageType } from '@mastra/core/memory';
+import type { NumericOperator } from '@mastra/core/vector/filter';
+import type { OperatorSupport } from '@mastra/core/vector/filter';
+import type { ParamsToArgs } from '@mastra/core/vector';
+import type { QueryResult } from '@mastra/core/vector';
+import type { QueryVectorArgs } from '@mastra/core/vector';
+import type { QueryVectorParams } from '@mastra/core/vector';
+import type { RegexOperator } from '@mastra/core/vector/filter';
+import type { StorageColumn } from '@mastra/core/storage';
+import type { StorageGetMessagesArg } from '@mastra/core/storage';
+import type { StorageThreadType } from '@mastra/core/memory';
+import type { TABLE_NAMES } from '@mastra/core/storage';
+import type { UpsertVectorParams } from '@mastra/core/vector';
+import type { VectorFilter } from '@mastra/core/vector/filter';
+import type { WorkflowRunState } from '@mastra/core/workflows';
+
+export declare const baseTestConfigs: {
+    smokeTests: {
+        dimension: number;
+        size: number;
+        k: number;
+        queryCount: number;
+    }[];
+    '64': {
+        dimension: number;
+        size: number;
+        k: number;
+        queryCount: number;
+    }[];
+    '384': {
+        dimension: number;
+        size: number;
+        k: number;
+        queryCount: number;
+    }[];
+    '1024': {
+        dimension: number;
+        size: number;
+        k: number;
+        queryCount: number;
+    }[];
+    stressTests: {
+        dimension: number;
+        size: number;
+        k: number;
+        queryCount: number;
+    }[];
+};
+
+export declare function buildFilterQuery(filter: VectorFilter, minScore: number): FilterResult;
+
+export declare const calculateRecall: (actual: number[], expected: number[], k: number) => number;
+
+export declare const calculateTimeout: (dimension: number, size: number, k: number) => number;
+
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+
+export declare const FILTER_OPERATORS: Record<string, OperatorFn>;
+
+declare type FilterOperator = {
+    sql: string;
+    needsValue: boolean;
+    transformValue?: () => any;
+};
+
+export declare interface FilterResult {
+    sql: string;
+    values: any[];
+}
+
+export declare const findNearestBruteForce: (query: number[], vectors: number[][], k: number) => number[];
+
+export declare const formatTable: (data: any[], columns: string[]) => string;
+
+export declare const generateClusteredVectors: (count: number, dim: number, numClusters?: number) => number[][];
+
+export declare const generateRandomVectors: (count: number, dim: number) => number[][];
+
+export declare const generateSkewedVectors: (count: number, dim: number) => number[][];
+
+export declare const getHNSWConfig: (indexConfig: IndexConfig) => {
+    m: number;
+    efConstruction: number;
+};
+
+export declare function getIndexDescription({ type, hnsw, }: {
+    type: IndexType;
+    hnsw: {
+        m: number;
+        efConstruction: number;
+    };
+}): string;
+
+export declare const getListCount: (indexConfig: IndexConfig, size: number) => number | undefined;
+
+export declare function getSearchEf(k: number, m: number): {
+    default: number;
+    lower: number;
+    higher: number;
+};
+
+export declare const groupBy: <T, K extends keyof T>(array: T[], key: K | ((item: T) => string), reducer?: (group: T[]) => any) => Record<string, any>;
+
+export declare const handleKey: (key: string) => string;
+
+declare interface HNSWConfig {
+    m?: number;
+    efConstruction?: number;
+}
+
+export declare interface IndexConfig {
+    type?: IndexType;
+    ivf?: IVFConfig;
+    hnsw?: HNSWConfig;
+}
+
+export declare type IndexType = 'ivfflat' | 'hnsw' | 'flat';
+
+declare interface IVFConfig {
+    lists?: number;
+}
+
+export declare function measureLatency<T>(fn: () => Promise<T>): Promise<[number, T]>;
+
+declare type OperatorFn = (key: string, paramIndex: number, value?: any) => FilterOperator;
+
+export declare type OperatorType = BasicOperator | NumericOperator | ArrayOperator | ElementOperator | LogicalOperator | '$contains' | Exclude<RegexOperator, '$options'>;
+
+declare type PgCreateIndexArgs = [...CreateIndexArgs, IndexConfig?, boolean?];
+
+declare interface PgCreateIndexParams extends CreateIndexParams {
+    indexConfig?: IndexConfig;
+    buildIndex?: boolean;
+}
+
+declare type PgDefineIndexArgs = [string, 'cosine' | 'euclidean' | 'dotproduct', IndexConfig];
+
+declare interface PgDefineIndexParams {
+    indexName: string;
+    metric: 'cosine' | 'euclidean' | 'dotproduct';
+    indexConfig: IndexConfig;
+}
+
+/**
+ * Translates MongoDB-style filters to PG compatible filters.
+ *
+ * Key differences from MongoDB:
+ *
+ * Logical Operators ($and, $or, $nor):
+ * - Can be used at the top level or nested within fields
+ * - Can take either a single condition or an array of conditions
+ *
+ */
+export declare class PGFilterTranslator extends BaseFilterTranslator {
+    protected getSupportedOperators(): OperatorSupport;
+    translate(filter?: VectorFilter): VectorFilter;
+    private translateNode;
+    private translateRegexPattern;
+}
+
+declare interface PGIndexStats extends IndexStats {
+    type: IndexType;
+    config: {
+        m?: number;
+        efConstruction?: number;
+        lists?: number;
+        probes?: number;
+    };
+}
+export { PGIndexStats }
+export { PGIndexStats as PGIndexStats_alias_1 }
+
+declare type PgQueryVectorArgs = [...QueryVectorArgs, number?, number?, number?];
+
+declare interface PgQueryVectorParams extends QueryVectorParams {
+    minScore?: number;
+    /**
+     * HNSW search parameter. Controls the size of the dynamic candidate
+     * list during search. Higher values improve accuracy at the cost of speed.
+     */
+    ef?: number;
+    /**
+     * IVFFlat probe parameter. Number of cells to visit during search.
+     * Higher values improve accuracy at the cost of speed.
+     */
+    probes?: number;
+}
+
+declare class PgVector extends MastraVector {
+    private pool;
+    private describeIndexCache;
+    private createdIndexes;
+    private mutexesByName;
+    private schema?;
+    private setupSchemaPromise;
+    private installVectorExtensionPromise;
+    private vectorExtensionInstalled;
+    private schemaSetupComplete;
+    constructor(connectionString: string);
+    constructor(config: {
+        connectionString: string;
+        schemaName?: string;
+    });
+    private getMutexByName;
+    private getTableName;
+    transformFilter(filter?: VectorFilter): VectorFilter;
+    getIndexInfo(indexName: string): Promise<PGIndexStats>;
+    query(...args: ParamsToArgs<PgQueryVectorParams> | PgQueryVectorArgs): Promise<QueryResult[]>;
+    upsert(...args: ParamsToArgs<UpsertVectorParams>): Promise<string[]>;
+    private hasher;
+    private getIndexCacheKey;
+    private cachedIndexExists;
+    private setupSchema;
+    createIndex(...args: ParamsToArgs<PgCreateIndexParams> | PgCreateIndexArgs): Promise<void>;
+    /**
+     * @deprecated This function is deprecated. Use buildIndex instead
+     */
+    defineIndex(indexName: string, metric: "cosine" | "euclidean" | "dotproduct" | undefined, indexConfig: IndexConfig): Promise<void>;
+    buildIndex(...args: ParamsToArgs<PgDefineIndexParams> | PgDefineIndexArgs): Promise<void>;
+    private setupIndex;
+    private installVectorExtension;
+    listIndexes(): Promise<string[]>;
+    describeIndex(indexName: string): Promise<PGIndexStats>;
+    deleteIndex(indexName: string): Promise<void>;
+    truncateIndex(indexName: string): Promise<void>;
+    disconnect(): Promise<void>;
+    updateIndexById(indexName: string, id: string, update: {
+        vector?: number[];
+        metadata?: Record<string, any>;
+    }): Promise<void>;
+    deleteIndexById(indexName: string, id: string): Promise<void>;
+}
+export { PgVector }
+export { PgVector as PgVector_alias_1 }
+
+declare type PostgresConfig = {
+    schemaName?: string;
+    /**
+     * @deprecated Use `schemaName` instead. Support for `schema` will be removed in a future release.
+     */
+    schema?: string;
+} & ({
+    host: string;
+    port: number;
+    database: string;
+    user: string;
+    password: string;
+    ssl?: boolean | ISSLConfig;
+} | {
+    connectionString: string;
+});
+export { PostgresConfig }
+export { PostgresConfig as PostgresConfig_alias_1 }
+
+declare class PostgresStore extends MastraStorage {
+    private db;
+    private pgp;
+    private schema?;
+    private setupSchemaPromise;
+    private schemaSetupComplete;
+    constructor(config: PostgresConfig);
+    private getTableName;
+    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
+    private transformEvalRow;
+    batchInsert({ tableName, records }: {
+        tableName: TABLE_NAMES;
+        records: Record<string, any>[];
+    }): Promise<void>;
+    getTraces({ name, scope, page, perPage, attributes, filters, }: {
+        name?: string;
+        scope?: string;
+        page: number;
+        perPage: number;
+        attributes?: Record<string, string>;
+        filters?: Record<string, any>;
+    }): Promise<any[]>;
+    private setupSchema;
+    createTable({ tableName, schema, }: {
+        tableName: TABLE_NAMES;
+        schema: Record<string, StorageColumn>;
+    }): Promise<void>;
+    clearTable({ tableName }: {
+        tableName: TABLE_NAMES;
+    }): Promise<void>;
+    insert({ tableName, record }: {
+        tableName: TABLE_NAMES;
+        record: Record<string, any>;
+    }): Promise<void>;
+    load<R>({ tableName, keys }: {
+        tableName: TABLE_NAMES;
+        keys: Record<string, string>;
+    }): Promise<R | null>;
+    getThreadById({ threadId }: {
+        threadId: string;
+    }): Promise<StorageThreadType | null>;
+    getThreadsByResourceId({ resourceId }: {
+        resourceId: string;
+    }): Promise<StorageThreadType[]>;
+    saveThread({ thread }: {
+        thread: StorageThreadType;
+    }): Promise<StorageThreadType>;
+    updateThread({ id, title, metadata, }: {
+        id: string;
+        title: string;
+        metadata: Record<string, unknown>;
+    }): Promise<StorageThreadType>;
+    deleteThread({ threadId }: {
+        threadId: string;
+    }): Promise<void>;
+    getMessages<T = unknown>({ threadId, selectBy }: StorageGetMessagesArg): Promise<T>;
+    saveMessages({ messages }: {
+        messages: MessageType[];
+    }): Promise<MessageType[]>;
+    persistWorkflowSnapshot({ workflowName, runId, snapshot, }: {
+        workflowName: string;
+        runId: string;
+        snapshot: WorkflowRunState;
+    }): Promise<void>;
+    loadWorkflowSnapshot({ workflowName, runId, }: {
+        workflowName: string;
+        runId: string;
+    }): Promise<WorkflowRunState | null>;
+    getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, }?: {
+        workflowName?: string;
+        fromDate?: Date;
+        toDate?: Date;
+        limit?: number;
+        offset?: number;
+    }): Promise<{
+        runs: Array<{
+            workflowName: string;
+            runId: string;
+            snapshot: WorkflowRunState | string;
+            createdAt: Date;
+            updatedAt: Date;
+        }>;
+        total: number;
+    }>;
+    close(): Promise<void>;
+}
+export { PostgresStore }
+export { PostgresStore as PostgresStore_alias_1 }
+
+export declare interface TestConfig {
+    dimension: number;
+    size: number;
+    k: number;
+    queryCount: number;
+}
+
+export declare interface TestResult {
+    distribution: string;
+    dimension: number;
+    type: IndexType;
+    size: number;
+    k?: number;
+    metrics: {
+        recall?: number;
+        minRecall?: number;
+        maxRecall?: number;
+        latency?: {
+            p50: number;
+            p95: number;
+            lists?: number;
+            vectorsPerList?: number;
+            m?: number;
+            ef?: number;
+        };
+        clustering?: {
+            numLists?: number;
+            avgVectorsPerList?: number;
+            recommendedLists?: number;
+            distribution?: string;
+        };
+    };
+}
+
+export declare function warmupQuery(vectorDB: PgVector, indexName: string, dimension: number, k: number): Promise<void>;
+
+export { }