@mastra/pg 1.0.0-beta.12 → 1.0.0-beta.14

package/dist/docs/storage/01-reference.md

@@ -5,27 +5,27 @@
 
 ---
 
-## Reference: Storage Composition
+## Reference: Composite Storage
 
 > Documentation for combining multiple storage backends in Mastra.
 
-MastraStorage can compose storage domains from different adapters. Use it when you need different databases for different purposes. For example, use LibSQL for memory and PostgreSQL for workflows.
+`MastraStorage` can compose storage domains from different providers. Use it when you need different databases for different purposes. For example, use LibSQL for memory and PostgreSQL for workflows.
 
 ## Installation
 
-MastraStorage is included in `@mastra/core`:
+`MastraStorage` is included in `@mastra/core`:
 
 ```bash
 npm install @mastra/core@beta
 ```
 
-You'll also need to install the storage adapters you want to compose:
+You'll also need to install the storage providers you want to compose:
 
 ```bash
 npm install @mastra/pg@beta @mastra/libsql@beta
 ```
 
-## Storage Domains
+## Storage domains
 
 Mastra organizes storage into five specialized domains, each handling a specific type of data. Each domain can be backed by a different storage adapter, and domain classes are exported from each storage package.
 
@@ -43,13 +43,13 @@ Mastra organizes storage into five specialized domains, each handling a specific
 
 Import domain classes directly from each store package and compose them:
 
-```typescript
+```typescript title="src/mastra/index.ts"
 import { MastraStorage } from "@mastra/core/storage";
 import { WorkflowsPG, ScoresPG } from "@mastra/pg";
 import { MemoryLibSQL } from "@mastra/libsql";
 import { Mastra } from "@mastra/core";
 
-const mastra = new Mastra({
+export const mastra = new Mastra({
   storage: new MastraStorage({
     id: "composite",
     domains: {
@@ -65,7 +65,7 @@ const mastra = new Mastra({
 
 Use `default` to specify a fallback storage, then override specific domains:
 
-```typescript
+```typescript title="src/mastra/index.ts"
 import { MastraStorage } from "@mastra/core/storage";
 import { PostgresStore } from "@mastra/pg";
 import { MemoryLibSQL } from "@mastra/libsql";
@@ -76,7 +76,7 @@ const pgStore = new PostgresStore({
   connectionString: process.env.DATABASE_URL,
 });
 
-const mastra = new Mastra({
+export const mastra = new Mastra({
   storage: new MastraStorage({
     id: "composite",
     default: pgStore,
@@ -91,9 +91,9 @@ const mastra = new Mastra({
 
 ## Initialization
 
-MastraStorage initializes each configured domain independently. When passed to the Mastra class, `init()` is called automatically:
+`MastraStorage` initializes each configured domain independently. When passed to the Mastra class, `init()` is called automatically:
 
-```typescript
+```typescript title="src/mastra/index.ts"
 import { MastraStorage } from "@mastra/core/storage";
 import { MemoryPG, WorkflowsPG, ScoresPG } from "@mastra/pg";
 import { Mastra } from "@mastra/core";
@@ -107,7 +107,7 @@ const storage = new MastraStorage({
   },
 });
 
-const mastra = new Mastra({
+export const mastra = new Mastra({
   storage, // init() called automatically
 });
 ```
@@ -132,7 +132,7 @@ const memoryStore = await storage.getStore("memory");
 const thread = await memoryStore?.getThreadById({ threadId: "..." });
 ```
 
-## Use Cases
+## Use cases
 
 ### Separate databases for different workloads
 
@@ -197,6 +197,7 @@ The DynamoDB storage implementation provides a scalable and performant NoSQL dat
 - Compatible with AWS DynamoDB Local for development
 - Stores Thread, Message, Trace, Eval, and Workflow data
 - Optimized for serverless environments
+- Configurable TTL (Time To Live) for automatic data expiration per entity type
 
 ## Installation
 
@@ -224,7 +225,7 @@ import { DynamoDBStore } from "@mastra/dynamodb";
 
 // Initialize the DynamoDB storage
 const storage = new DynamoDBStore({
-  name: "dynamodb", // A name for this storage instance
+  id: "dynamodb", // Unique identifier for this storage instance
   config: {
     tableName: "mastra-single-table", // Name of your DynamoDB table
     region: "us-east-1", // Optional: AWS region, defaults to 'us-east-1'
@@ -258,7 +259,7 @@ For local development, you can use [DynamoDB Local](https://docs.aws.amazon.com/
     import { DynamoDBStore } from "@mastra/dynamodb";
 
     const storage = new DynamoDBStore({
-      name: "dynamodb-local",
+      id: "dynamodb-local",
       config: {
         tableName: "mastra-single-table", // Ensure this table is created in your local DynamoDB
         region: "localhost", // Can be any string for local, 'localhost' is common
@@ -274,6 +275,96 @@ For local development, you can use [DynamoDB Local](https://docs.aws.amazon.com/
 
 ## Parameters
 
+## TTL (Time To Live) Configuration
+
+DynamoDB TTL allows you to automatically delete items after a specified time period. This is useful for:
+
+- **Cost optimization**: Automatically remove old data to reduce storage costs
+- **Data lifecycle management**: Implement retention policies for compliance
+- **Performance**: Prevent tables from growing indefinitely
+- **Privacy compliance**: Automatically purge personal data after specified periods
+
+### Enabling TTL
+
+To use TTL, you must:
+
+1. **Configure TTL in DynamoDBStore** (shown below)
+2. **Enable TTL on your DynamoDB table** via AWS Console or CLI, specifying the attribute name (default: `ttl`)
+
+```typescript
+import { DynamoDBStore } from "@mastra/dynamodb";
+
+const storage = new DynamoDBStore({
+  name: "dynamodb",
+  config: {
+    tableName: "mastra-single-table",
+    region: "us-east-1",
+    ttl: {
+      // Messages expire after 30 days
+      message: {
+        enabled: true,
+        defaultTtlSeconds: 30 * 24 * 60 * 60, // 30 days
+      },
+      // Threads expire after 90 days
+      thread: {
+        enabled: true,
+        defaultTtlSeconds: 90 * 24 * 60 * 60, // 90 days
+      },
+      // Traces expire after 7 days with custom attribute name
+      trace: {
+        enabled: true,
+        attributeName: "expiresAt", // Custom TTL attribute
+        defaultTtlSeconds: 7 * 24 * 60 * 60, // 7 days
+      },
+      // Workflow snapshots don't expire
+      workflow_snapshot: {
+        enabled: false,
+      },
+    },
+  },
+});
+```
+
+### Supported Entity Types
+
+TTL can be configured for these entity types:
+
+| Entity | Description |
+|--------|-------------|
+| `thread` | Conversation threads |
+| `message` | Messages within threads |
+| `trace` | Observability traces |
+| `eval` | Evaluation results |
+| `workflow_snapshot` | Workflow state snapshots |
+| `resource` | User/resource data |
+| `score` | Scoring results |
+
+### TTL Entity Configuration
+
+Each entity type accepts the following configuration:
+
+### Enabling TTL on Your DynamoDB Table
+
+After configuring TTL in your code, you must enable TTL on the DynamoDB table itself:
+
+**Using AWS CLI:**
+
+```bash
+aws dynamodb update-time-to-live \
+  --table-name mastra-single-table \
+  --time-to-live-specification "Enabled=true, AttributeName=ttl"
+```
+
+**Using AWS Console:**
+
+1. Go to the DynamoDB console
+2. Select your table
+3. Go to "Additional settings" tab
+4. Under "Time to Live (TTL)", click "Manage TTL"
+5. Enable TTL and specify the attribute name (default: `ttl`)
+
+> **Note**: DynamoDB deletes expired items within 48 hours after expiration. Items remain queryable until actually deleted.
+
 ## AWS IAM Permissions
 
 The IAM role or user executing the code needs appropriate permissions to interact with the specified DynamoDB table and its indexes. Below is a sample policy. Replace `${YOUR_TABLE_NAME}` with your actual table name and `${YOUR_AWS_REGION}` and `${YOUR_AWS_ACCOUNT_ID}` with appropriate values.
@@ -431,6 +522,7 @@ import { Mastra } from "@mastra/core";
 import { PostgresStore } from "@mastra/pg";
 
 const storage = new PostgresStore({
+  id: 'pg-storage',
   connectionString: process.env.DATABASE_URL,
 });
 
@@ -477,6 +569,75 @@ This enables direct queries and custom transaction management. When using these
 
 This approach is intended for advanced scenarios where low-level access is required.
 
+### Using with Next.js
+
+When using `PostgresStore` in Next.js applications, [Hot Module Replacement (HMR)](https://nextjs.org/docs/architecture/fast-refresh) during development can cause multiple storage instances to be created, resulting in this warning:
+
+```
+WARNING: Creating a duplicate database object for the same connection.
+```
+
+To prevent this, store the `PostgresStore` instance on the global object so it persists across HMR reloads:
+
+```typescript title="src/mastra/storage.ts"
+import { PostgresStore } from "@mastra/pg";
+import { Memory } from "@mastra/memory";
+
+// Extend the global type to include our instances
+declare global {
+  var pgStore: PostgresStore | undefined;
+  var memory: Memory | undefined;
+}
+
+// Get or create the PostgresStore instance
+function getPgStore(): PostgresStore {
+  if (!global.pgStore) {
+    if (!process.env.DATABASE_URL) {
+      throw new Error("DATABASE_URL is not defined in environment variables");
+    }
+    global.pgStore = new PostgresStore({
+      id: "pg-storage",
+      connectionString: process.env.DATABASE_URL,
+      ssl:
+        process.env.DATABASE_SSL === "true"
+          ? { rejectUnauthorized: false }
+          : false,
+    });
+  }
+  return global.pgStore;
+}
+
+// Get or create the Memory instance
+function getMemory(): Memory {
+  if (!global.memory) {
+    global.memory = new Memory({
+      storage: getPgStore(),
+    });
+  }
+  return global.memory;
+}
+
+export const storage = getPgStore();
+export const memory = getMemory();
+```
+
+Then use the exported instances in your Mastra configuration:
+
+```typescript title="src/mastra/index.ts"
+import { Mastra } from "@mastra/core/mastra";
+import { storage } from "./storage";
+
+export const mastra = new Mastra({
+  storage,
+  // ...other config
+});
+```
+
+This pattern ensures only one `PostgresStore` instance is created regardless of how many times the module is reloaded during development. The same pattern can be applied to other storage providers like `LibSQLStore`.
+
+> **Note:**
+This singleton pattern is only necessary during local development with HMR. In production builds, modules are only loaded once.
+
 ## Usage Example
 
 ### Adding memory to an agent

package/dist/index.cjs

@@ -2102,7 +2102,7 @@ var PgDB = class extends base.MastraBase {
           SELECT 1 FROM information_schema.tables
           WHERE table_schema = $1 AND table_name = $2
         )`,
-        [this.schemaName || "mastra", tableName]
+        [this.schemaName || "public", tableName]
       );
       if (tableExists?.exists) {
         await this.client.none(`TRUNCATE TABLE ${tableNameWithSchema} CASCADE`);
@@ -3219,27 +3219,52 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
       );
     }
   }
-  async listThreadsByResourceId(args) {
-    const { resourceId, page = 0, perPage: perPageInput, orderBy } = args;
-    if (page < 0) {
+  async listThreads(args) {
+    const { page = 0, perPage: perPageInput, orderBy, filter } = args;
+    try {
+      this.validatePaginationInput(page, perPageInput ?? 100);
+    } catch (error$1) {
       throw new error.MastraError({
-        id: storage.createStorageErrorId("PG", "LIST_THREADS_BY_RESOURCE_ID", "INVALID_PAGE"),
+        id: storage.createStorageErrorId("PG", "LIST_THREADS", "INVALID_PAGE"),
         domain: error.ErrorDomain.STORAGE,
         category: error.ErrorCategory.USER,
-        text: "Page number must be non-negative",
-        details: {
-          resourceId,
-          page
-        }
+        text: error$1 instanceof Error ? error$1.message : "Invalid pagination parameters",
+        details: { page, ...perPageInput !== void 0 && { perPage: perPageInput } }
       });
     }
-    const { field, direction } = this.parseOrderBy(orderBy);
     const perPage = storage.normalizePerPage(perPageInput, 100);
+    try {
+      this.validateMetadataKeys(filter?.metadata);
+    } catch (error$1) {
+      throw new error.MastraError({
+        id: storage.createStorageErrorId("PG", "LIST_THREADS", "INVALID_METADATA_KEY"),
+        domain: error.ErrorDomain.STORAGE,
+        category: error.ErrorCategory.USER,
+        text: error$1 instanceof Error ? error$1.message : "Invalid metadata key",
+        details: { metadataKeys: filter?.metadata ? Object.keys(filter.metadata).join(", ") : "" }
+      });
+    }
+    const { field, direction } = this.parseOrderBy(orderBy);
     const { offset, perPage: perPageForResponse } = storage.calculatePagination(page, perPageInput, perPage);
     try {
       const tableName = getTableName3({ indexName: storage.TABLE_THREADS, schemaName: getSchemaName3(this.#schema) });
-      const baseQuery = `FROM ${tableName} WHERE "resourceId" = $1`;
-      const queryParams = [resourceId];
+      const whereClauses = [];
+      const queryParams = [];
+      let paramIndex = 1;
+      if (filter?.resourceId) {
+        whereClauses.push(`"resourceId" = $${paramIndex}`);
+        queryParams.push(filter.resourceId);
+        paramIndex++;
+      }
+      if (filter?.metadata && Object.keys(filter.metadata).length > 0) {
+        for (const [key, value] of Object.entries(filter.metadata)) {
+          whereClauses.push(`metadata::jsonb @> $${paramIndex}::jsonb`);
+          queryParams.push(JSON.stringify({ [key]: value }));
+          paramIndex++;
+        }
+      }
+      const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
+      const baseQuery = `FROM ${tableName} ${whereClause}`;
       const countQuery = `SELECT COUNT(*) ${baseQuery}`;
       const countResult = await this.#db.client.one(countQuery, queryParams);
       const total = parseInt(countResult.count, 10);
@@ -3253,7 +3278,7 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
         };
       }
       const limitValue = perPageInput === false ? total : perPage;
-      const dataQuery = `SELECT id, "resourceId", title, metadata, "createdAt", "createdAtZ", "updatedAt", "updatedAtZ" ${baseQuery} ORDER BY "${field}" ${direction} LIMIT $2 OFFSET $3`;
+      const dataQuery = `SELECT id, "resourceId", title, metadata, "createdAt", "createdAtZ", "updatedAt", "updatedAtZ" ${baseQuery} ORDER BY "${field}" ${direction} LIMIT $${paramIndex} OFFSET $${paramIndex + 1}`;
       const rows = await this.#db.client.manyOrNone(
         dataQuery,
         [...queryParams, limitValue, offset]
@@ -3277,11 +3302,12 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
     } catch (error$1) {
       const mastraError = new error.MastraError(
         {
-          id: storage.createStorageErrorId("PG", "LIST_THREADS_BY_RESOURCE_ID", "FAILED"),
+          id: storage.createStorageErrorId("PG", "LIST_THREADS", "FAILED"),
           domain: error.ErrorDomain.STORAGE,
           category: error.ErrorCategory.THIRD_PARTY,
           details: {
-            resourceId,
+            ...filter?.resourceId && { resourceId: filter.resourceId },
+            hasMetadataFilter: !!filter?.metadata,
             page
           }
         },
@@ -4757,6 +4783,11 @@ var ScoresPG = class _ScoresPG extends storage.ScoresStorage {
   }
   async init() {
     await this.#db.createTable({ tableName: storage.TABLE_SCORERS, schema: storage.TABLE_SCHEMAS[storage.TABLE_SCORERS] });
+    await this.#db.alterTable({
+      tableName: storage.TABLE_SCORERS,
+      schema: storage.TABLE_SCHEMAS[storage.TABLE_SCORERS],
+      ifNotExists: ["spanId", "requestContext"]
+    });
     await this.createDefaultIndexes();
     await this.createCustomIndexes();
   }
@@ -5108,23 +5139,8 @@ function getTableName5({ indexName, schemaName }) {
   const quotedIndexName = `"${indexName}"`;
   return schemaName ? `${schemaName}.${quotedIndexName}` : quotedIndexName;
 }
-function parseWorkflowRun(row) {
-  let parsedSnapshot = row.snapshot;
-  if (typeof parsedSnapshot === "string") {
-    try {
-      parsedSnapshot = JSON.parse(row.snapshot);
-    } catch (e) {
-      console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
-    }
-  }
-  return {
-    workflowName: row.workflow_name,
-    runId: row.run_id,
-    snapshot: parsedSnapshot,
-    resourceId: row.resourceId,
-    createdAt: new Date(row.createdAtZ || row.createdAt),
-    updatedAt: new Date(row.updatedAtZ || row.updatedAt)
-  };
+function sanitizeJsonForPg(jsonString) {
+  return jsonString.replace(/\\u(0000|[Dd][89A-Fa-f][0-9A-Fa-f]{2})/g, "");
 }
 var WorkflowsPG = class _WorkflowsPG extends storage.WorkflowsStorage {
   #db;
@@ -5141,6 +5157,24 @@ var WorkflowsPG = class _WorkflowsPG extends storage.WorkflowsStorage {
     this.#skipDefaultIndexes = skipDefaultIndexes;
     this.#indexes = indexes?.filter((idx) => _WorkflowsPG.MANAGED_TABLES.includes(idx.table));
   }
+  parseWorkflowRun(row) {
+    let parsedSnapshot = row.snapshot;
+    if (typeof parsedSnapshot === "string") {
+      try {
+        parsedSnapshot = JSON.parse(row.snapshot);
+      } catch (e) {
+        this.logger.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
+      }
+    }
+    return {
+      workflowName: row.workflow_name,
+      runId: row.run_id,
+      snapshot: parsedSnapshot,
+      resourceId: row.resourceId,
+      createdAt: new Date(row.createdAtZ || row.createdAt),
+      updatedAt: new Date(row.updatedAtZ || row.updatedAt)
+    };
+  }
   /**
    * Returns default index definitions for the workflows domain tables.
    * Currently no default indexes are defined for workflows.
@@ -5213,12 +5247,13 @@ var WorkflowsPG = class _WorkflowsPG extends storage.WorkflowsStorage {
       const now = /* @__PURE__ */ new Date();
       const createdAtValue = createdAt ? createdAt : now;
       const updatedAtValue = updatedAt ? updatedAt : now;
+      const sanitizedSnapshot = sanitizeJsonForPg(JSON.stringify(snapshot));
       await this.#db.client.none(
         `INSERT INTO ${getTableName5({ indexName: storage.TABLE_WORKFLOW_SNAPSHOT, schemaName: getSchemaName5(this.#schema) })} (workflow_name, run_id, "resourceId", snapshot, "createdAt", "updatedAt")
                  VALUES ($1, $2, $3, $4, $5, $6)
                  ON CONFLICT (workflow_name, run_id) DO UPDATE
                  SET "resourceId" = $3, snapshot = $4, "updatedAt" = $6`,
-        [workflowName, runId, resourceId, JSON.stringify(snapshot), createdAtValue, updatedAtValue]
+        [workflowName, runId, resourceId, sanitizedSnapshot, createdAtValue, updatedAtValue]
       );
     } catch (error$1) {
       throw new error.MastraError(
@@ -5281,7 +5316,7 @@ var WorkflowsPG = class _WorkflowsPG extends storage.WorkflowsStorage {
       if (!result) {
         return null;
       }
-      return parseWorkflowRun(result);
+      return this.parseWorkflowRun(result);
     } catch (error$1) {
       throw new error.MastraError(
         {
@@ -5337,7 +5372,9 @@ var WorkflowsPG = class _WorkflowsPG extends storage.WorkflowsStorage {
         paramIndex++;
       }
       if (status) {
-        conditions.push(`snapshot::jsonb ->> 'status' = $${paramIndex}`);
+        conditions.push(
+          `regexp_replace(snapshot::text, '\\\\u(0000|[Dd][89A-Fa-f][0-9A-Fa-f]{2})', '', 'g')::jsonb ->> 'status' = $${paramIndex}`
+        );
         values.push(status);
         paramIndex++;
       }
@@ -5382,7 +5419,7 @@ var WorkflowsPG = class _WorkflowsPG extends storage.WorkflowsStorage {
       const queryValues = usePagination ? [...values, normalizedPerPage, offset] : values;
       const result = await this.#db.client.manyOrNone(query, queryValues);
       const runs = (result || []).map((row) => {
-        return parseWorkflowRun(row);
+        return this.parseWorkflowRun(row);
       });
       return { runs, total: total || runs.length };
     } catch (error$1) {
@@ -5415,7 +5452,7 @@ var PostgresStore = class extends storage.MastraStorage {
     try {
       validateConfig("PostgresStore", config);
       super({ id: config.id, name: "PostgresStore", disableInit: config.disableInit });
-      this.schema = config.schemaName || "public";
+      this.schema = utils.parseSqlIdentifier(config.schemaName || "public", "schema name");
       if (isPoolConfig(config)) {
         this.#pool = config.pool;
         this.#ownsPool = false;