@mastra/pg 1.7.2 → 1.8.0

package/dist/index.cjs

@@ -748,11 +748,22 @@ var PgVector = class extends vector.MastraVector {
     }
     return major > 0 || major === 0 && minor >= 7;
   }
+  /** Checks if pgvector >= 0.7.0 (required for bit type). */
+  supportsBit() {
+    return this.supportsHalfvec();
+  }
+  /** Checks if pgvector >= 0.7.0 (required for sparsevec type). */
+  supportsSparsevec() {
+    return this.supportsHalfvec();
+  }
   /**
    * Gets the properly qualified vector type name
-   * @param vectorType - The type of vector storage ('vector' or 'halfvec')
+   * @param vectorType - The type of vector storage
    */
-  getVectorTypeName(vectorType = "vector") {
+  getVectorTypeName(vectorType = "vector", dimension) {
+    if (vectorType === "bit") {
+      return dimension ? `bit(${dimension})` : "bit";
+    }
     if (this.vectorExtensionSchema) {
       if (this.vectorExtensionSchema === "pg_catalog") {
         return vectorType;
@@ -763,20 +774,76 @@ var PgVector = class extends vector.MastraVector {
     return vectorType;
   }
   /**
-   * Gets the operator class for index creation based on metric and vector type.
-   * pgvector uses different operator classes for vector vs halfvec types.
+   * Returns the operator class, distance operator, and score expression for a
+   * standard (non-bit) vector type prefix and metric.
    */
-  getMetricOperatorClass(metric, vectorType) {
-    const prefix = vectorType === "halfvec" ? "halfvec" : "vector";
+  getMetricOps(prefix, metric) {
     switch (metric) {
-      case "cosine":
-        return `${prefix}_cosine_ops`;
       case "euclidean":
-        return `${prefix}_l2_ops`;
+        return {
+          operatorClass: `${prefix}_l2_ops`,
+          distanceOperator: "<->",
+          scoreExpr: (d) => `1.0 / (1.0 + (${d}))`
+        };
       case "dotproduct":
-        return `${prefix}_ip_ops`;
+        return {
+          operatorClass: `${prefix}_ip_ops`,
+          distanceOperator: "<#>",
+          scoreExpr: (d) => `(${d}) * -1`
+        };
+      default:
+        return {
+          operatorClass: `${prefix}_cosine_ops`,
+          distanceOperator: "<=>",
+          scoreExpr: (d) => `1 - (${d})`
+        };
+    }
+  }
+  /**
+   * Returns all vector-type-specific operations for the given vectorType and metric.
+   */
+  getVectorOps(vectorType, metric) {
+    switch (vectorType) {
+      case "bit":
+        return {
+          operatorClass: metric === "jaccard" ? "bit_jaccard_ops" : "bit_hamming_ops",
+          distanceOperator: metric === "jaccard" ? "<%>" : "<~>",
+          scoreExpr: (d) => metric === "jaccard" ? `1 - (${d})` : `1 - ((${d})::float / bit_length(embedding))`,
+          formatVector: (v) => v.map((x) => x ? "1" : "0").join(""),
+          parseEmbedding: (e) => e.split("").map((c) => c === "1" ? 1 : 0)
+        };
+      case "sparsevec":
+        return {
+          ...this.getMetricOps("sparsevec", metric),
+          formatVector: (v, dimension) => {
+            const dim = dimension ?? v.length;
+            const nonZero = v.map((val, i) => val !== 0 ? `${i + 1}:${val}` : null).filter(Boolean).join(",");
+            return `{${nonZero}}/${dim}`;
+          },
+          parseEmbedding: (e) => {
+            const match = e.match(/^\{([^}]*)\}\/(\d+)$/);
+            if (!match) return [];
+            const dim = parseInt(match[2], 10);
+            const result = new Array(dim).fill(0);
+            const entries = match[1];
+            if (entries.trim()) {
+              for (const entry of entries.split(",")) {
+                const [idxStr, valStr] = entry.split(":");
+                const idx = parseInt(idxStr.trim(), 10) - 1;
+                result[idx] = parseFloat(valStr.trim());
+              }
+            }
+            return result;
+          }
+        };
+      case "halfvec":
+      case "vector":
       default:
-        return `${prefix}_cosine_ops`;
+        return {
+          ...this.getMetricOps(vectorType === "halfvec" ? "halfvec" : "vector", metric),
+          formatVector: (v) => `[${v.join(",")}]`,
+          parseEmbedding: (e) => JSON.parse(e)
+        };
     }
   }
   getTableName(indexName) {
@@ -879,10 +946,12 @@ var PgVector = class extends vector.MastraVector {
     const client = await this.pool.connect();
     try {
       await client.query("BEGIN");
-      const vectorStr = `[${queryVector.join(",")}]`;
       const translatedFilter = this.transformFilter(filter);
       const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter, minScore, topK);
       const indexInfo = await this.getIndexInfo({ indexName });
+      const metric = indexInfo.metric ?? "cosine";
+      const ops = this.getVectorOps(indexInfo.vectorType, metric);
+      const vectorStr = ops.formatVector(queryVector, indexInfo.dimension);
       if (indexInfo.type === "hnsw") {
         const calculatedEf = ef ?? Math.max(topK, (indexInfo?.config?.m ?? 16) * topK);
         const searchEf = Math.min(1e3, Math.max(1, calculatedEf));
@@ -892,12 +961,14 @@ var PgVector = class extends vector.MastraVector {
         await client.query(`SET LOCAL ivfflat.probes = ${probes}`);
       }
       const { tableName } = this.getTableName(indexName);
-      const qualifiedVectorType = this.getVectorTypeName(indexInfo.vectorType);
+      const qualifiedVectorType = this.getVectorTypeName(indexInfo.vectorType, indexInfo.dimension);
+      const distanceExpr = `embedding ${ops.distanceOperator} '${vectorStr}'::${qualifiedVectorType}`;
+      const scoreExpr = ops.scoreExpr(distanceExpr);
       const query = `
         WITH vector_scores AS (
           SELECT
             vector_id as id,
-            1 - (embedding <=> '${vectorStr}'::${qualifiedVectorType}) as score,
+            ${scoreExpr} as score,
             metadata
             ${includeVector ? ", embedding" : ""}
           FROM ${tableName}
@@ -914,7 +985,7 @@ var PgVector = class extends vector.MastraVector {
         id,
         score,
         metadata,
-        ...includeVector && embedding && { vector: JSON.parse(embedding) }
+        ...includeVector && embedding && { vector: ops.parseEmbedding(embedding) }
       }));
     } catch (error$1) {
       await client.query("ROLLBACK");
@@ -963,8 +1034,10 @@ var PgVector = class extends vector.MastraVector {
       }
       const vectorIds = ids || vectors.map(() => crypto.randomUUID());
       const indexInfo = await this.getIndexInfo({ indexName });
-      const qualifiedVectorType = this.getVectorTypeName(indexInfo.vectorType);
+      const qualifiedVectorType = this.getVectorTypeName(indexInfo.vectorType, indexInfo.dimension);
+      const ops = this.getVectorOps(indexInfo.vectorType, indexInfo.metric ?? "cosine");
       for (let i = 0; i < vectors.length; i++) {
+        const vectorStr = ops.formatVector(vectors[i], indexInfo.dimension);
         const query = `
           INSERT INTO ${tableName} (vector_id, embedding, metadata)
           VALUES ($1, $2::${qualifiedVectorType}, $3::jsonb)
@@ -974,7 +1047,7 @@ var PgVector = class extends vector.MastraVector {
             metadata = $3::jsonb
           RETURNING embedding::text
         `;
-        await client.query(query, [vectorIds[i], `[${vectors[i]?.join(",")}]`, JSON.stringify(metadata?.[i] || {})]);
+        await client.query(query, [vectorIds[i], vectorStr, JSON.stringify(metadata?.[i] || {})]);
       }
       await client.query("COMMIT");
       this.logger?.debug(`Upserted ${vectors.length} vectors to ${indexName}`, {
@@ -1030,9 +1103,17 @@ var PgVector = class extends vector.MastraVector {
     dimension,
     metric,
     type,
-    vectorType = "vector"
+    vectorType = "vector",
+    metadataIndexes
   }) {
-    const input = indexName + dimension + metric + (type || "ivfflat") + vectorType;
+    const input = JSON.stringify([
+      indexName,
+      dimension,
+      metric,
+      type || "ivfflat",
+      vectorType,
+      metadataIndexes?.toSorted() ?? []
+    ]);
     return (await this.hasher).h32(input);
   }
   cachedIndexExists(indexName, newKey) {
@@ -1083,11 +1164,13 @@ var PgVector = class extends vector.MastraVector {
   async createIndex({
     indexName,
     dimension,
-    metric = "cosine",
+    metric: rawMetric = "cosine",
     indexConfig = {},
     buildIndex = true,
-    vectorType = "vector"
+    vectorType = "vector",
+    metadataIndexes
   }) {
+    const metric = vectorType === "bit" && rawMetric !== "hamming" && rawMetric !== "jaccard" ? "hamming" : rawMetric;
     const { tableName } = this.getTableName(indexName);
     try {
       if (!indexName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
@@ -1096,8 +1179,17 @@ var PgVector = class extends vector.MastraVector {
       if (!Number.isInteger(dimension) || dimension <= 0) {
         throw new Error("Dimension must be a positive integer");
       }
-      if (vectorType !== "vector" && vectorType !== "halfvec") {
-        throw new Error('vectorType must be "vector" or "halfvec"');
+      if (vectorType !== "vector" && vectorType !== "halfvec" && vectorType !== "bit" && vectorType !== "sparsevec") {
+        throw new Error('vectorType must be "vector", "halfvec", "bit", or "sparsevec"');
+      }
+      if (vectorType === "bit" && dimension > 64e3) {
+        throw new Error("bit vectors support up to 64,000 dimensions for indexes");
+      }
+      if ((metric === "hamming" || metric === "jaccard") && vectorType !== "bit") {
+        throw new Error(`${metric} metric is only valid with vectorType 'bit'`);
+      }
+      if (indexConfig?.type === "ivfflat" && vectorType === "bit" && metric === "jaccard") {
+        throw new Error("IVFFlat indexes do not support Jaccard distance for bit vectors. Use HNSW instead.");
       }
     } catch (error$1) {
       const mastraError = new error.MastraError(
@@ -1119,7 +1211,8 @@ var PgVector = class extends vector.MastraVector {
       dimension,
       type: indexConfig.type,
       metric,
-      vectorType
+      vectorType,
+      metadataIndexes
     });
     if (this.cachedIndexExists(indexName, indexCacheKey)) {
       return;
@@ -1147,6 +1240,33 @@ var PgVector = class extends vector.MastraVector {
             }
           });
         }
+        if (vectorType === "bit" && !this.supportsBit()) {
+          throw new error.MastraError({
+            id: storage.createVectorErrorId("PG", "CREATE_INDEX", "VECTOR_TYPE_NOT_SUPPORTED"),
+            text: `${vectorType} type requires pgvector >= 0.7.0, but version ${this.vectorExtensionVersion || "unknown"} is installed. Either upgrade pgvector or use vectorType: 'vector'.`,
+            domain: error.ErrorDomain.MASTRA_VECTOR,
+            category: error.ErrorCategory.USER,
+            details: {
+              indexName,
+              vectorType,
+              installedVersion: this.vectorExtensionVersion
+            }
+          });
+        }
+        if (vectorType === "sparsevec" && !this.supportsSparsevec()) {
+          throw new error.MastraError({
+            id: storage.createVectorErrorId("PG", "CREATE_INDEX", "VECTOR_TYPE_NOT_SUPPORTED"),
+            text: `${vectorType} type requires pgvector >= 0.7.0, but version ${this.vectorExtensionVersion || "unknown"} is installed. Either upgrade pgvector or use vectorType: 'vector'.`,
+            domain: error.ErrorDomain.MASTRA_VECTOR,
+            category: error.ErrorCategory.USER,
+            details: {
+              indexName,
+              requestedVectorType: vectorType,
+              pgvectorVersion: this.vectorExtensionVersion || "unknown",
+              requiredVersion: "0.7.0"
+            }
+          });
+        }
         if (this.schema && this.vectorExtensionSchema && this.schema !== this.vectorExtensionSchema && this.vectorExtensionSchema !== "pg_catalog") {
           await client.query(`SET search_path TO ${this.getSchemaName()}, "${this.vectorExtensionSchema}"`);
         }
@@ -1164,6 +1284,9 @@ var PgVector = class extends vector.MastraVector {
         if (buildIndex) {
           await this.setupIndex({ indexName, metric, indexConfig, vectorType }, client);
         }
+        if (metadataIndexes?.length) {
+          await this.createMetadataIndexes(tableName, indexName, metadataIndexes);
+        }
       } catch (error) {
         this.createdIndexes.delete(indexName);
         this.indexVectorTypes.delete(indexName);
@@ -1187,10 +1310,10 @@ var PgVector = class extends vector.MastraVector {
       throw mastraError;
     });
   }
-  async buildIndex({ indexName, metric = "cosine", indexConfig }) {
+  async buildIndex({ indexName, metric = "cosine", indexConfig, vectorType }) {
     const client = await this.pool.connect();
     try {
-      await this.setupIndex({ indexName, metric, indexConfig }, client);
+      await this.setupIndex({ indexName, metric, indexConfig, vectorType }, client);
     } catch (error$1) {
       const mastraError = new error.MastraError(
         {
@@ -1213,7 +1336,26 @@ var PgVector = class extends vector.MastraVector {
     const mutex = this.getMutexByName(`build-${indexName}`);
     await mutex.runExclusive(async () => {
       const isConfigEmpty = !indexConfig || Object.keys(indexConfig).length === 0 || !indexConfig.type && !indexConfig.ivf && !indexConfig.hnsw;
-      const indexType = isConfigEmpty ? "ivfflat" : indexConfig.type || "ivfflat";
+      const defaultIndexType = vectorType === "sparsevec" ? "hnsw" : "ivfflat";
+      const indexType = isConfigEmpty ? defaultIndexType : indexConfig.type || defaultIndexType;
+      if (indexType === "ivfflat" && vectorType === "sparsevec") {
+        throw new error.MastraError({
+          id: storage.createVectorErrorId("PG", "BUILD_INDEX", "UNSUPPORTED_INDEX_TYPE"),
+          text: `IVFFlat indexes do not support sparsevec type. Use HNSW instead.`,
+          domain: error.ErrorDomain.MASTRA_VECTOR,
+          category: error.ErrorCategory.USER,
+          details: { indexName, vectorType, indexType }
+        });
+      }
+      if (indexType === "ivfflat" && vectorType === "bit" && metric === "jaccard") {
+        throw new error.MastraError({
+          id: storage.createVectorErrorId("PG", "BUILD_INDEX", "UNSUPPORTED_INDEX_TYPE"),
+          text: `IVFFlat indexes do not support Jaccard distance for bit vectors. Use HNSW instead.`,
+          domain: error.ErrorDomain.MASTRA_VECTOR,
+          category: error.ErrorCategory.USER,
+          details: { indexName, vectorType, indexType, metric }
+        });
+      }
       const { tableName, vectorIndexName } = this.getTableName(indexName);
       let existingIndexInfo = null;
       let dimension = 0;
@@ -1271,7 +1413,7 @@ var PgVector = class extends vector.MastraVector {
         return;
       }
       const effectiveVectorType = existingIndexInfo?.vectorType ?? vectorType;
-      const metricOp = this.getMetricOperatorClass(metric, effectiveVectorType);
+      const metricOp = this.getVectorOps(effectiveVectorType, metric).operatorClass;
       let indexSQL;
       if (indexType === "hnsw") {
         const m = indexConfig.hnsw?.m ?? 8;
@@ -1303,6 +1445,21 @@ var PgVector = class extends vector.MastraVector {
       await client.query(indexSQL);
     });
   }
+  async createMetadataIndexes(tableName, indexName, metadataFields) {
+    const hasher = await this.hasher;
+    for (const field of metadataFields) {
+      const fieldHash = hasher.h32(field).toString(16);
+      const prefix = indexName.slice(0, 63 - "_md__idx".length - fieldHash.length);
+      const metadataIdxName = `"${prefix}_md_${fieldHash}_idx"`;
+      const escapedField = field.replace(/'/g, "''");
+      await this.pool.query(
+        `
+        CREATE INDEX CONCURRENTLY IF NOT EXISTS ${metadataIdxName}
+        ON ${tableName} ((metadata->>'${escapedField}'))
+      `
+      );
+    }
+  }
   async installVectorExtension(client) {
     if (this.vectorExtensionInstalled) {
       return;
@@ -1389,7 +1546,7 @@ var PgVector = class extends vector.MastraVector {
           WHERE c.table_schema = t.table_schema
           AND c.table_name = t.table_name
           AND c.column_name = 'embedding'
-          AND c.udt_name IN ('vector', 'halfvec')
+          AND c.udt_name IN ('vector', 'halfvec', 'bit', 'sparsevec')
         )
         AND EXISTS (
           SELECT 1
@@ -1432,14 +1589,15 @@ var PgVector = class extends vector.MastraVector {
         FROM information_schema.columns
         WHERE table_schema = $1
           AND table_name = $2
-          AND udt_name IN ('vector', 'halfvec')
+          AND udt_name IN ('vector', 'halfvec', 'bit', 'sparsevec')
         LIMIT 1;
       `;
       const tableExists = await client.query(tableExistsQuery, [this.schema || "public", indexName]);
       if (tableExists.rows.length === 0) {
         throw new Error(`Vector table ${tableName} does not exist`);
       }
-      const vectorType = tableExists.rows[0].udt_name === "halfvec" ? "halfvec" : "vector";
+      const udtName = tableExists.rows[0].udt_name;
+      const vectorType = udtName === "halfvec" ? "halfvec" : udtName === "bit" ? "bit" : udtName === "sparsevec" ? "sparsevec" : "vector";
       const dimensionQuery = `
                 SELECT atttypmod as dimension
                 FROM pg_attribute
@@ -1471,7 +1629,7 @@ var PgVector = class extends vector.MastraVector {
         index_def: "",
         operator_class: "cosine"
       };
-      const metric = operator_class.includes("l2") ? "euclidean" : operator_class.includes("ip") ? "dotproduct" : "cosine";
+      const metric = operator_class.includes("hamming") ? "hamming" : operator_class.includes("jaccard") ? "jaccard" : operator_class.includes("l2") ? "euclidean" : operator_class.includes("ip") ? "dotproduct" : "cosine";
       const config = {};
       if (index_method === "hnsw") {
         const m = index_def.match(/m\s*=\s*'?(\d+)'?/)?.[1];
@@ -1606,13 +1764,14 @@ var PgVector = class extends vector.MastraVector {
       client = await this.pool.connect();
       const { tableName } = this.getTableName(indexName);
       const indexInfo = await this.getIndexInfo({ indexName });
-      const qualifiedVectorType = this.getVectorTypeName(indexInfo.vectorType);
+      const qualifiedVectorType = this.getVectorTypeName(indexInfo.vectorType, indexInfo.dimension);
+      const ops = this.getVectorOps(indexInfo.vectorType, indexInfo.metric ?? "cosine");
       let updateParts = [];
       let values = [];
       let valueIndex = 1;
       if (update.vector) {
         updateParts.push(`embedding = $${valueIndex}::${qualifiedVectorType}`);
-        values.push(`[${update.vector.join(",")}]`);
+        values.push(ops.formatVector(update.vector, indexInfo.dimension));
         valueIndex++;
       }
       if (update.metadata) {
@@ -2180,6 +2339,8 @@ var PgDB = class extends base.MastraBase {
   client;
   schemaName;
   skipDefaultIndexes;
+  /** Cache of actual table columns: tableName -> Set<columnName> */
+  tableColumnsCache = /* @__PURE__ */ new Map();
   constructor(config) {
     super({
       component: "STORAGE",
@@ -2189,6 +2350,40 @@ var PgDB = class extends base.MastraBase {
     this.schemaName = config.schemaName;
     this.skipDefaultIndexes = config.skipDefaultIndexes;
   }
+  /**
+   * Gets the set of column names that actually exist in the database table.
+   * Results are cached; the cache is invalidated when alterTable() adds new columns.
+   */
+  async getTableColumns(tableName) {
+    const cached = this.tableColumnsCache.get(tableName);
+    if (cached) return cached;
+    const schema = this.schemaName || "public";
+    const rows = await this.client.manyOrNone(
+      `SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2`,
+      [schema, tableName]
+    );
+    const columns = new Set(rows.map((r) => r.column_name));
+    if (columns.size > 0) {
+      this.tableColumnsCache.set(tableName, columns);
+    }
+    return columns;
+  }
+  /**
+   * Filters a record to only include columns that exist in the actual database table.
+   * Unknown columns are silently dropped to ensure forward compatibility when newer
+   * domain packages add fields that haven't been migrated yet.
+   */
+  async filterRecordToKnownColumns(tableName, record) {
+    const knownColumns = await this.getTableColumns(tableName);
+    if (knownColumns.size === 0) return record;
+    const filtered = {};
+    for (const [key, value] of Object.entries(record)) {
+      if (knownColumns.has(key)) {
+        filtered[key] = value;
+      }
+    }
+    return filtered;
+  }
   async hasColumn(table, column) {
     const schema = this.schemaName || "public";
     const result = await this.client.oneOrNone(
@@ -2305,9 +2500,11 @@ var PgDB = class extends base.MastraBase {
   async insert({ tableName, record }) {
     try {
       this.addTimestampZColumns(record);
+      const filteredRecord = await this.filterRecordToKnownColumns(tableName, record);
       const schemaName = getSchemaName(this.schemaName);
-      const columns = Object.keys(record).map((col) => utils.parseSqlIdentifier(col, "column name"));
-      const values = this.prepareValuesForInsert(record, tableName);
+      const columns = Object.keys(filteredRecord).map((col) => utils.parseSqlIdentifier(col, "column name"));
+      if (columns.length === 0) return;
+      const values = this.prepareValuesForInsert(filteredRecord, tableName);
       const placeholders = values.map((_, i) => `$${i + 1}`).join(", ");
       const fullTableName = getTableName({ indexName: tableName, schemaName });
       const columnList = columns.map((c) => `"${c}"`).join(", ");
@@ -2443,6 +2640,8 @@ Note: This migration may take some time for large tables.
         },
         error$1
       );
+    } finally {
+      this.tableColumnsCache.delete(tableName);
     }
   }
   async setupTimestampTriggers(tableName) {
@@ -2762,6 +2961,8 @@ Note: This migration may take some time for large tables.
         },
         error$1
       );
+    } finally {
+      this.tableColumnsCache.delete(tableName);
     }
   }
   async load({ tableName, keys }) {
@@ -2838,6 +3039,8 @@ Note: This migration may take some time for large tables.
         },
         error$1
       );
+    } finally {
+      this.tableColumnsCache.delete(tableName);
     }
   }
   async createIndex(options) {
@@ -3082,10 +3285,12 @@ Note: This migration may take some time for large tables.
     data
   }) {
     try {
+      const filteredData = await this.filterRecordToKnownColumns(tableName, data);
+      if (Object.keys(filteredData).length === 0) return;
       const setColumns = [];
       const setValues = [];
       let paramIndex = 1;
-      Object.entries(data).forEach(([key, value]) => {
+      Object.entries(filteredData).forEach(([key, value]) => {
         const parsedKey = utils.parseSqlIdentifier(key, "column name");
         setColumns.push(`"${parsedKey}" = $${paramIndex++}`);
         setValues.push(this.prepareValue(value, key, tableName));
@@ -4144,9 +4349,18 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
       compositePrimaryKey: storage.TABLE_CONFIGS[storage.TABLE_DATASET_ITEMS]?.compositePrimaryKey
     });
     await this.#db.createTable({ tableName: storage.TABLE_DATASET_VERSIONS, schema: storage.DATASET_VERSIONS_SCHEMA });
+    await this.#addColumnIfNotExists(storage.TABLE_DATASETS, "requestContextSchema", "JSONB");
+    await this.#addColumnIfNotExists(storage.TABLE_DATASET_ITEMS, "requestContext", "JSONB");
     await this.createDefaultIndexes();
     await this.createCustomIndexes();
   }
+  async #addColumnIfNotExists(table, column, sqlType) {
+    const exists = await this.#db.hasColumn(table, column);
+    if (!exists) {
+      const fullTableName = getTableName2({ indexName: table, schemaName: getSchemaName2(this.#schema) });
+      await this.#db.client.none(`ALTER TABLE ${fullTableName} ADD COLUMN "${column}" ${sqlType}`);
+    }
+  }
   getDefaultIndexDefinitions() {
     return [
       { name: "idx_dataset_items_dataset_validto", table: storage.TABLE_DATASET_ITEMS, columns: ["datasetId", "validTo"] },
@@ -4202,6 +4416,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
       metadata: row.metadata ? storage.safelyParseJSON(row.metadata) : void 0,
       inputSchema: row.inputSchema ? storage.safelyParseJSON(row.inputSchema) : void 0,
       groundTruthSchema: row.groundTruthSchema ? storage.safelyParseJSON(row.groundTruthSchema) : void 0,
+      requestContextSchema: row.requestContextSchema ? storage.safelyParseJSON(row.requestContextSchema) : void 0,
       version: row.version,
       createdAt: storage.ensureDate(row.createdAtZ || row.createdAt),
       updatedAt: storage.ensureDate(row.updatedAtZ || row.updatedAt)
@@ -4214,6 +4429,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
       datasetVersion: row.datasetVersion,
       input: storage.safelyParseJSON(row.input),
       groundTruth: row.groundTruth ? storage.safelyParseJSON(row.groundTruth) : void 0,
+      requestContext: row.requestContext ? storage.safelyParseJSON(row.requestContext) : void 0,
       metadata: row.metadata ? storage.safelyParseJSON(row.metadata) : void 0,
       createdAt: storage.ensureDate(row.createdAtZ || row.createdAt),
       updatedAt: storage.ensureDate(row.updatedAtZ || row.updatedAt)
@@ -4228,6 +4444,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
       isDeleted: Boolean(row.isDeleted),
       input: storage.safelyParseJSON(row.input),
       groundTruth: row.groundTruth ? storage.safelyParseJSON(row.groundTruth) : void 0,
+      requestContext: row.requestContext ? storage.safelyParseJSON(row.requestContext) : void 0,
       metadata: row.metadata ? storage.safelyParseJSON(row.metadata) : void 0,
       createdAt: storage.ensureDate(row.createdAtZ || row.createdAt),
       updatedAt: storage.ensureDate(row.updatedAtZ || row.updatedAt)
@@ -4256,6 +4473,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
           metadata: input.metadata ?? null,
           inputSchema: input.inputSchema ?? null,
           groundTruthSchema: input.groundTruthSchema ?? null,
+          requestContextSchema: input.requestContextSchema ?? null,
           version: 0,
           createdAt: nowIso,
           updatedAt: nowIso
@@ -4268,6 +4486,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
         metadata: input.metadata,
         inputSchema: input.inputSchema ?? void 0,
         groundTruthSchema: input.groundTruthSchema ?? void 0,
+        requestContextSchema: input.requestContextSchema ?? void 0,
         version: 0,
         createdAt: now,
         updatedAt: now
@@ -4335,6 +4554,10 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
         setClauses.push(`"groundTruthSchema" = $${paramIndex++}`);
         values.push(args.groundTruthSchema === null ? null : JSON.stringify(args.groundTruthSchema));
       }
+      if (args.requestContextSchema !== void 0) {
+        setClauses.push(`"requestContextSchema" = $${paramIndex++}`);
+        values.push(args.requestContextSchema === null ? null : JSON.stringify(args.requestContextSchema));
+      }
       values.push(args.id);
       await this.#db.client.none(
         `UPDATE ${tableName} SET ${setClauses.join(", ")} WHERE "id" = $${paramIndex}`,
@@ -4347,6 +4570,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
         metadata: args.metadata ?? existing.metadata,
         inputSchema: (args.inputSchema !== void 0 ? args.inputSchema : existing.inputSchema) ?? void 0,
         groundTruthSchema: (args.groundTruthSchema !== void 0 ? args.groundTruthSchema : existing.groundTruthSchema) ?? void 0,
+        requestContextSchema: (args.requestContextSchema !== void 0 ? args.requestContextSchema : existing.requestContextSchema) ?? void 0,
         updatedAt: new Date(now)
       };
     } catch (error$1) {
@@ -4462,13 +4686,14 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
         );
         newVersion = row.version;
         await t.none(
-          `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,false,$4,$5,$6,$7,$8,$9,$10)`,
+          `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","requestContext","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,false,$4,$5,$6,$7,$8,$9,$10,$11)`,
           [
             id,
             args.datasetId,
             newVersion,
             JSON.stringify(args.input),
             jsonbArg(args.groundTruth),
+            jsonbArg(args.requestContext),
             jsonbArg(args.metadata),
             nowIso,
             nowIso,
@@ -4487,6 +4712,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
         datasetVersion: newVersion,
         input: args.input,
         groundTruth: args.groundTruth,
+        requestContext: args.requestContext,
         metadata: args.metadata,
         createdAt: now,
         updatedAt: now
@@ -4533,6 +4759,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
       const nowIso = now.toISOString();
       const mergedInput = args.input ?? existing.input;
       const mergedGroundTruth = args.groundTruth ?? existing.groundTruth;
+      const mergedRequestContext = args.requestContext ?? existing.requestContext;
       const mergedMetadata = args.metadata ?? existing.metadata;
       let newVersion;
       await this.#db.client.tx(async (t) => {
@@ -4546,13 +4773,14 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
           [newVersion, args.id]
         );
         await t.none(
-          `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,false,$4,$5,$6,$7,$8,$9,$10)`,
+          `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","requestContext","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,false,$4,$5,$6,$7,$8,$9,$10,$11)`,
           [
             args.id,
             args.datasetId,
             newVersion,
             JSON.stringify(mergedInput),
             jsonbArg(mergedGroundTruth),
+            jsonbArg(mergedRequestContext),
             jsonbArg(mergedMetadata),
             existing.createdAt.toISOString(),
             existing.createdAt.toISOString(),
@@ -4570,6 +4798,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
         datasetVersion: newVersion,
         input: mergedInput,
         groundTruth: mergedGroundTruth,
+        requestContext: mergedRequestContext,
         metadata: mergedMetadata,
         updatedAt: now
       };
@@ -4616,13 +4845,14 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
           [newVersion, id]
         );
         await t.none(
-          `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,true,$4,$5,$6,$7,$8,$9,$10)`,
+          `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","requestContext","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,true,$4,$5,$6,$7,$8,$9,$10,$11)`,
           [
             id,
             datasetId,
             newVersion,
             JSON.stringify(existing.input),
             jsonbArg(existing.groundTruth),
+            jsonbArg(existing.requestContext),
             jsonbArg(existing.metadata),
             existing.createdAt.toISOString(),
             existing.createdAt.toISOString(),
@@ -4677,13 +4907,14 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
         newVersion = row.version;
         for (const { id, input: itemInput } of itemsWithIds) {
           await t.none(
-            `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,false,$4,$5,$6,$7,$8,$9,$10)`,
+            `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","requestContext","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,false,$4,$5,$6,$7,$8,$9,$10,$11)`,
             [
               id,
               input.datasetId,
               newVersion,
               JSON.stringify(itemInput.input),
               jsonbArg(itemInput.groundTruth),
+              jsonbArg(itemInput.requestContext),
               jsonbArg(itemInput.metadata),
               nowIso,
               nowIso,
@@ -4703,6 +4934,7 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
         datasetVersion: newVersion,
         input: itemInput.input,
         groundTruth: itemInput.groundTruth,
+        requestContext: itemInput.requestContext,
         metadata: itemInput.metadata,
         createdAt: now,
         updatedAt: now
@@ -4758,13 +4990,14 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
             [newVersion, item.id]
           );
           await t.none(
-            `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,true,$4,$5,$6,$7,$8,$9,$10)`,
+            `INSERT INTO ${itemsTable} ("id","datasetId","datasetVersion","validTo","isDeleted","input","groundTruth","requestContext","metadata","createdAt","createdAtZ","updatedAt","updatedAtZ") VALUES ($1,$2,$3,NULL,true,$4,$5,$6,$7,$8,$9,$10,$11)`,
             [
               item.id,
               input.datasetId,
               newVersion,
               JSON.stringify(item.input),
               jsonbArg(item.groundTruth),
+              jsonbArg(item.requestContext),
               jsonbArg(item.metadata),
               item.createdAt.toISOString(),
               item.createdAt.toISOString(),
@@ -7171,39 +7404,73 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
    * issues on large tables (see GitHub issue #11150). The old approach required
    * scanning and sorting ALL messages in a thread to assign row numbers.
    *
-   * The new approach uses the existing (thread_id, createdAt) index to efficiently
-   * fetch only the messages needed by using createdAt as a cursor.
+   * The current approach uses two phases for optimal performance:
+   * 1. Batch-fetch all target messages' metadata (thread_id, createdAt) in one query
+   * 2. Build cursor subqueries using "createdAt" directly (not COALESCE) so that
+   *    the existing (thread_id, createdAt DESC) index can be used for index scans
+   *    instead of sequential scans. This fixes GitHub issue #11702 where semantic
+   *    recall latency scaled linearly with message count (~30s for 7.4k messages).
    */
+  _sortMessages(messages, field, direction) {
+    return messages.sort((a, b) => {
+      const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
+      const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
+      if (aValue == null && bValue == null) return a.id.localeCompare(b.id);
+      if (aValue == null) return 1;
+      if (bValue == null) return -1;
+      if (aValue === bValue) {
+        return a.id.localeCompare(b.id);
+      }
+      if (typeof aValue === "number" && typeof bValue === "number") {
+        return direction === "ASC" ? aValue - bValue : bValue - aValue;
+      }
+      return direction === "ASC" ? String(aValue).localeCompare(String(bValue)) : String(bValue).localeCompare(String(aValue));
+    });
+  }
   async _getIncludedMessages({ include }) {
     if (!include || include.length === 0) return null;
     const tableName = getTableName3({ indexName: storage.TABLE_MESSAGES, schemaName: getSchemaName3(this.#schema) });
     const selectColumns = `id, content, role, type, "createdAt", "createdAtZ", thread_id AS "threadId", "resourceId"`;
+    const targetIds = include.map((inc) => inc.id).filter(Boolean);
+    if (targetIds.length === 0) return null;
+    const idPlaceholders = targetIds.map((_, i) => "$" + (i + 1)).join(", ");
+    const targetRows = await this.#db.client.manyOrNone(`SELECT id, thread_id, "createdAt" FROM ${tableName} WHERE id IN (${idPlaceholders})`, targetIds);
+    if (targetRows.length === 0) return null;
+    const targetMap = new Map(targetRows.map((r) => [r.id, { threadId: r.thread_id, createdAt: r.createdAt }]));
     const unionQueries = [];
     const params = [];
     let paramIdx = 1;
     for (const inc of include) {
       const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
+      const target = targetMap.get(id);
+      if (!target) continue;
+      const p1 = "$" + paramIdx;
+      const p2 = "$" + (paramIdx + 1);
+      const p3 = "$" + (paramIdx + 2);
       unionQueries.push(`(
         SELECT ${selectColumns}
         FROM ${tableName} m
-        WHERE m.thread_id = (SELECT thread_id FROM ${tableName} WHERE id = $${paramIdx})
-          AND COALESCE(m."createdAtZ", m."createdAt") <= (SELECT COALESCE("createdAtZ", "createdAt") FROM ${tableName} WHERE id = $${paramIdx})
-        ORDER BY COALESCE(m."createdAtZ", m."createdAt") DESC
-        LIMIT $${paramIdx + 1}
+        WHERE m.thread_id = ${p1}
+          AND m."createdAt" <= ${p2}
+        ORDER BY m."createdAt" DESC, m.id DESC
+        LIMIT ${p3}
       )`);
-      params.push(id, withPreviousMessages + 1);
-      paramIdx += 2;
+      params.push(target.threadId, target.createdAt, withPreviousMessages + 1);
+      paramIdx += 3;
       if (withNextMessages > 0) {
+        const p4 = "$" + paramIdx;
+        const p5 = "$" + (paramIdx + 1);
+        const p6 = "$" + (paramIdx + 2);
         unionQueries.push(`(
           SELECT ${selectColumns}
           FROM ${tableName} m
-          WHERE m.thread_id = (SELECT thread_id FROM ${tableName} WHERE id = $${paramIdx})
-            AND COALESCE(m."createdAtZ", m."createdAt") > (SELECT COALESCE("createdAtZ", "createdAt") FROM ${tableName} WHERE id = $${paramIdx})
-          ORDER BY COALESCE(m."createdAtZ", m."createdAt") ASC
-          LIMIT $${paramIdx + 1}
+          WHERE m.thread_id = ${p4}
+            AND m."createdAt" > ${p5}
+          ORDER BY m."createdAt" ASC, m.id ASC
+          LIMIT ${p6}
         )`);
-        params.push(id, withNextMessages);
-        paramIdx += 2;
+        params.push(target.threadId, target.createdAt, withNextMessages);
+        paramIdx += 3;
       }
     }
     if (unionQueries.length === 0) return null;
@@ -7211,7 +7478,7 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
     if (unionQueries.length === 1) {
       finalQuery = unionQueries[0].slice(1, -1);
     } else {
-      finalQuery = `SELECT * FROM (${unionQueries.join(" UNION ALL ")}) AS combined ORDER BY "createdAt" ASC`;
+      finalQuery = `SELECT * FROM (${unionQueries.join(" UNION ALL ")}) AS combined ORDER BY "createdAt" ASC, id ASC`;
     }
     const includedRows = await this.#db.client.manyOrNone(finalQuery, params);
     const seen = /* @__PURE__ */ new Set();
@@ -7325,6 +7592,24 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
         queryParams.push(filter.dateRange.end);
       }
       const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+      if (perPage === 0 && (!include || include.length === 0)) {
+        return { messages: [], total: 0, page, perPage: perPageForResponse, hasMore: false };
+      }
+      if (perPage === 0 && include && include.length > 0) {
+        const includeMessages = await this._getIncludedMessages({ include });
+        if (!includeMessages || includeMessages.length === 0) {
+          return { messages: [], total: 0, page, perPage: perPageForResponse, hasMore: false };
+        }
+        const messagesWithParsedContent2 = includeMessages.map((row) => this.parseRow(row));
+        const list2 = new agent.MessageList().add(messagesWithParsedContent2, "memory");
+        return {
+          messages: this._sortMessages(list2.get.all.db(), field, direction),
+          total: 0,
+          page,
+          perPage: perPageForResponse,
+          hasMore: false
+        };
+      }
       const countQuery = `SELECT COUNT(*) FROM ${tableName} ${whereClause}`;
       const countResult = await this.#db.client.one(countQuery, queryParams);
       const total = parseInt(countResult.count, 10);
@@ -7355,21 +7640,7 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
       }
       const messagesWithParsedContent = messages.map((row) => this.parseRow(row));
       const list = new agent.MessageList().add(messagesWithParsedContent, "memory");
-      let finalMessages = list.get.all.db();
-      finalMessages = finalMessages.sort((a, b) => {
-        const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
-        const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
-        if (aValue == null && bValue == null) return a.id.localeCompare(b.id);
-        if (aValue == null) return 1;
-        if (bValue == null) return -1;
-        if (aValue === bValue) {
-          return a.id.localeCompare(b.id);
-        }
-        if (typeof aValue === "number" && typeof bValue === "number") {
-          return direction === "ASC" ? aValue - bValue : bValue - aValue;
-        }
-        return direction === "ASC" ? String(aValue).localeCompare(String(bValue)) : String(bValue).localeCompare(String(aValue));
-      });
+      const finalMessages = this._sortMessages(list.get.all.db(), field, direction);
       const threadIdSet = new Set(threadIds);
       const returnedThreadMessageIds = new Set(
         finalMessages.filter((m) => m.threadId && threadIdSet.has(m.threadId)).map((m) => m.id)
@@ -7458,6 +7729,30 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
         queryParams.push(filter.dateRange.end);
       }
       const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+      if (perPage === 0 && (!include || include.length === 0)) {
+        return { messages: [], total: 0, page, perPage: perPageForResponse, hasMore: false };
+      }
+      if (perPage === 0 && include && include.length > 0) {
+        const includeMessages = await this._getIncludedMessages({ include });
+        if (!includeMessages || includeMessages.length === 0) {
+          return {
+            messages: [],
+            total: 0,
+            page,
+            perPage: perPageForResponse,
+            hasMore: false
+          };
+        }
+        const messagesWithParsedContent2 = includeMessages.map((row) => this.parseRow(row));
+        const list2 = new agent.MessageList().add(messagesWithParsedContent2, "memory");
+        return {
+          messages: this._sortMessages(list2.get.all.db(), field, direction),
+          total: 0,
+          page,
+          perPage: perPageForResponse,
+          hasMore: false
+        };
+      }
       const countQuery = `SELECT COUNT(*) FROM ${tableName} ${whereClause}`;
       const countResult = await this.#db.client.one(countQuery, queryParams);
       const total = parseInt(countResult.count, 10);
@@ -7488,21 +7783,7 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
       }
       const messagesWithParsedContent = messages.map((row) => this.parseRow(row));
       const list = new agent.MessageList().add(messagesWithParsedContent, "memory");
-      let finalMessages = list.get.all.db();
-      finalMessages = finalMessages.sort((a, b) => {
-        const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
-        const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
-        if (aValue == null && bValue == null) return a.id.localeCompare(b.id);
-        if (aValue == null) return 1;
-        if (bValue == null) return -1;
-        if (aValue === bValue) {
-          return a.id.localeCompare(b.id);
-        }
-        if (typeof aValue === "number" && typeof bValue === "number") {
-          return direction === "ASC" ? aValue - bValue : bValue - aValue;
-        }
-        return direction === "ASC" ? String(aValue).localeCompare(String(bValue)) : String(bValue).localeCompare(String(aValue));
-      });
+      const finalMessages = this._sortMessages(list.get.all.db(), field, direction);
       const hasMore = perPageInput !== false && offset + perPage < total;
       return {
         messages: finalMessages,
@@ -8956,6 +9237,11 @@ var ObservabilityPG = class _ObservabilityPG extends storage.ObservabilityStorag
   }
   async init() {
     await this.#db.createTable({ tableName: storage.TABLE_SPANS, schema: storage.TABLE_SCHEMAS[storage.TABLE_SPANS] });
+    await this.#db.alterTable({
+      tableName: storage.TABLE_SPANS,
+      schema: storage.TABLE_SCHEMAS[storage.TABLE_SPANS],
+      ifNotExists: ["requestContext"]
+    });
     await this.createDefaultIndexes();
     await this.createCustomIndexes();
   }
@@ -9310,7 +9596,8 @@ var ObservabilityPG = class _ObservabilityPG extends storage.ObservabilityStorag
   }
   async listTraces(args) {
     const { filters, pagination, orderBy } = storage.listTracesArgsSchema.parse(args);
-    const { page, perPage } = pagination;
+    const page = pagination?.page ?? 0;
+    const perPage = pagination?.perPage ?? 10;
     const tableName = getTableName2({
       indexName: storage.TABLE_SPANS,
       schemaName: getSchemaName2(this.#schema)
@@ -9432,10 +9719,11 @@ var ObservabilityPG = class _ObservabilityPG extends storage.ObservabilityStorag
         }
       }
       const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
-      const sortField = `${orderBy.field}Z`;
-      const sortDirection = orderBy.direction;
+      const orderField = orderBy?.field ?? "startedAt";
+      const sortField = `${orderField}Z`;
+      const sortDirection = orderBy?.direction ?? "DESC";
       let orderClause;
-      if (orderBy.field === "endedAt") {
+      if (orderField === "endedAt") {
         const nullsOrder = sortDirection === "DESC" ? "NULLS FIRST" : "NULLS LAST";
         orderClause = `ORDER BY r."${sortField}" ${sortDirection} ${nullsOrder}`;
       } else {