@mastra/vectorize 0.0.0-storage-20250225005900

package/README.md

@@ -0,0 +1,39 @@
+# @mastra/vectorize
+
+Vector store implementation for Vectorize, a managed vector database service optimized for AI applications.
+
+## Installation
+
+```bash
+npm install @mastra/vectorize
+```
+
+## Usage
+
+```typescript
+import { VectorizeStore } from '@mastra/vectorize';
+
+const vectorStore = new VectorizeStore({
+  // configuration options
+});
+```
+
+## Configuration
+
+The Vectorize vector store requires the following configuration:
+
+- `VECTORIZE_API_KEY`: Your Vectorize API key
+- `VECTORIZE_INDEX_NAME`: Name of the index to use
+- `VECTORIZE_PROJECT_ID`: Your Vectorize project ID
+
+## Features
+
+- Purpose-built for AI and ML workloads
+- High-performance vector similarity search
+- Automatic index optimization
+- Scalable architecture
+- Real-time updates and queries
+
+## Related Links
+
+- [Vectorize Documentation](https://www.vectorize.com/docs)

package/dist/_tsup-dts-rollup.d.ts

@@ -0,0 +1,39 @@
+import { BaseFilterTranslator } from '@mastra/core/filter';
+import Cloudflare from 'cloudflare';
+import { Filter } from '@mastra/core/filter';
+import { MastraVector } from '@mastra/core/vector';
+import { OperatorSupport } from '@mastra/core/filter';
+import { QueryResult } from '@mastra/core/vector';
+
+declare class CloudflareVector extends MastraVector {
+    client: Cloudflare;
+    accountId: string;
+    constructor({ accountId, apiToken }: {
+        accountId: string;
+        apiToken: string;
+    });
+    upsert(indexName: string, vectors: number[][], metadata?: Record<string, any>[], ids?: string[]): Promise<string[]>;
+    transformFilter(filter?: Filter): Filter | undefined;
+    createIndex(indexName: string, dimension: number, metric?: 'cosine' | 'euclidean' | 'dotproduct'): Promise<void>;
+    query(indexName: string, queryVector: number[], topK?: number, filter?: Filter, includeVector?: boolean): Promise<QueryResult[]>;
+    listIndexes(): Promise<string[]>;
+    describeIndex(indexName: string): Promise<{
+        dimension: number;
+        count: number;
+        metric: "cosine" | "euclidean" | "dotproduct";
+    }>;
+    deleteIndex(indexName: string): Promise<void>;
+    createMetadataIndex(indexName: string, propertyName: string, indexType: 'string' | 'number' | 'boolean'): Promise<void>;
+    deleteMetadataIndex(indexName: string, propertyName: string): Promise<void>;
+    listMetadataIndexes(indexName: string): Promise<Cloudflare.Vectorize.Indexes.MetadataIndex.MetadataIndexListResponse.MetadataIndex[]>;
+}
+export { CloudflareVector }
+export { CloudflareVector as CloudflareVector_alias_1 }
+
+export declare class VectorizeFilterTranslator extends BaseFilterTranslator {
+    protected getSupportedOperators(): OperatorSupport;
+    translate(filter?: Filter): Filter | undefined;
+    private translateNode;
+}
+
+export { }

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { CloudflareVector } from './_tsup-dts-rollup.js';

package/dist/index.js

@@ -0,0 +1,165 @@
+import { BaseFilterTranslator } from '@mastra/core/filter';
+import { MastraVector } from '@mastra/core/vector';
+import Cloudflare from 'cloudflare';
+
+// src/vector/index.ts
+var VectorizeFilterTranslator = class extends BaseFilterTranslator {
+  getSupportedOperators() {
+    return {
+      ...BaseFilterTranslator.DEFAULT_OPERATORS,
+      logical: [],
+      array: ["$in", "$nin"],
+      element: [],
+      regex: [],
+      custom: []
+    };
+  }
+  translate(filter) {
+    if (this.isEmpty(filter)) return filter;
+    this.validateFilter(filter);
+    return this.translateNode(filter);
+  }
+  translateNode(node, currentPath = "") {
+    if (this.isRegex(node)) {
+      throw new Error("Regex is not supported in Vectorize");
+    }
+    if (this.isPrimitive(node)) return { $eq: this.normalizeComparisonValue(node) };
+    if (Array.isArray(node)) return { $in: this.normalizeArrayValues(node) };
+    const entries = Object.entries(node);
+    const firstEntry = entries[0];
+    if (entries.length === 1 && firstEntry && this.isOperator(firstEntry[0])) {
+      const [operator, value] = firstEntry;
+      return { [operator]: this.normalizeComparisonValue(value) };
+    }
+    const result = {};
+    for (const [key, value] of entries) {
+      const newPath = currentPath ? `${currentPath}.${key}` : key;
+      if (this.isOperator(key)) {
+        result[key] = this.normalizeComparisonValue(value);
+        continue;
+      }
+      if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+        if (Object.keys(value).length === 0) {
+          result[newPath] = this.translateNode(value);
+          continue;
+        }
+        const hasOperators = Object.keys(value).some((k) => this.isOperator(k));
+        if (hasOperators) {
+          result[newPath] = this.translateNode(value);
+        } else {
+          Object.assign(result, this.translateNode(value, newPath));
+        }
+      } else {
+        result[newPath] = this.translateNode(value);
+      }
+    }
+    return result;
+  }
+};
+
+// src/vector/index.ts
+var CloudflareVector = class extends MastraVector {
+  client;
+  accountId;
+  constructor({ accountId, apiToken }) {
+    super();
+    this.accountId = accountId;
+    this.client = new Cloudflare({
+      apiKey: apiToken
+    });
+  }
+  async upsert(indexName, vectors, metadata, ids) {
+    const generatedIds = ids || vectors.map(() => crypto.randomUUID());
+    const ndjson = vectors.map((vector, index) => ({
+      id: generatedIds[index],
+      values: vector,
+      metadata: metadata?.[index]
+    })).map((record) => JSON.stringify(record)).join("\n");
+    await this.client.vectorize.indexes.upsert(indexName, {
+      account_id: this.accountId,
+      body: ndjson
+    });
+    return generatedIds;
+  }
+  transformFilter(filter) {
+    const translator = new VectorizeFilterTranslator();
+    const translatedFilter = translator.translate(filter);
+    return translatedFilter;
+  }
+  async createIndex(indexName, dimension, metric = "cosine") {
+    await this.client.vectorize.indexes.create({
+      account_id: this.accountId,
+      config: {
+        dimensions: dimension,
+        metric: metric === "dotproduct" ? "dot-product" : metric
+      },
+      name: indexName
+    });
+  }
+  async query(indexName, queryVector, topK = 10, filter, includeVector = false) {
+    const translatedFilter = this.transformFilter(filter);
+    const response = await this.client.vectorize.indexes.query(indexName, {
+      account_id: this.accountId,
+      vector: queryVector,
+      returnValues: includeVector,
+      returnMetadata: "all",
+      topK,
+      filter: translatedFilter
+    });
+    return response?.matches?.map((match) => {
+      return {
+        id: match.id,
+        metadata: match.metadata,
+        score: match.score,
+        vector: match.values
+      };
+    }) || [];
+  }
+  async listIndexes() {
+    const res = await this.client.vectorize.indexes.list({
+      account_id: this.accountId
+    });
+    return res?.result?.map((index) => index.name) || [];
+  }
+  async describeIndex(indexName) {
+    const index = await this.client.vectorize.indexes.get(indexName, {
+      account_id: this.accountId
+    });
+    const described = await this.client.vectorize.indexes.info(indexName, {
+      account_id: this.accountId
+    });
+    return {
+      dimension: described?.dimensions,
+      // Since vector_count is not available in the response,
+      // we might need a separate API call to get the count if needed
+      count: described?.vectorCount || 0,
+      metric: index?.config?.metric
+    };
+  }
+  async deleteIndex(indexName) {
+    await this.client.vectorize.indexes.delete(indexName, {
+      account_id: this.accountId
+    });
+  }
+  async createMetadataIndex(indexName, propertyName, indexType) {
+    await this.client.vectorize.indexes.metadataIndex.create(indexName, {
+      account_id: this.accountId,
+      propertyName,
+      indexType
+    });
+  }
+  async deleteMetadataIndex(indexName, propertyName) {
+    await this.client.vectorize.indexes.metadataIndex.delete(indexName, {
+      account_id: this.accountId,
+      propertyName
+    });
+  }
+  async listMetadataIndexes(indexName) {
+    const res = await this.client.vectorize.indexes.metadataIndex.list(indexName, {
+      account_id: this.accountId
+    });
+    return res?.metadataIndexes ?? [];
+  }
+};
+
+export { CloudflareVector };

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@mastra/vectorize",
+  "version": "0.0.0-storage-20250225005900",
+  "description": "Cloudflare Vectorize store provider for Mastra",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "dependencies": {
+    "cloudflare": "^4.0.0",
+    "@mastra/core": "^0.0.0-storage-20250225005900"
+  },
+  "devDependencies": {
+    "@microsoft/api-extractor": "^7.49.2",
+    "@types/node": "^22.13.1",
+    "tsup": "^8.0.1",
+    "typescript": "^5.7.3",
+    "vitest": "^3.0.5"
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --experimental-dts --clean --treeshake",
+    "build:watch": "pnpm build --watch",
+    "test": "vitest run"
+  }
+}

package/src/index.ts

@@ -0,0 +1 @@
+export * from './vector';

package/src/vector/filter.test.ts

@@ -0,0 +1,198 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+
+import { VectorizeFilterTranslator } from './filter';
+
+describe('VectorizeFilterTranslator', () => {
+  let translator: VectorizeFilterTranslator;
+
+  beforeEach(() => {
+    translator = new VectorizeFilterTranslator();
+  });
+
+  describe('translate', () => {
+    it('handles empty filters', () => {
+      expect(translator.translate({})).toEqual({});
+      expect(translator.translate(null as any)).toEqual(null);
+      expect(translator.translate(undefined as any)).toEqual(undefined);
+    });
+
+    // Basic cases
+    it('converts implicit equality to explicit $eq', () => {
+      const filter = { field: 'value' };
+      expect(translator.translate(filter)).toEqual({
+        field: { $eq: 'value' },
+      });
+    });
+
+    it('handles comparison operators', () => {
+      const filter = {
+        age: { $gt: 25 },
+        price: { $lte: 100 },
+        status: { $ne: 'inactive' },
+        quantity: { $gte: 10 },
+        rating: { $lt: 5 },
+      };
+      expect(translator.translate(filter)).toEqual(filter);
+    });
+
+    it('handles $in operator', () => {
+      const filter = {
+        tags: { $in: ['important', 'urgent'] },
+      };
+      expect(translator.translate(filter)).toEqual(filter);
+    });
+
+    it('handles $nin operator', () => {
+      const filter = {
+        status: { $nin: ['deleted', 'archived'] },
+      };
+      expect(translator.translate(filter)).toEqual(filter);
+    });
+
+    it('handles null values', () => {
+      const filter = {
+        field: null,
+        other: { $eq: null },
+      };
+      expect(translator.translate(filter)).toEqual({
+        field: { $eq: null },
+        other: { $eq: null },
+      });
+    });
+
+    it('handles empty objects', () => {
+      const filter = { field: {} };
+      expect(translator.translate(filter)).toEqual({ field: {} });
+    });
+
+    it('flattens nested objects to dot notation', () => {
+      const filter = {
+        user: {
+          profile: {
+            age: { $gt: 25 },
+          },
+        },
+      };
+      expect(translator.translate(filter)).toEqual({
+        'user.profile.age': { $gt: 25 },
+      });
+    });
+
+    it('normalizes date values', () => {
+      const date = new Date('2024-01-01');
+      const filter = { timestamp: { $gt: date } };
+      expect(translator.translate(filter)).toEqual({
+        timestamp: { $gt: date.toISOString() },
+      });
+    });
+
+    it('handles multiple operators on same field', () => {
+      const filter = {
+        price: { $gt: 100, $lt: 200 },
+        quantity: { $gte: 10, $lte: 20 },
+      };
+      expect(translator.translate(filter)).toEqual(filter);
+    });
+
+    it('handles arrays of mixed types', () => {
+      const filter = {
+        field: { $in: [123, 'string', true] },
+      };
+      expect(translator.translate(filter)).toEqual(filter);
+    });
+
+    it('handles empty arrays in $in and $nin', () => {
+      const filter = {
+        field1: { $in: [] },
+        field2: { $nin: [] },
+      };
+      expect(translator.translate(filter)).toEqual(filter);
+    });
+
+    it('handles deeply nested null values', () => {
+      const filter = {
+        user: {
+          profile: {
+            lastLogin: null,
+            settings: { theme: { $eq: null } },
+          },
+        },
+      };
+      expect(translator.translate(filter)).toEqual({
+        'user.profile.lastLogin': { $eq: null },
+        'user.profile.settings.theme': { $eq: null },
+      });
+    });
+
+    it('preserves order of multiple operators', () => {
+      const filter = {
+        field: {
+          $gt: 0,
+          $lt: 10,
+        },
+      };
+      expect(translator.translate(filter)).toEqual(filter);
+    });
+
+    it('preserves order of range operators', () => {
+      // Valid range operator combinations
+      const filters = [
+        { field: { $gt: 0, $lt: 10 } },
+        { field: { $gte: 0, $lte: 10 } },
+        { field: { $gt: 0, $lte: 10 } },
+        { field: { $gte: 0, $lt: 10 } },
+      ];
+
+      filters.forEach(filter => {
+        expect(translator.translate(filter)).toEqual(filter);
+      });
+    });
+  });
+
+  describe('validate operators', () => {
+    it('ensure all operator filters are supported', () => {
+      const supportedFilters = [
+        { field: { $eq: 'value' } },
+        { field: { $ne: 'value' } },
+        { field: { $gt: 'value' } },
+        { field: { $gte: 'value' } },
+        { field: { $lt: 'value' } },
+        { field: { $lte: 'value' } },
+        { field: { $in: ['value'] } },
+        { field: { $nin: ['value'] } },
+      ];
+      supportedFilters.forEach(filter => {
+        expect(() => translator.translate(filter)).not.toThrow();
+      });
+    });
+    it('throws error for unsupported operators', () => {
+      const unsupportedFilters = [
+        { field: { $regex: 'pattern' } },
+        { field: { $exists: true } },
+        { field: { $elemMatch: { $gt: 5 } } },
+        { field: { $nor: [{ $eq: 'value' }] } },
+        { field: { $not: [{ $eq: 'value' }] } },
+        { field: { $regex: 'pattern', $options: 'i' } },
+        { field: { $and: [{ $eq: 'value' }] } },
+        { field: { $or: [{ $eq: 'value' }] } },
+        { field: { $all: [{ $eq: 'value' }] } },
+        { field: { $contains: 'value' } },
+      ];
+
+      unsupportedFilters.forEach(filter => {
+        expect(() => translator.translate(filter)).toThrow(/Unsupported operator/);
+      });
+    });
+    it('throws error for regex operators', () => {
+      const filter = { field: /pattern/i };
+      expect(() => translator.translate(filter)).toThrow();
+    });
+    it('throws error for non-logical operators at top level', () => {
+      const invalidFilters = [{ $gt: 100 }, { $in: ['value1', 'value2'] }, { $eq: true }];
+
+      invalidFilters.forEach(filter => {
+        expect(() => translator.translate(filter)).toThrow(/Invalid top-level operator/);
+      });
+    });
+  });
+});

package/src/vector/filter.ts

@@ -0,0 +1,68 @@
+import { BaseFilterTranslator, type Filter, type FieldCondition, type OperatorSupport } from '@mastra/core/filter';
+
+export class VectorizeFilterTranslator extends BaseFilterTranslator {
+  protected override getSupportedOperators(): OperatorSupport {
+    return {
+      ...BaseFilterTranslator.DEFAULT_OPERATORS,
+      logical: [],
+      array: ['$in', '$nin'],
+      element: [],
+      regex: [],
+      custom: [],
+    };
+  }
+
+  translate(filter?: Filter): Filter | undefined {
+    if (this.isEmpty(filter)) return filter;
+    this.validateFilter(filter as Filter);
+    return this.translateNode(filter);
+  }
+
+  private translateNode(node: Filter | FieldCondition, currentPath: string = ''): any {
+    if (this.isRegex(node)) {
+      throw new Error('Regex is not supported in Vectorize');
+    }
+    if (this.isPrimitive(node)) return { $eq: this.normalizeComparisonValue(node) };
+    if (Array.isArray(node)) return { $in: this.normalizeArrayValues(node) };
+
+    const entries = Object.entries(node as Record<string, any>);
+    const firstEntry = entries[0];
+
+    // Handle single operator case
+    if (entries.length === 1 && firstEntry && this.isOperator(firstEntry[0])) {
+      const [operator, value] = firstEntry;
+      return { [operator]: this.normalizeComparisonValue(value) };
+    }
+
+    // Process each entry
+    const result: Record<string, any> = {};
+    for (const [key, value] of entries) {
+      const newPath = currentPath ? `${currentPath}.${key}` : key;
+
+      if (this.isOperator(key)) {
+        result[key] = this.normalizeComparisonValue(value);
+        continue;
+      }
+
+      if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+        if (Object.keys(value).length === 0) {
+          result[newPath] = this.translateNode(value);
+          continue;
+        }
+
+        // Check if the nested object contains operators
+        const hasOperators = Object.keys(value).some(k => this.isOperator(k));
+        if (hasOperators) {
+          result[newPath] = this.translateNode(value);
+        } else {
+          // For objects without operators, flatten them
+          Object.assign(result, this.translateNode(value, newPath));
+        }
+      } else {
+        result[newPath] = this.translateNode(value);
+      }
+    }
+
+    return result;
+  }
+}