@mastra/longmemeval 0.0.0-add-libsql-changeset-20250910154739

package/src/storage/benchmark-vector.ts

@@ -0,0 +1,234 @@
+import { MastraVector } from '@mastra/core/vector';
+import type {
+  QueryVectorParams,
+  QueryResult,
+  UpsertVectorParams,
+  CreateIndexParams,
+  IndexStats,
+  UpdateVectorParams,
+  DeleteVectorParams,
+  DescribeIndexParams,
+  DeleteIndexParams,
+} from '@mastra/core/vector';
+import { VectorDB, type Document } from 'imvectordb';
+import { writeFile, readFile } from 'fs/promises';
+import { existsSync } from 'fs';
+
+type DBMode = 'read' | 'read-write';
+
+export class BenchmarkVectorStore extends MastraVector {
+  private indexes: Map<string, VectorDB> = new Map();
+  private indexConfigs: Map<string, { dimension: number; metric?: 'cosine' | 'euclidean' | 'dotproduct' }> = new Map();
+  private documentsStore: Map<string, Map<string, Document>> = new Map();
+  private mode: DBMode;
+
+  constructor(mode: DBMode = 'read-write') {
+    super();
+    this.mode = mode;
+  }
+
+  async createIndex({ indexName, dimension, metric = 'cosine' }: CreateIndexParams): Promise<void> {
+    if (this.mode === 'read') return;
+    if (this.indexes.has(indexName)) {
+      await this.validateExistingIndex(indexName, dimension, metric);
+      return;
+    }
+
+    this.indexes.set(indexName, new VectorDB());
+    this.indexConfigs.set(indexName, { dimension, metric });
+    this.documentsStore.set(indexName, new Map());
+  }
+
+  async query({ indexName, queryVector, topK = 10 }: QueryVectorParams): Promise<QueryResult[]> {
+    const db = this.indexes.get(indexName);
+    if (!db) {
+      throw new Error(`Index ${indexName} not found`);
+    }
+
+    const results = await db.query(queryVector, topK);
+    return results.map((r: any) => ({
+      id: r.document.id,
+      score: r.similarity,
+      metadata: r.document.metadata,
+    }));
+  }
+
+  async upsert({ indexName, vectors, metadata, ids }: UpsertVectorParams): Promise<string[]> {
+    if (this.mode === 'read') return [];
+    const db = this.indexes.get(indexName);
+    const docsStore = this.documentsStore.get(indexName);
+
+    if (!db || !docsStore) {
+      throw new Error(`Index ${indexName} not found`);
+    }
+
+    const vectorIds = ids || vectors.map(() => crypto.randomUUID());
+
+    for (let i = 0; i < vectors.length; i++) {
+      const doc: Document = {
+        id: vectorIds[i],
+        embedding: vectors[i],
+        metadata: metadata?.[i] || {},
+      };
+
+      // Store in VectorDB
+      db.add(doc);
+      // Also store in our documents map for persistence
+      docsStore.set(doc.id, doc);
+    }
+
+    return vectorIds;
+  }
+
+  async listIndexes(): Promise<string[]> {
+    return Array.from(this.indexes.keys());
+  }
+
+  async describeIndex({ indexName }: DescribeIndexParams): Promise<IndexStats> {
+    const db = this.indexes.get(indexName);
+    const config = this.indexConfigs.get(indexName);
+
+    if (!db || !config) {
+      throw new Error(`Index ${indexName} not found`);
+    }
+
+    // Get count from the database
+    const count = db.size();
+
+    return {
+      dimension: config.dimension,
+      count: count,
+      metric: config.metric,
+    };
+  }
+
+  async deleteIndex({ indexName }: DeleteIndexParams): Promise<void> {
+    if (this.mode === 'read') return;
+    const db = this.indexes.get(indexName);
+    if (db) {
+      await db.terminate();
+    }
+    this.indexes.delete(indexName);
+    this.indexConfigs.delete(indexName);
+    this.documentsStore.delete(indexName);
+  }
+
+  async updateVector({ indexName, id, update }: UpdateVectorParams): Promise<void> {
+    if (this.mode === 'read') return;
+    const db = this.indexes.get(indexName);
+    const docsStore = this.documentsStore.get(indexName);
+
+    if (!db || !docsStore) {
+      throw new Error(`Index ${indexName} not found`);
+    }
+
+    // Get the existing document
+    const doc = db.get(id);
+    if (!doc) {
+      throw new Error(`Vector with id ${id} not found in index ${indexName}`);
+    }
+
+    // Remove old version
+    db.del(doc);
+    docsStore.delete(id);
+
+    // Create updated document
+    const updatedDoc: Document = {
+      id,
+      embedding: update.vector || doc.embedding,
+      metadata: update.metadata || doc.metadata,
+    };
+
+    // Add updated version
+    db.add(updatedDoc);
+    docsStore.set(id, updatedDoc);
+  }
+
+  async deleteVector({ indexName, id }: DeleteVectorParams): Promise<void> {
+    if (this.mode === 'read') return;
+    const db = this.indexes.get(indexName);
+    const docsStore = this.documentsStore.get(indexName);
+
+    if (!db || !docsStore) {
+      throw new Error(`Index ${indexName} not found`);
+    }
+
+    const doc = db.get(id);
+    if (doc) {
+      db.del(doc);
+      docsStore.delete(id);
+    }
+  }
+
+  /**
+   * Persist the current vector store state to a JSON file
+   */
+  async persist(filePath: string): Promise<void> {
+    if (this.mode === 'read') return;
+    const data: Record<string, any> = {};
+
+    for (const [indexName, docsStore] of this.documentsStore) {
+      const config = this.indexConfigs.get(indexName);
+      const documents = Array.from(docsStore.values());
+
+      data[indexName] = {
+        config,
+        documents,
+      };
+    }
+
+    await writeFile(filePath, JSON.stringify(data, null, 2));
+  }
+
+  /**
+   * Hydrate vector store state from a JSON file
+   */
+  async hydrate(filePath: string): Promise<void> {
+    if (!existsSync(filePath)) {
+      throw new Error(`Vector store file not found: ${filePath}`);
+    }
+
+    const content = await readFile(filePath, 'utf-8');
+    const data = JSON.parse(content);
+
+    // Clear existing data
+    for (const db of this.indexes.values()) {
+      await db.terminate();
+    }
+    this.indexes.clear();
+    this.indexConfigs.clear();
+    this.documentsStore.clear();
+
+    // Restore data
+    for (const [indexName, indexData] of Object.entries(data)) {
+      const { config, documents } = indexData as any;
+
+      // Create new index
+      const db = new VectorDB();
+      this.indexes.set(indexName, db);
+      this.indexConfigs.set(indexName, config);
+
+      const docsStore = new Map<string, Document>();
+      this.documentsStore.set(indexName, docsStore);
+
+      // Restore documents
+      for (const doc of documents) {
+        db.add(doc);
+        docsStore.set(doc.id, doc);
+      }
+    }
+  }
+
+  /**
+   * Clear all data and start fresh
+   */
+  async clear(): Promise<void> {
+    if (this.mode === 'read') return;
+    for (const db of this.indexes.values()) {
+      await db.terminate();
+    }
+    this.indexes.clear();
+    this.indexConfigs.clear();
+    this.documentsStore.clear();
+  }
+}

package/src/storage/index.ts

@@ -0,0 +1,2 @@
+export { BenchmarkStore } from './benchmark-store';
+export { BenchmarkVectorStore } from './benchmark-vector';

package/src/test-utils/mock-embeddings.ts

@@ -0,0 +1,54 @@
+import { readFileSync } from 'fs';
+import { join } from 'path';
+
+// Load fixture embeddings - use relative path from project root
+const embeddingsPath = join(process.cwd(), 'src', '__fixtures__', 'embeddings.json');
+let fixtureEmbeddings: Record<string, number[]> = {};
+
+try {
+  fixtureEmbeddings = JSON.parse(readFileSync(embeddingsPath, 'utf-8'));
+} catch (error) {
+  console.warn('Warning: Could not load fixture embeddings, using random embeddings instead');
+}
+
+/**
+ * Mock embedding function that returns fixture embeddings or generates random ones
+ */
+export function createMockEmbedding() {
+  return {
+    doEmbed: async ({ values }: { values: string[] }) => {
+      const embeddings = values.map(text => {
+        // Return fixture embedding if available
+        if (fixtureEmbeddings[text]) {
+          return fixtureEmbeddings[text];
+        }
+
+        // Otherwise generate a deterministic "random" embedding based on the text
+        // This ensures the same text always gets the same embedding
+        const seed = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0);
+        const embedding = new Array(1536).fill(0).map((_, i) => {
+          // Use a simple deterministic function to generate values
+          return Math.sin(seed + i) * Math.cos(seed * i * 0.01);
+        });
+
+        return embedding;
+      });
+
+      return { embeddings };
+    },
+  };
+}
+
+/**
+ * Get a specific fixture embedding by text
+ */
+export function getFixtureEmbedding(text: string): number[] | undefined {
+  return fixtureEmbeddings[text];
+}
+
+/**
+ * Get all available fixture texts
+ */
+export function getFixtureTexts(): string[] {
+  return Object.keys(fixtureEmbeddings);
+}

package/src/test-utils/mock-model.ts

@@ -0,0 +1,49 @@
+import type { LanguageModelV1, LanguageModelV1CallWarning, LanguageModelV1StreamPart } from '@ai-sdk/provider';
+import { createIdGenerator } from '@ai-sdk/provider-utils';
+
+const generateId = createIdGenerator();
+
+export class MockLanguageModelV1 implements LanguageModelV1 {
+  readonly specificationVersion = 'v1';
+  readonly defaultObjectGenerationMode = 'json' as const;
+  readonly provider = 'mock-provider';
+  readonly modelId = 'mock-model';
+
+  private readonly generateFunc: (options: any) => Promise<{
+    rawCall: { rawPrompt: any; rawSettings: any };
+    finishReason: 'stop' | 'length' | 'content-filter' | 'tool-calls' | 'error' | 'other' | 'unknown';
+    usage: { promptTokens: number; completionTokens: number };
+    text?: string;
+    toolCalls?: any[];
+    warnings?: LanguageModelV1CallWarning[];
+  }>;
+
+  constructor({
+    doGenerate,
+  }: {
+    doGenerate: (options: any) => Promise<{
+      rawCall: { rawPrompt: any; rawSettings: any };
+      finishReason: 'stop' | 'length' | 'content-filter' | 'tool-calls' | 'error' | 'other' | 'unknown';
+      usage: { promptTokens: number; completionTokens: number };
+      text?: string;
+      toolCalls?: any[];
+      warnings?: LanguageModelV1CallWarning[];
+    }>;
+  }) {
+    this.generateFunc = doGenerate;
+  }
+
+  async doGenerate(
+    options: Parameters<LanguageModelV1['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<LanguageModelV1['doGenerate']>>> {
+    return this.generateFunc(options);
+  }
+
+  async doStream(options: Parameters<LanguageModelV1['doStream']>[0]): Promise<{
+    stream: ReadableStream<LanguageModelV1StreamPart>;
+    rawCall: { rawPrompt: any; rawSettings: any };
+    warnings?: LanguageModelV1CallWarning[];
+  }> {
+    throw new Error('Streaming not supported in mock model');
+  }
+}

package/tests/data-loader.test.ts

@@ -0,0 +1,96 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import { DatasetLoader } from '../src/data/loader';
+import type { LongMemEvalQuestion } from '../src/data/types';
+import { join } from 'path';
+
+describe('DatasetLoader', () => {
+  let loader: DatasetLoader;
+  let testLoader: DatasetLoader;
+
+  beforeAll(() => {
+    loader = new DatasetLoader();
+    // Create a test loader that points to our fixtures directory
+    testLoader = new DatasetLoader(join(process.cwd(), 'src', '__fixtures__'));
+  });
+
+  describe('validateDataset', () => {
+    it('should validate correct dataset structure', () => {
+      const validData: LongMemEvalQuestion[] = [
+        {
+          question_id: 'test_001',
+          question_type: 'single-session-user',
+          question: 'What is my favorite color?',
+          answer: 'Blue',
+          question_date: '2024-01-01',
+          haystack_session_ids: ['session_1'],
+          haystack_dates: ['2024-01-01'],
+          haystack_sessions: [
+            [
+              { role: 'user', content: 'My favorite color is blue.' },
+              { role: 'assistant', content: 'I understand your favorite color is blue.' },
+            ],
+          ],
+          answer_session_ids: ['session_1'],
+        },
+      ];
+
+      // Should not throw
+      expect(() => {
+        // Access private method through any type
+        (loader as any).validateDataset(validData);
+      }).not.toThrow();
+    });
+
+    it('should throw on invalid dataset structure', () => {
+      const invalidData = [
+        {
+          question_id: 'test_001',
+          // Missing required fields
+        },
+      ];
+
+      expect(() => {
+        (loader as any).validateDataset(invalidData);
+      }).toThrow('Missing required field');
+    });
+  });
+
+  describe('loadDataset', () => {
+    it('should load and validate test dataset fixture', async () => {
+      // Use the test-dataset from fixtures
+      const questions = await testLoader.loadDataset('test-dataset' as any);
+
+      expect(questions).toHaveLength(3);
+      expect(questions[0].question_id).toBe('test-001');
+      expect(questions[0].question_type).toBe('single-session-user');
+      expect(questions[1].question_type).toBe('multi-session');
+
+      // Verify the dataset structure
+      expect(questions[0].haystack_sessions).toHaveLength(1);
+      expect(questions[1].haystack_sessions).toHaveLength(2);
+
+      // Check answers
+      expect(questions[0].answer).toBe('Blue');
+      expect(questions[1].answer).toBe('A golden retriever named Max');
+    });
+
+    it('should throw helpful error when dataset file not found', async () => {
+      // Use the regular loader which points to the data directory
+      await expect(loader.loadDataset('non-existent-dataset' as any)).rejects.toThrow('Dataset file not found');
+    });
+  });
+
+  describe('getDatasetStats', () => {
+    it('should calculate stats for test dataset', async () => {
+      const stats = await testLoader.getDatasetStats('test-dataset' as any);
+
+      expect(stats.totalQuestions).toBe(3);
+      expect(stats.questionsByType['single-session-user']).toBe(2);
+      expect(stats.questionsByType['multi-session']).toBe(1);
+      expect(stats.avgSessionsPerQuestion).toBeCloseTo(1.33, 2);
+      expect(stats.avgTurnsPerSession).toBe(2);
+      expect(stats.totalTokensEstimate).toBeGreaterThan(0);
+      expect(stats.abstentionQuestions).toBe(0); // None of our test questions are abstention questions
+    });
+  });
+});

package/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "lib": ["es2022"],
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true,
+    "moduleResolution": "bundler",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "tests"]
+}

package/vitest.config.ts

@@ -0,0 +1,9 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    testTimeout: 30000,
+  },
+});