@ai.ntellect/core 0.1.3 → 0.1.4

package/dist/index.d.ts

@@ -5,4 +5,3 @@ export * from "./services/queue";
 export * from "./types";
 export * from "./memory/cache";
 export * from "./memory/persistent";
-export type * from "./memory/persistent.d.ts";

package/dist/llm/evaluator/context.js

@@ -9,6 +9,7 @@ exports.evaluatorContext = {
             "Verify if all required actions were executed successfully",
             "Check if the results match the initial goal",
             "Identify any missing or incomplete information",
+            "Examples of relavant information: link symbol to token address, name to wallet, etc.",
         ],
         warnings: [
             "NEVER modify the results directly",
@@ -30,7 +31,10 @@ exports.evaluatorContext = {
       1. Success status with explanation (no action needed)
       2. Next actions needed (if any)
       3. Why you are doing the next actions or why you are not doing them
-      4. Extract relevant semantic information to remember (eg. token symbols, token addresses, etc.). No need to remember the actions or the news.
+      4. Extract relevant information to remember. No need to remember specific numbers.
+      5. If there are no important results, let importantToRemembers be empty. No need to say something like "No relevant information found".
+      6. For each facts, generate a hypothetical query to search in the persistent memory.
+      7. For each facts, generate a memoryType (You have 3 memory types: episodic, semantic, procedural)
     `;
     },
 };

package/dist/llm/evaluator/index.js

@@ -25,7 +25,9 @@ class Evaluator {
                         }),
                     })),
                     why: zod_1.z.string(),
+                    isImportantToRemember: zod_1.z.boolean(),
                     importantToRemembers: zod_1.z.array(zod_1.z.object({
+                        memoryType: zod_1.z.string(),
                         hypotheticalQuery: zod_1.z.string(),
                         result: zod_1.z.string(),
                     })),
@@ -40,10 +42,18 @@ class Evaluator {
                     parameters: action.parameters || {},
                 })),
             };
-            if (validatedResponse.importantToRemembers.length > 0) {
+            if (validatedResponse.isImportantToRemember) {
                 for (const item of validatedResponse.importantToRemembers) {
                     // Check if the item is already in the memory
-                    const memories = await this.memory.findBestMatches(item.hypotheticalQuery);
+                    const memories = await this.memory.findBestMatches(item.hypotheticalQuery, {
+                        similarityThreshold: 95,
+                    });
+                    if (memories.length > 0) {
+                        console.log("Similar memorie found, no need to remember", {
+                            memories,
+                        });
+                        continue;
+                    }
                     if (memories.length === 0) {
                         console.log("Adding to memory", {
                             query: item.hypotheticalQuery,
@@ -51,10 +61,10 @@ class Evaluator {
                         });
                         await this.memory.storeMemory({
                             id: crypto.randomUUID(),
-                            purpose: "importantToRemember",
+                            purpose: item.memoryType,
                             query: item.hypotheticalQuery,
                             data: item.result,
-                            scope: types_1.MemoryScope.USER,
+                            scope: types_1.MemoryScope.GLOBAL,
                             createdAt: new Date(),
                         });
                     }

package/dist/memory/persistent.d.ts

@@ -0,0 +1,60 @@
+import { Memory, MemoryScope } from "../types";
+interface SearchOptions {
+    scope?: MemoryScope;
+    userId?: string;
+    maxResults?: number;
+    similarityThreshold?: number;
+}
+interface ProcessedChunk {
+    content: string;
+    embedding: number[];
+}
+/**
+ * Handles persistent memory storage using Meilisearch API
+ */
+export declare class PersistentMemory {
+    private readonly host;
+    private readonly apiKey;
+    private readonly INDEX_PREFIX;
+    constructor(options: {
+        host: string;
+        apiKey: string;
+        indexPrefix?: string;
+    });
+    /**
+     * Initialize indexes
+     */
+    init(): Promise<void>;
+    /**
+     * Make API request to Meilisearch
+     */
+    private _makeRequest;
+    /**
+     * Get index name based on scope and userId
+     */
+    private _getIndexName;
+    /**
+     * Get or create an index with proper settings
+     */
+    private _getOrCreateIndex;
+    processContent(content: string): Promise<ProcessedChunk[]>;
+    /**
+     * Store a memory in the database
+     */
+    storeMemory(memory: Memory): Promise<unknown>;
+    /**
+     * Find best matching memories
+     */
+    findBestMatches(query: string, options?: SearchOptions): Promise<{
+        data: any;
+        purpose: string;
+        query: string;
+        chunk: string;
+        similarityPercentage: number;
+    }[]>;
+    /**
+     * Delete memories for a given scope and user
+     */
+    deleteMemories(scope: MemoryScope, userId?: string): Promise<unknown>;
+}
+export {};

package/dist/memory/persistent.js

@@ -0,0 +1,207 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PersistentMemory = void 0;
+const openai_1 = require("@ai-sdk/openai");
+const ai_1 = require("ai");
+const text_splitter_1 = require("langchain/text_splitter");
+const types_1 = require("../types");
+/**
+ * Handles persistent memory storage using Meilisearch API
+ */
+class PersistentMemory {
+    constructor(options) {
+        this.host = options.host;
+        this.apiKey = options.apiKey;
+        this.INDEX_PREFIX = options.indexPrefix || "memory_";
+    }
+    /**
+     * Initialize indexes
+     */
+    async init() {
+        // Create global index
+        await this._getOrCreateIndex(this._getIndexName(types_1.MemoryScope.GLOBAL));
+        // Create user index
+        await this._getOrCreateIndex(this._getIndexName(types_1.MemoryScope.USER));
+    }
+    /**
+     * Make API request to Meilisearch
+     */
+    async _makeRequest(path, options = {}) {
+        const url = `${this.host}${path}`;
+        console.log("Making request to:", url);
+        const response = await fetch(url, {
+            ...options,
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+                ...options.headers,
+            },
+        });
+        if (!response.ok) {
+            console.log({ response });
+            throw new Error(`Meilisearch API error: ${response.statusText}`);
+        }
+        return response.json();
+    }
+    /**
+     * Get index name based on scope and userId
+     */
+    _getIndexName(scope, userId) {
+        if (scope === "global") {
+            return `${this.INDEX_PREFIX}global`;
+        }
+        return `${this.INDEX_PREFIX}user_${userId}`;
+    }
+    /**
+     * Get or create an index with proper settings
+     */
+    async _getOrCreateIndex(indexName) {
+        try {
+            // Try to create index
+            await this._makeRequest("/indexes", {
+                method: "POST",
+                body: JSON.stringify({
+                    uid: indexName,
+                    primaryKey: "id",
+                }),
+            });
+            // Update index settings
+            const settings = {
+                searchableAttributes: ["query", "purpose", "chunks.content"],
+                sortableAttributes: ["createdAt"],
+            };
+            await this._makeRequest(`/indexes/${indexName}/settings`, {
+                method: "PATCH",
+                body: JSON.stringify(settings),
+            });
+        }
+        catch (error) {
+            // Index might already exist, which is fine
+            if (!error.message.includes("already exists")) {
+                throw error;
+            }
+        }
+    }
+    async processContent(content) {
+        // Split content into chunks
+        const textSplitter = new text_splitter_1.RecursiveCharacterTextSplitter({
+            chunkSize: 1000,
+        });
+        const chunks = await textSplitter.createDocuments([content]);
+        // Generate embeddings for all chunks
+        const { embeddings } = await (0, ai_1.embedMany)({
+            model: openai_1.openai.embedding("text-embedding-3-small"),
+            values: chunks.map((chunk) => chunk.pageContent),
+        });
+        // Create processed chunks with embeddings
+        return chunks.map((chunk, i) => ({
+            content: chunk.pageContent,
+            embedding: embeddings[i],
+        }));
+    }
+    /**
+     * Store a memory in the database
+     */
+    async storeMemory(memory) {
+        const indexName = this._getIndexName(memory.scope, memory.userId);
+        await this._getOrCreateIndex(indexName);
+        const chunks = await this.processContent(memory.query);
+        const document = {
+            ...memory,
+            chunks,
+            createdAt: memory.createdAt.toISOString(),
+        };
+        const response = await this._makeRequest(`/indexes/${indexName}/documents`, {
+            method: "POST",
+            body: JSON.stringify([document]),
+        });
+        console.log("Stored memory response:", response);
+        return response;
+    }
+    /**
+     * Find best matching memories
+     */
+    async findBestMatches(query, options = {}) {
+        console.log("\n🔍 Searching in persistent memory:", query);
+        // Generate embedding for the query
+        const { embedding: queryEmbedding } = await (0, ai_1.embed)({
+            model: openai_1.openai.embedding("text-embedding-3-small"),
+            value: query,
+        });
+        const searchResults = [];
+        // Requête Meilisearch
+        const searchBody = {
+            q: query,
+        };
+        // Search in global memories
+        if (!options.scope || options.scope === "global") {
+            const globalIndex = this._getIndexName(types_1.MemoryScope.GLOBAL);
+            console.log("Searching in global index:", globalIndex);
+            try {
+                const globalResults = await this._makeRequest(`/indexes/${globalIndex}/search`, {
+                    method: "POST",
+                    body: JSON.stringify(searchBody),
+                });
+                if (globalResults?.hits) {
+                    searchResults.push(...globalResults.hits);
+                }
+            }
+            catch (error) {
+                console.error("Error searching global index:", error);
+            }
+        }
+        // Search in user memories
+        if (options.userId &&
+            (!options.scope || options.scope === types_1.MemoryScope.USER)) {
+            const userIndex = this._getIndexName(types_1.MemoryScope.USER, options.userId);
+            const userResults = await this._makeRequest(`/indexes/${userIndex}/search`, {
+                method: "POST",
+                body: JSON.stringify(searchBody),
+            });
+            if (userResults.hits) {
+                searchResults.push(...userResults.hits);
+            }
+        }
+        console.log("Found in persistent memory:", searchResults);
+        // Process and filter results using cosine similarity
+        const results = searchResults
+            .flatMap((hit) => {
+            // Calculate similarities for each chunk
+            const chunkSimilarities = hit.chunks.map((chunk) => ({
+                data: hit.data,
+                purpose: hit.purpose,
+                query: hit.query,
+                chunk: chunk.content,
+                similarityPercentage: ((0, ai_1.cosineSimilarity)(queryEmbedding, chunk.embedding) + 1) * 50,
+            }));
+            // Return the chunk with highest similarity
+            return chunkSimilarities.reduce((best, current) => current.similarityPercentage > best.similarityPercentage
+                ? current
+                : best, chunkSimilarities[0]);
+        })
+            .filter((match) => match.similarityPercentage >= (options.similarityThreshold || 70))
+            .sort((a, b) => b.similarityPercentage - a.similarityPercentage);
+        // Log results
+        if (results.length > 0) {
+            console.log("\n✨ Best matches found:");
+            results.forEach((match) => {
+                console.log(`- ${match.query} : ${match.similarityPercentage.toFixed(2)}% (${match.purpose})`);
+                console.log(`  Matching content: "${match.chunk}"`);
+            });
+        }
+        else {
+            console.log("No matches found");
+        }
+        return results;
+    }
+    /**
+     * Delete memories for a given scope and user
+     */
+    async deleteMemories(scope, userId) {
+        const indexName = this._getIndexName(scope, userId);
+        return this._makeRequest(`/indexes/${indexName}`, {
+            method: "DELETE",
+        });
+    }
+}
+exports.PersistentMemory = PersistentMemory;

package/dist/test.js

@@ -107,7 +107,7 @@ exports.getRssNews = {
         host: "http://localhost:7700",
         apiKey: "aSampleMasterKey",
     });
-    const orchestrator = new orchestrator_1.Orchestrator([exports.getRssNews, exports.getChainsTVL], memory);
+    const orchestrator = new orchestrator_1.Orchestrator([], memory);
     const agent = new agent_1.Agent({
         user: {
             id: "1",
@@ -117,7 +117,7 @@ exports.getRssNews = {
         stream: false,
         maxEvaluatorIteration: 1,
     });
-    const prompt = "tu connais avax";
+    const prompt = "fais moi une analyse ethereum";
     const context = prompt;
     const result = await agent.process(prompt, context, {
         onMessage: (message) => {

package/index.ts

@@ -7,4 +7,3 @@ export * from "./types";
 
 export * from "./memory/cache";
 export * from "./memory/persistent";
-export type * from "./memory/persistent.d.ts";

package/llm/evaluator/context.ts

@@ -8,6 +8,7 @@ export const evaluatorContext = {
       "Verify if all required actions were executed successfully",
       "Check if the results match the initial goal",
       "Identify any missing or incomplete information",
+      "Examples of relavant information: link symbol to token address, name to wallet, etc.",
     ],
     warnings: [
       "NEVER modify the results directly",
@@ -29,7 +30,10 @@ export const evaluatorContext = {
       1. Success status with explanation (no action needed)
       2. Next actions needed (if any)
       3. Why you are doing the next actions or why you are not doing them
-      4. Extract relevant semantic information to remember (eg. token symbols, token addresses, etc.). No need to remember the actions or the news.
+      4. Extract relevant information to remember. No need to remember specific numbers.
+      5. If there are no important results, let importantToRemembers be empty. No need to say something like "No relevant information found".
+      6. For each facts, generate a hypothetical query to search in the persistent memory.
+      7. For each facts, generate a memoryType (You have 3 memory types: episodic, semantic, procedural)
     `;
   },
 };

package/llm/evaluator/index.ts

@@ -30,8 +30,10 @@ export class Evaluator {
             })
           ),
           why: z.string(),
+          isImportantToRemember: z.boolean(),
           importantToRemembers: z.array(
             z.object({
+              memoryType: z.string(),
               hypotheticalQuery: z.string(),
               result: z.string(),
             })
@@ -49,12 +51,21 @@ export class Evaluator {
         })),
       };
 
-      if (validatedResponse.importantToRemembers.length > 0) {
+      if (validatedResponse.isImportantToRemember) {
         for (const item of validatedResponse.importantToRemembers) {
           // Check if the item is already in the memory
           const memories = await this.memory.findBestMatches(
-            item.hypotheticalQuery
+            item.hypotheticalQuery,
+            {
+              similarityThreshold: 95,
+            }
           );
+          if (memories.length > 0) {
+            console.log("Similar memorie found, no need to remember", {
+              memories,
+            });
+            continue;
+          }
           if (memories.length === 0) {
             console.log("Adding to memory", {
               query: item.hypotheticalQuery,
@@ -62,10 +73,10 @@ export class Evaluator {
             });
             await this.memory.storeMemory({
               id: crypto.randomUUID(),
-              purpose: "importantToRemember",
+              purpose: item.memoryType,
               query: item.hypotheticalQuery,
               data: item.result,
-              scope: MemoryScope.USER,
+              scope: MemoryScope.GLOBAL,
               createdAt: new Date(),
             });
           }

package/memory/persistent.ts

@@ -0,0 +1,300 @@
+import { openai } from "@ai-sdk/openai";
+import { cosineSimilarity, embed, embedMany } from "ai";
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+import { Memory, MemoryScope } from "../types";
+
+interface SearchOptions {
+  scope?: MemoryScope;
+  userId?: string;
+  maxResults?: number;
+  similarityThreshold?: number;
+}
+
+interface MeilisearchSettings {
+  searchableAttributes?: string[];
+  sortableAttributes?: string[];
+}
+
+interface MeilisearchResponse {
+  hits: Array<{
+    query: string;
+    purpose: string;
+    data?: any;
+    chunks: Array<{
+      content: string;
+      embedding: number[];
+    }>;
+  }>;
+}
+
+interface SearchParams {
+  q?: string;
+  offset?: number;
+  limit?: number;
+  filter?: string | string[];
+  facets?: string[];
+  attributesToRetrieve?: string[];
+  attributesToSearchOn?: string[];
+  sort?: string[];
+  matchingStrategy?: "last" | "all" | "frequency";
+}
+
+interface ProcessedChunk {
+  content: string;
+  embedding: number[];
+}
+
+/**
+ * Handles persistent memory storage using Meilisearch API
+ */
+export class PersistentMemory {
+  private readonly host: string;
+  private readonly apiKey: string;
+  private readonly INDEX_PREFIX: string;
+
+  constructor(options: { host: string; apiKey: string; indexPrefix?: string }) {
+    this.host = options.host;
+    this.apiKey = options.apiKey;
+    this.INDEX_PREFIX = options.indexPrefix || "memory_";
+  }
+
+  /**
+   * Initialize indexes
+   */
+  async init() {
+    // Create global index
+    await this._getOrCreateIndex(this._getIndexName(MemoryScope.GLOBAL));
+
+    // Create user index
+    await this._getOrCreateIndex(this._getIndexName(MemoryScope.USER));
+  }
+
+  /**
+   * Make API request to Meilisearch
+   */
+  private async _makeRequest<T = unknown>(
+    path: string,
+    options: RequestInit = {}
+  ): Promise<T> {
+    const url = `${this.host}${path}`;
+    console.log("Making request to:", url);
+    const response = await fetch(url, {
+      ...options,
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+        ...options.headers,
+      },
+    });
+
+    if (!response.ok) {
+      console.log({ response });
+      throw new Error(`Meilisearch API error: ${response.statusText}`);
+    }
+
+    return response.json() as Promise<T>;
+  }
+
+  /**
+   * Get index name based on scope and userId
+   */
+  private _getIndexName(scope: MemoryScope, userId?: string): string {
+    if (scope === "global") {
+      return `${this.INDEX_PREFIX}global`;
+    }
+    return `${this.INDEX_PREFIX}user_${userId}`;
+  }
+
+  /**
+   * Get or create an index with proper settings
+   */
+  private async _getOrCreateIndex(indexName: string) {
+    try {
+      // Try to create index
+      await this._makeRequest("/indexes", {
+        method: "POST",
+        body: JSON.stringify({
+          uid: indexName,
+          primaryKey: "id",
+        }),
+      });
+
+      // Update index settings
+      const settings: MeilisearchSettings = {
+        searchableAttributes: ["query", "purpose", "chunks.content"],
+        sortableAttributes: ["createdAt"],
+      };
+
+      await this._makeRequest(`/indexes/${indexName}/settings`, {
+        method: "PATCH",
+        body: JSON.stringify(settings),
+      });
+    } catch (error: any) {
+      // Index might already exist, which is fine
+      if (!error.message.includes("already exists")) {
+        throw error;
+      }
+    }
+  }
+
+  async processContent(content: string): Promise<ProcessedChunk[]> {
+    // Split content into chunks
+    const textSplitter = new RecursiveCharacterTextSplitter({
+      chunkSize: 1000,
+    });
+    const chunks = await textSplitter.createDocuments([content]);
+
+    // Generate embeddings for all chunks
+    const { embeddings } = await embedMany({
+      model: openai.embedding("text-embedding-3-small"),
+      values: chunks.map((chunk) => chunk.pageContent),
+    });
+
+    // Create processed chunks with embeddings
+    return chunks.map((chunk, i) => ({
+      content: chunk.pageContent,
+      embedding: embeddings[i],
+    }));
+  }
+
+  /**
+   * Store a memory in the database
+   */
+  async storeMemory(memory: Memory) {
+    const indexName = this._getIndexName(memory.scope, memory.userId);
+    await this._getOrCreateIndex(indexName);
+
+    const chunks = await this.processContent(memory.query);
+
+    const document = {
+      ...memory,
+      chunks,
+      createdAt: memory.createdAt.toISOString(),
+    };
+
+    const response = await this._makeRequest(
+      `/indexes/${indexName}/documents`,
+      {
+        method: "POST",
+        body: JSON.stringify([document]),
+      }
+    );
+    console.log("Stored memory response:", response);
+    return response;
+  }
+
+  /**
+   * Find best matching memories
+   */
+  async findBestMatches(query: string, options: SearchOptions = {}) {
+    console.log("\n🔍 Searching in persistent memory:", query);
+
+    // Generate embedding for the query
+    const { embedding: queryEmbedding } = await embed({
+      model: openai.embedding("text-embedding-3-small"),
+      value: query,
+    });
+
+    const searchResults = [];
+
+    // Requête Meilisearch
+    const searchBody = {
+      q: query,
+    };
+
+    // Search in global memories
+    if (!options.scope || options.scope === "global") {
+      const globalIndex = this._getIndexName(MemoryScope.GLOBAL);
+      console.log("Searching in global index:", globalIndex);
+      try {
+        const globalResults = await this._makeRequest<MeilisearchResponse>(
+          `/indexes/${globalIndex}/search`,
+          {
+            method: "POST",
+            body: JSON.stringify(searchBody),
+          }
+        );
+        if (globalResults?.hits) {
+          searchResults.push(...globalResults.hits);
+        }
+      } catch (error) {
+        console.error("Error searching global index:", error);
+      }
+    }
+
+    // Search in user memories
+    if (
+      options.userId &&
+      (!options.scope || options.scope === MemoryScope.USER)
+    ) {
+      const userIndex = this._getIndexName(MemoryScope.USER, options.userId);
+      const userResults = await this._makeRequest<MeilisearchResponse>(
+        `/indexes/${userIndex}/search`,
+        {
+          method: "POST",
+          body: JSON.stringify(searchBody),
+        }
+      );
+      if (userResults.hits) {
+        searchResults.push(...userResults.hits);
+      }
+    }
+
+    console.log("Found in persistent memory:", searchResults);
+
+    // Process and filter results using cosine similarity
+    const results = searchResults
+      .flatMap((hit) => {
+        // Calculate similarities for each chunk
+        const chunkSimilarities = hit.chunks.map((chunk) => ({
+          data: hit.data,
+          purpose: hit.purpose,
+          query: hit.query,
+          chunk: chunk.content,
+          similarityPercentage:
+            (cosineSimilarity(queryEmbedding, chunk.embedding) + 1) * 50,
+        }));
+
+        // Return the chunk with highest similarity
+        return chunkSimilarities.reduce(
+          (best, current) =>
+            current.similarityPercentage > best.similarityPercentage
+              ? current
+              : best,
+          chunkSimilarities[0]
+        );
+      })
+      .filter(
+        (match) =>
+          match.similarityPercentage >= (options.similarityThreshold || 70)
+      )
+      .sort((a, b) => b.similarityPercentage - a.similarityPercentage);
+
+    // Log results
+    if (results.length > 0) {
+      console.log("\n✨ Best matches found:");
+      results.forEach((match) => {
+        console.log(
+          `- ${match.query} : ${match.similarityPercentage.toFixed(2)}% (${
+            match.purpose
+          })`
+        );
+        console.log(`  Matching content: "${match.chunk}"`);
+      });
+    } else {
+      console.log("No matches found");
+    }
+
+    return results;
+  }
+
+  /**
+   * Delete memories for a given scope and user
+   */
+  async deleteMemories(scope: MemoryScope, userId?: string) {
+    const indexName = this._getIndexName(scope, userId);
+    return this._makeRequest(`/indexes/${indexName}`, {
+      method: "DELETE",
+    });
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai.ntellect/core",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "",
   "main": "dist/index.js",
   "scripts": {

package/test.ts

@@ -127,7 +127,7 @@ export const getRssNews = {
     host: "http://localhost:7700",
     apiKey: "aSampleMasterKey",
   });
-  const orchestrator = new Orchestrator([getRssNews, getChainsTVL], memory);
+  const orchestrator = new Orchestrator([], memory);
   const agent = new Agent({
     user: {
       id: "1",
@@ -138,9 +138,8 @@ export const getRssNews = {
     maxEvaluatorIteration: 1,
   });
 
-  const prompt = "tu connais avax";
+  const prompt = "fais moi une analyse ethereum";
   const context = prompt;
-
   const result = await agent.process(prompt, context, {
     onMessage: (message) => {
       console.log({ message });

package/memory/persistent.d.ts

@@ -1,28 +0,0 @@
-import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
-import { Index, MeiliSearch } from "meilisearch";
-import {
-  MatchOptions,
-  MatchResult,
-  Memory,
-  MemoryChunk,
-  MemoryScopeType,
-} from "../types";
-
-export class PersistentMemory {
-  private client: MeiliSearch;
-  private readonly INDEX_PREFIX: string;
-  private textSplitter: RecursiveCharacterTextSplitter;
-
-  constructor(options: PersistentMemoryOptions);
-
-  private _getIndexName(scope: MemoryScopeType, userId?: string): string;
-  private _getOrCreateIndex(indexName: string): Promise<Index>;
-  private _processContent(content: string): Promise<MemoryChunk[]>;
-
-  storeMemory(memory: Memory): Promise<void>;
-  findBestMatches(
-    query: string,
-    options?: MatchOptions
-  ): Promise<MatchResult[]>;
-  deleteMemories(scope: MemoryScopeType, userId?: string): Promise<void>;
-}

package/memory/persistent.js

@@ -1,198 +0,0 @@
-import { openai } from "@ai-sdk/openai";
-import { cosineSimilarity, embed, embedMany } from "ai";
-import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
-import { MeiliSearch } from "meilisearch";
-
-const MemoryScope = {
-  GLOBAL: "global",
-  USER: "user",
-};
-
-export class PersistentMemory {
-  /**
-   * @param {Object} options
-   * @param {string} options.host - MeiliSearch host URL
-   * @param {string} options.apiKey - MeiliSearch API key
-   * @param {string} [options.indexPrefix="memory_"] - Prefix for index names
-   */
-  constructor(options) {
-    this.client = new MeiliSearch({
-      host: options.host,
-      apiKey: options.apiKey,
-    });
-    this.INDEX_PREFIX = options.indexPrefix || "memory_";
-    this.textSplitter = new RecursiveCharacterTextSplitter({
-      chunkSize: 1000,
-    });
-  }
-
-  /**
-   * Get the index name based on scope and userId
-   * @private
-   */
-  _getIndexName(scope, userId) {
-    if (scope === MemoryScope.GLOBAL) {
-      return `${this.INDEX_PREFIX}global`;
-    }
-    return `${this.INDEX_PREFIX}user_${userId}`;
-  }
-
-  /**
-   * Get or create an index with proper settings
-   * @private
-   */
-  async _getOrCreateIndex(indexName) {
-    const index = this.client.index(indexName);
-
-    try {
-      await this.client.createIndex(indexName, { primaryKey: "id" });
-      await index.updateSettings({
-        searchableAttributes: ["query", "purpose", "chunks.content"],
-        sortableAttributes: ["createdAt"],
-      });
-    } catch (error) {
-      // Index might already exist, which is fine
-      if (!error.message.includes("already exists")) {
-        throw error;
-      }
-    }
-
-    return index;
-  }
-
-  /**
-   * Process content into chunks with embeddings
-   * @private
-   */
-  async _processContent(content) {
-    // Split content into chunks
-    const chunks = await this.textSplitter.createDocuments([content]);
-
-    // Generate embeddings for all chunks
-    const { embeddings } = await embedMany({
-      model: openai.embedding("text-embedding-3-small"),
-      values: chunks.map((chunk) => chunk.pageContent),
-    });
-
-    // Create processed chunks with embeddings
-    return chunks.map((chunk, i) => ({
-      content: chunk.pageContent,
-      embedding: embeddings[i],
-    }));
-  }
-
-  /**
-   * Store a memory in the database
-   * @param {Object} memory - Memory object to store
-   */
-  async storeMemory(memory) {
-    const indexName = this._getIndexName(memory.scope, memory.userId);
-    const index = await this._getOrCreateIndex(indexName);
-
-    // Process the query into chunks with embeddings
-    const chunks = await this._processContent(memory.query);
-
-    const result = await index.addDocuments([
-      {
-        ...memory,
-        chunks,
-        createdAt: memory.createdAt.toISOString(),
-      },
-    ]);
-    return result;
-  }
-
-  /**
-   * Find best matching memories using cosine similarity
-   * @param {string} query - Search query
-   * @param {Object} options - Search options
-   */
-  async findBestMatches(query, options = {}) {
-    console.log("\n🔍 Searching in persistent memory:", query);
-
-    // Generate embedding for the query
-    const { embedding: queryEmbedding } = await embed({
-      model: openai.embedding("text-embedding-3-small"),
-      value: query,
-    });
-    const searchResults = [];
-
-    // Search in global memories
-    if (!options.scope || options.scope === MemoryScope.GLOBAL) {
-      const globalIndex = await this._getOrCreateIndex(
-        this._getIndexName(MemoryScope.GLOBAL)
-      );
-      const globalResults = await globalIndex.search(query, {
-        limit: options.maxResults || 10,
-      });
-      searchResults.push(...globalResults.hits);
-    }
-
-    // Search in user memories
-    if (
-      options.userId &&
-      (!options.scope || options.scope === MemoryScope.USER)
-    ) {
-      const userIndex = await this._getOrCreateIndex(
-        this._getIndexName(MemoryScope.USER, options.userId)
-      );
-      const userResults = await userIndex.search(query, {
-        limit: options.maxResults || 10,
-      });
-      searchResults.push(...userResults.hits);
-    }
-
-    // Process and filter results using cosine similarity
-    const results = searchResults
-      .flatMap((hit) => {
-        // Calculate similarities for each chunk
-        console.log(hit);
-        const chunkSimilarities = hit.chunks.map((chunk) => ({
-          data: hit.data,
-          purpose: hit.purpose,
-          chunk: chunk.content,
-          similarityPercentage:
-            (cosineSimilarity(queryEmbedding, chunk.embedding) + 1) * 50,
-        }));
-        console.log({ chunkSimilarities });
-        // Return the chunk with highest similarity
-        return chunkSimilarities.reduce(
-          (best, current) =>
-            current.similarityPercentage > best.similarityPercentage
-              ? current
-              : best,
-          chunkSimilarities[0]
-        );
-      })
-      .filter(
-        (match) =>
-          match.similarityPercentage >= (options.similarityThreshold || 70)
-      )
-      .sort((a, b) => b.similarityPercentage - a.similarityPercentage);
-
-    // Log results
-    if (results.length > 0) {
-      console.log("\n✨ Best matches found:");
-      results.forEach((match) => {
-        console.log(
-          `- ${match.purpose} (${match.similarityPercentage.toFixed(2)}%)`
-        );
-        console.log(`  Matching chunk: "${match.chunk}"`);
-      });
-    } else {
-      console.log("No matches found");
-    }
-
-    return results;
-  }
-
-  /**
-   * Delete memories for a given scope and user
-   * @param {string} scope - Memory scope
-   * @param {string} [userId] - User ID for user-specific memories
-   */
-  async deleteMemories(scope, userId) {
-    const indexName = this._getIndexName(scope, userId);
-    await this.client.deleteIndex(indexName);
-  }
-}