@ai.ntellect/core 0.1.3 → 0.1.5

package/dist/index.d.ts

@@ -5,4 +5,3 @@ export * from "./services/queue";
 export * from "./types";
 export * from "./memory/cache";
 export * from "./memory/persistent";
-export type * from "./memory/persistent.d.ts";

package/dist/llm/evaluator/context.js

@@ -9,6 +9,7 @@ exports.evaluatorContext = {
             "Verify if all required actions were executed successfully",
             "Check if the results match the initial goal",
             "Identify any missing or incomplete information",
+            "Examples of relavant information: link symbol to token address, name to wallet, etc.",
         ],
         warnings: [
             "NEVER modify the results directly",
@@ -30,7 +31,10 @@ exports.evaluatorContext = {
       1. Success status with explanation (no action needed)
       2. Next actions needed (if any)
       3. Why you are doing the next actions or why you are not doing them
-      4. Extract relevant semantic information to remember (eg. token symbols, token addresses, etc.). No need to remember the actions or the news.
+      4. Extract relevant information to remember. No need to remember specific numbers.
+      5. If there are no important results, let importantToRemembers be empty. No need to say something like "No relevant information found".
+      6. For each facts, generate a hypothetical query to search in the persistent memory.
+      7. For each facts, generate a memoryType (You have 3 memory types: episodic, semantic, procedural)
     `;
     },
 };

package/dist/llm/evaluator/index.js

@@ -25,7 +25,9 @@ class Evaluator {
                         }),
                     })),
                     why: zod_1.z.string(),
+                    isImportantToRemember: zod_1.z.boolean(),
                     importantToRemembers: zod_1.z.array(zod_1.z.object({
+                        memoryType: zod_1.z.string(),
                         hypotheticalQuery: zod_1.z.string(),
                         result: zod_1.z.string(),
                     })),
@@ -40,10 +42,18 @@ class Evaluator {
                     parameters: action.parameters || {},
                 })),
             };
-            if (validatedResponse.importantToRemembers.length > 0) {
+            if (validatedResponse.isImportantToRemember) {
                 for (const item of validatedResponse.importantToRemembers) {
                     // Check if the item is already in the memory
-                    const memories = await this.memory.findBestMatches(item.hypotheticalQuery);
+                    const memories = await this.memory.searchSimilarQueries(item.hypotheticalQuery, {
+                        similarityThreshold: 95,
+                    });
+                    if (memories.length > 0) {
+                        console.log("Similar memorie found, no need to remember", {
+                            memories,
+                        });
+                        continue;
+                    }
                     if (memories.length === 0) {
                         console.log("Adding to memory", {
                             query: item.hypotheticalQuery,
@@ -51,10 +61,10 @@ class Evaluator {
                         });
                         await this.memory.storeMemory({
                             id: crypto.randomUUID(),
-                            purpose: "importantToRemember",
+                            purpose: item.memoryType,
                             query: item.hypotheticalQuery,
                             data: item.result,
-                            scope: types_1.MemoryScope.USER,
+                            scope: types_1.MemoryScope.GLOBAL,
                             createdAt: new Date(),
                         });
                     }
@@ -72,7 +82,7 @@ class Evaluator {
                 if (error.value.importantToRemembers.length > 0) {
                     for (const item of error.value.importantToRemembers) {
                         // Check if the item is already in the memory
-                        const memories = await this.memory.findBestMatches(item.hypotheticalQuery);
+                        const memories = await this.memory.searchSimilarQueries(item.hypotheticalQuery);
                         if (memories.length === 0) {
                             console.log("Adding to memory", {
                                 query: item.hypotheticalQuery,

package/dist/llm/orchestrator/context.js

@@ -9,6 +9,7 @@ exports.orchestratorContext = {
             "If there is no action to do, you must answer in the 'answer' field.",
             "If some parameters are not clear or missing, YOU MUST ask the user for them.",
             "ALWAYS use the same language as user request. (If it's English, use English, if it's French, use French, etc.)",
+            "Before executing multiple actions, you must search the memory for similar queries.",
         ],
         warnings: ["NEVER repeat same actions if the user doesn't ask for it."],
     },

package/dist/llm/orchestrator/index.js

@@ -18,7 +18,7 @@ class Orchestrator {
                     query: zod_1.z.string(),
                 }),
                 execute: async (params) => {
-                    const memories = await this.memory.findBestMatches(params.value);
+                    const memories = await this.memory.searchSimilarQueries(params.value);
                     return memories;
                 },
             },

package/dist/memory/persistent.d.ts

@@ -0,0 +1,60 @@
+import { Memory, MemoryScope } from "../types";
+interface SearchOptions {
+    scope?: MemoryScope;
+    userId?: string;
+    maxResults?: number;
+    similarityThreshold?: number;
+}
+interface ProcessedChunk {
+    content: string;
+    embedding: number[];
+}
+/**
+ * Handles persistent memory storage using Meilisearch API
+ */
+export declare class PersistentMemory {
+    private readonly host;
+    private readonly apiKey;
+    private readonly INDEX_PREFIX;
+    constructor(options: {
+        host: string;
+        apiKey: string;
+        indexPrefix?: string;
+    });
+    /**
+     * Initialize indexes
+     */
+    init(): Promise<void>;
+    /**
+     * Make API request to Meilisearch
+     */
+    private _makeRequest;
+    /**
+     * Get index name based on scope and userId
+     */
+    private _getIndexName;
+    /**
+     * Get or create an index with proper settings
+     */
+    private _getOrCreateIndex;
+    processContent(content: string): Promise<ProcessedChunk[]>;
+    /**
+     * Store a memory in the database
+     */
+    storeMemory(memory: Memory): Promise<unknown>;
+    /**
+     * Find best matching memories
+     */
+    searchSimilarQueries(query: string, options?: SearchOptions): Promise<{
+        data: any;
+        purpose: string;
+        query: string;
+        chunk: string;
+        similarityPercentage: number;
+    }[]>;
+    /**
+     * Delete memories for a given scope and user
+     */
+    deleteMemories(scope: MemoryScope, userId?: string): Promise<unknown>;
+}
+export {};

package/dist/memory/persistent.js

@@ -0,0 +1,207 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PersistentMemory = void 0;
+const openai_1 = require("@ai-sdk/openai");
+const ai_1 = require("ai");
+const text_splitter_1 = require("langchain/text_splitter");
+const types_1 = require("../types");
+/**
+ * Handles persistent memory storage using Meilisearch API
+ */
+class PersistentMemory {
+    constructor(options) {
+        this.host = options.host;
+        this.apiKey = options.apiKey;
+        this.INDEX_PREFIX = options.indexPrefix || "memory_";
+    }
+    /**
+     * Initialize indexes
+     */
+    async init() {
+        // Create global index
+        await this._getOrCreateIndex(this._getIndexName(types_1.MemoryScope.GLOBAL));
+        // Create user index
+        await this._getOrCreateIndex(this._getIndexName(types_1.MemoryScope.USER));
+    }
+    /**
+     * Make API request to Meilisearch
+     */
+    async _makeRequest(path, options = {}) {
+        const url = `${this.host}${path}`;
+        console.log("Making request to:", url);
+        const response = await fetch(url, {
+            ...options,
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+                ...options.headers,
+            },
+        });
+        if (!response.ok) {
+            console.log({ response });
+            throw new Error(`Meilisearch API error: ${response.statusText}`);
+        }
+        return response.json();
+    }
+    /**
+     * Get index name based on scope and userId
+     */
+    _getIndexName(scope, userId) {
+        if (scope === "global") {
+            return `${this.INDEX_PREFIX}global`;
+        }
+        return `${this.INDEX_PREFIX}user_${userId}`;
+    }
+    /**
+     * Get or create an index with proper settings
+     */
+    async _getOrCreateIndex(indexName) {
+        try {
+            // Try to create index
+            await this._makeRequest("/indexes", {
+                method: "POST",
+                body: JSON.stringify({
+                    uid: indexName,
+                    primaryKey: "id",
+                }),
+            });
+            // Update index settings
+            const settings = {
+                searchableAttributes: ["query", "purpose", "chunks.content"],
+                sortableAttributes: ["createdAt"],
+            };
+            await this._makeRequest(`/indexes/${indexName}/settings`, {
+                method: "PATCH",
+                body: JSON.stringify(settings),
+            });
+        }
+        catch (error) {
+            // Index might already exist, which is fine
+            if (!error.message.includes("already exists")) {
+                throw error;
+            }
+        }
+    }
+    async processContent(content) {
+        // Split content into chunks
+        const textSplitter = new text_splitter_1.RecursiveCharacterTextSplitter({
+            chunkSize: 1000,
+        });
+        const chunks = await textSplitter.createDocuments([content]);
+        // Generate embeddings for all chunks
+        const { embeddings } = await (0, ai_1.embedMany)({
+            model: openai_1.openai.embedding("text-embedding-3-small"),
+            values: chunks.map((chunk) => chunk.pageContent),
+        });
+        // Create processed chunks with embeddings
+        return chunks.map((chunk, i) => ({
+            content: chunk.pageContent,
+            embedding: embeddings[i],
+        }));
+    }
+    /**
+     * Store a memory in the database
+     */
+    async storeMemory(memory) {
+        const indexName = this._getIndexName(memory.scope, memory.userId);
+        await this._getOrCreateIndex(indexName);
+        const chunks = await this.processContent(memory.query);
+        const document = {
+            ...memory,
+            chunks,
+            createdAt: memory.createdAt.toISOString(),
+        };
+        const response = await this._makeRequest(`/indexes/${indexName}/documents`, {
+            method: "POST",
+            body: JSON.stringify([document]),
+        });
+        console.log("Stored memory response:", response);
+        return response;
+    }
+    /**
+     * Find best matching memories
+     */
+    async searchSimilarQueries(query, options = {}) {
+        console.log("\n🔍 Searching in persistent memory:", query);
+        // Generate embedding for the query
+        const { embedding: queryEmbedding } = await (0, ai_1.embed)({
+            model: openai_1.openai.embedding("text-embedding-3-small"),
+            value: query,
+        });
+        const searchResults = [];
+        // Requête Meilisearch
+        const searchBody = {
+            q: query,
+        };
+        // Search in global memories
+        if (!options.scope || options.scope === "global") {
+            const globalIndex = this._getIndexName(types_1.MemoryScope.GLOBAL);
+            console.log("Searching in global index:", globalIndex);
+            try {
+                const globalResults = await this._makeRequest(`/indexes/${globalIndex}/search`, {
+                    method: "POST",
+                    body: JSON.stringify(searchBody),
+                });
+                if (globalResults?.hits) {
+                    searchResults.push(...globalResults.hits);
+                }
+            }
+            catch (error) {
+                console.error("Error searching global index:", error);
+            }
+        }
+        // Search in user memories
+        if (options.userId &&
+            (!options.scope || options.scope === types_1.MemoryScope.USER)) {
+            const userIndex = this._getIndexName(types_1.MemoryScope.USER, options.userId);
+            const userResults = await this._makeRequest(`/indexes/${userIndex}/search`, {
+                method: "POST",
+                body: JSON.stringify(searchBody),
+            });
+            if (userResults.hits) {
+                searchResults.push(...userResults.hits);
+            }
+        }
+        console.log("Found in persistent memory:", searchResults);
+        // Process and filter results using cosine similarity
+        const results = searchResults
+            .flatMap((hit) => {
+            // Calculate similarities for each chunk
+            const chunkSimilarities = hit.chunks.map((chunk) => ({
+                data: hit.data,
+                purpose: hit.purpose,
+                query: hit.query,
+                chunk: chunk.content,
+                similarityPercentage: ((0, ai_1.cosineSimilarity)(queryEmbedding, chunk.embedding) + 1) * 50,
+            }));
+            // Return the chunk with highest similarity
+            return chunkSimilarities.reduce((best, current) => current.similarityPercentage > best.similarityPercentage
+                ? current
+                : best, chunkSimilarities[0]);
+        })
+            .filter((match) => match.similarityPercentage >= (options.similarityThreshold || 70))
+            .sort((a, b) => b.similarityPercentage - a.similarityPercentage);
+        // Log results
+        if (results.length > 0) {
+            console.log("\n✨ Best matches found:");
+            results.forEach((match) => {
+                console.log(`- ${match.query} : ${match.similarityPercentage.toFixed(2)}% (${match.purpose})`);
+                console.log(`  Matching content: "${match.chunk}"`);
+            });
+        }
+        else {
+            console.log("No matches found");
+        }
+        return results;
+    }
+    /**
+     * Delete memories for a given scope and user
+     */
+    async deleteMemories(scope, userId) {
+        const indexName = this._getIndexName(scope, userId);
+        return this._makeRequest(`/indexes/${indexName}`, {
+            method: "DELETE",
+        });
+    }
+}
+exports.PersistentMemory = PersistentMemory;

package/dist/test.js

@@ -107,7 +107,7 @@ exports.getRssNews = {
         host: "http://localhost:7700",
         apiKey: "aSampleMasterKey",
     });
-    const orchestrator = new orchestrator_1.Orchestrator([exports.getRssNews, exports.getChainsTVL], memory);
+    const orchestrator = new orchestrator_1.Orchestrator([], memory);
     const agent = new agent_1.Agent({
         user: {
             id: "1",
@@ -117,7 +117,7 @@ exports.getRssNews = {
         stream: false,
         maxEvaluatorIteration: 1,
     });
-    const prompt = "tu connais avax";
+    const prompt = "fais moi une analyse ethereum";
     const context = prompt;
     const result = await agent.process(prompt, context, {
         onMessage: (message) => {

package/index.ts

@@ -7,4 +7,3 @@ export * from "./types";
 
 export * from "./memory/cache";
 export * from "./memory/persistent";
-export type * from "./memory/persistent.d.ts";

package/llm/evaluator/context.ts

@@ -8,6 +8,7 @@ export const evaluatorContext = {
       "Verify if all required actions were executed successfully",
       "Check if the results match the initial goal",
       "Identify any missing or incomplete information",
+      "Examples of relavant information: link symbol to token address, name to wallet, etc.",
     ],
     warnings: [
       "NEVER modify the results directly",
@@ -29,7 +30,10 @@ export const evaluatorContext = {
       1. Success status with explanation (no action needed)
       2. Next actions needed (if any)
       3. Why you are doing the next actions or why you are not doing them
-      4. Extract relevant semantic information to remember (eg. token symbols, token addresses, etc.). No need to remember the actions or the news.
+      4. Extract relevant information to remember. No need to remember specific numbers.
+      5. If there are no important results, let importantToRemembers be empty. No need to say something like "No relevant information found".
+      6. For each facts, generate a hypothetical query to search in the persistent memory.
+      7. For each facts, generate a memoryType (You have 3 memory types: episodic, semantic, procedural)
     `;
   },
 };

package/llm/evaluator/index.ts

@@ -30,8 +30,10 @@ export class Evaluator {
             })
           ),
           why: z.string(),
+          isImportantToRemember: z.boolean(),
           importantToRemembers: z.array(
             z.object({
+              memoryType: z.string(),
               hypotheticalQuery: z.string(),
               result: z.string(),
             })
@@ -49,12 +51,21 @@ export class Evaluator {
         })),
       };
 
-      if (validatedResponse.importantToRemembers.length > 0) {
+      if (validatedResponse.isImportantToRemember) {
         for (const item of validatedResponse.importantToRemembers) {
           // Check if the item is already in the memory
-          const memories = await this.memory.findBestMatches(
-            item.hypotheticalQuery
+          const memories = await this.memory.searchSimilarQueries(
+            item.hypotheticalQuery,
+            {
+              similarityThreshold: 95,
+            }
           );
+          if (memories.length > 0) {
+            console.log("Similar memorie found, no need to remember", {
+              memories,
+            });
+            continue;
+          }
           if (memories.length === 0) {
             console.log("Adding to memory", {
               query: item.hypotheticalQuery,
@@ -62,10 +73,10 @@ export class Evaluator {
             });
             await this.memory.storeMemory({
               id: crypto.randomUUID(),
-              purpose: "importantToRemember",
+              purpose: item.memoryType,
               query: item.hypotheticalQuery,
               data: item.result,
-              scope: MemoryScope.USER,
+              scope: MemoryScope.GLOBAL,
               createdAt: new Date(),
             });
           }
@@ -83,7 +94,7 @@ export class Evaluator {
         if (error.value.importantToRemembers.length > 0) {
           for (const item of error.value.importantToRemembers) {
             // Check if the item is already in the memory
-            const memories = await this.memory.findBestMatches(
+            const memories = await this.memory.searchSimilarQueries(
               item.hypotheticalQuery
             );
             if (memories.length === 0) {

package/llm/orchestrator/context.ts

@@ -8,6 +8,7 @@ export const orchestratorContext = {
       "If there is no action to do, you must answer in the 'answer' field.",
       "If some parameters are not clear or missing, YOU MUST ask the user for them.",
       "ALWAYS use the same language as user request. (If it's English, use English, if it's French, use French, etc.)",
+      "Before executing multiple actions, you must search the memory for similar queries.",
     ],
     warnings: ["NEVER repeat same actions if the user doesn't ask for it."],
   },

package/llm/orchestrator/index.ts

@@ -1,6 +1,7 @@
 import { openai } from "@ai-sdk/openai";
 import { generateObject } from "ai";
 import { z } from "zod";
+import { CacheMemory } from "../../memory/cache";
 import { PersistentMemory } from "../../memory/persistent";
 import { ActionSchema, BaseLLM } from "../../types";
 import { orchestratorContext } from "./context";
@@ -9,7 +10,6 @@ export class Orchestrator implements BaseLLM {
   private readonly model = openai("gpt-4o");
   public tools: ActionSchema[];
   private memory: PersistentMemory;
-
   constructor(tools: ActionSchema[], memory: PersistentMemory) {
     this.memory = memory;
     this.tools = [
@@ -22,7 +22,7 @@ export class Orchestrator implements BaseLLM {
           query: z.string(),
         }),
         execute: async (params) => {
-          const memories = await this.memory.findBestMatches(params.value);
+          const memories = await this.memory.searchSimilarQueries(params.value);
           return memories;
         },
       },

package/memory/persistent.ts

@@ -0,0 +1,300 @@
+import { openai } from "@ai-sdk/openai";
+import { cosineSimilarity, embed, embedMany } from "ai";
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+import { Memory, MemoryScope } from "../types";
+
+interface SearchOptions {
+  scope?: MemoryScope;
+  userId?: string;
+  maxResults?: number;
+  similarityThreshold?: number;
+}
+
+interface MeilisearchSettings {
+  searchableAttributes?: string[];
+  sortableAttributes?: string[];
+}
+
+interface MeilisearchResponse {
+  hits: Array<{
+    query: string;
+    purpose: string;
+    data?: any;
+    chunks: Array<{
+      content: string;
+      embedding: number[];
+    }>;
+  }>;
+}
+
+interface SearchParams {
+  q?: string;
+  offset?: number;
+  limit?: number;
+  filter?: string | string[];
+  facets?: string[];
+  attributesToRetrieve?: string[];
+  attributesToSearchOn?: string[];
+  sort?: string[];
+  matchingStrategy?: "last" | "all" | "frequency";
+}
+
+interface ProcessedChunk {
+  content: string;
+  embedding: number[];
+}
+
+/**
+ * Handles persistent memory storage using Meilisearch API
+ */
+export class PersistentMemory {
+  private readonly host: string;
+  private readonly apiKey: string;
+  private readonly INDEX_PREFIX: string;
+
+  constructor(options: { host: string; apiKey: string; indexPrefix?: string }) {
+    this.host = options.host;
+    this.apiKey = options.apiKey;
+    this.INDEX_PREFIX = options.indexPrefix || "memory_";
+  }
+
+  /**
+   * Initialize indexes
+   */
+  async init() {
+    // Create global index
+    await this._getOrCreateIndex(this._getIndexName(MemoryScope.GLOBAL));
+
+    // Create user index
+    await this._getOrCreateIndex(this._getIndexName(MemoryScope.USER));
+  }
+
+  /**
+   * Make API request to Meilisearch
+   */
+  private async _makeRequest<T = unknown>(
+    path: string,
+    options: RequestInit = {}
+  ): Promise<T> {
+    const url = `${this.host}${path}`;
+    console.log("Making request to:", url);
+    const response = await fetch(url, {
+      ...options,
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+        ...options.headers,
+      },
+    });
+
+    if (!response.ok) {
+      console.log({ response });
+      throw new Error(`Meilisearch API error: ${response.statusText}`);
+    }
+
+    return response.json() as Promise<T>;
+  }
+
+  /**
+   * Get index name based on scope and userId
+   */
+  private _getIndexName(scope: MemoryScope, userId?: string): string {
+    if (scope === "global") {
+      return `${this.INDEX_PREFIX}global`;
+    }
+    return `${this.INDEX_PREFIX}user_${userId}`;
+  }
+
+  /**
+   * Get or create an index with proper settings
+   */
+  private async _getOrCreateIndex(indexName: string) {
+    try {
+      // Try to create index
+      await this._makeRequest("/indexes", {
+        method: "POST",
+        body: JSON.stringify({
+          uid: indexName,
+          primaryKey: "id",
+        }),
+      });
+
+      // Update index settings
+      const settings: MeilisearchSettings = {
+        searchableAttributes: ["query", "purpose", "chunks.content"],
+        sortableAttributes: ["createdAt"],
+      };
+
+      await this._makeRequest(`/indexes/${indexName}/settings`, {
+        method: "PATCH",
+        body: JSON.stringify(settings),
+      });
+    } catch (error: any) {
+      // Index might already exist, which is fine
+      if (!error.message.includes("already exists")) {
+        throw error;
+      }
+    }
+  }
+
+  async processContent(content: string): Promise<ProcessedChunk[]> {
+    // Split content into chunks
+    const textSplitter = new RecursiveCharacterTextSplitter({
+      chunkSize: 1000,
+    });
+    const chunks = await textSplitter.createDocuments([content]);
+
+    // Generate embeddings for all chunks
+    const { embeddings } = await embedMany({
+      model: openai.embedding("text-embedding-3-small"),
+      values: chunks.map((chunk) => chunk.pageContent),
+    });
+
+    // Create processed chunks with embeddings
+    return chunks.map((chunk, i) => ({
+      content: chunk.pageContent,
+      embedding: embeddings[i],
+    }));
+  }
+
+  /**
+   * Store a memory in the database
+   */
+  async storeMemory(memory: Memory) {
+    const indexName = this._getIndexName(memory.scope, memory.userId);
+    await this._getOrCreateIndex(indexName);
+
+    const chunks = await this.processContent(memory.query);
+
+    const document = {
+      ...memory,
+      chunks,
+      createdAt: memory.createdAt.toISOString(),
+    };
+
+    const response = await this._makeRequest(
+      `/indexes/${indexName}/documents`,
+      {
+        method: "POST",
+        body: JSON.stringify([document]),
+      }
+    );
+    console.log("Stored memory response:", response);
+    return response;
+  }
+
+  /**
+   * Find best matching memories
+   */
+  async searchSimilarQueries(query: string, options: SearchOptions = {}) {
+    console.log("\n🔍 Searching in persistent memory:", query);
+
+    // Generate embedding for the query
+    const { embedding: queryEmbedding } = await embed({
+      model: openai.embedding("text-embedding-3-small"),
+      value: query,
+    });
+
+    const searchResults = [];
+
+    // Requête Meilisearch
+    const searchBody = {
+      q: query,
+    };
+
+    // Search in global memories
+    if (!options.scope || options.scope === "global") {
+      const globalIndex = this._getIndexName(MemoryScope.GLOBAL);
+      console.log("Searching in global index:", globalIndex);
+      try {
+        const globalResults = await this._makeRequest<MeilisearchResponse>(
+          `/indexes/${globalIndex}/search`,
+          {
+            method: "POST",
+            body: JSON.stringify(searchBody),
+          }
+        );
+        if (globalResults?.hits) {
+          searchResults.push(...globalResults.hits);
+        }
+      } catch (error) {
+        console.error("Error searching global index:", error);
+      }
+    }
+
+    // Search in user memories
+    if (
+      options.userId &&
+      (!options.scope || options.scope === MemoryScope.USER)
+    ) {
+      const userIndex = this._getIndexName(MemoryScope.USER, options.userId);
+      const userResults = await this._makeRequest<MeilisearchResponse>(
+        `/indexes/${userIndex}/search`,
+        {
+          method: "POST",
+          body: JSON.stringify(searchBody),
+        }
+      );
+      if (userResults.hits) {
+        searchResults.push(...userResults.hits);
+      }
+    }
+
+    console.log("Found in persistent memory:", searchResults);
+
+    // Process and filter results using cosine similarity
+    const results = searchResults
+      .flatMap((hit) => {
+        // Calculate similarities for each chunk
+        const chunkSimilarities = hit.chunks.map((chunk) => ({
+          data: hit.data,
+          purpose: hit.purpose,
+          query: hit.query,
+          chunk: chunk.content,
+          similarityPercentage:
+            (cosineSimilarity(queryEmbedding, chunk.embedding) + 1) * 50,
+        }));
+
+        // Return the chunk with highest similarity
+        return chunkSimilarities.reduce(
+          (best, current) =>
+            current.similarityPercentage > best.similarityPercentage
+              ? current
+              : best,
+          chunkSimilarities[0]
+        );
+      })
+      .filter(
+        (match) =>
+          match.similarityPercentage >= (options.similarityThreshold || 70)
+      )
+      .sort((a, b) => b.similarityPercentage - a.similarityPercentage);
+
+    // Log results
+    if (results.length > 0) {
+      console.log("\n✨ Best matches found:");
+      results.forEach((match) => {
+        console.log(
+          `- ${match.query} : ${match.similarityPercentage.toFixed(2)}% (${
+            match.purpose
+          })`
+        );
+        console.log(`  Matching content: "${match.chunk}"`);
+      });
+    } else {
+      console.log("No matches found");
+    }
+
+    return results;
+  }
+
+  /**
+   * Delete memories for a given scope and user
+   */
+  async deleteMemories(scope: MemoryScope, userId?: string) {
+    const indexName = this._getIndexName(scope, userId);
+    return this._makeRequest(`/indexes/${indexName}`, {
+      method: "DELETE",
+    });
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai.ntellect/core",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "",
   "main": "dist/index.js",
   "scripts": {

package/test.ts

@@ -127,7 +127,7 @@ export const getRssNews = {
     host: "http://localhost:7700",
     apiKey: "aSampleMasterKey",
   });
-  const orchestrator = new Orchestrator([getRssNews, getChainsTVL], memory);
+  const orchestrator = new Orchestrator([], memory);
   const agent = new Agent({
     user: {
       id: "1",
@@ -138,9 +138,8 @@ export const getRssNews = {
     maxEvaluatorIteration: 1,
   });
 
-  const prompt = "tu connais avax";
+  const prompt = "fais moi une analyse ethereum";
   const context = prompt;
-
   const result = await agent.process(prompt, context, {
     onMessage: (message) => {
       console.log({ message });

package/memory/persistent.d.ts

@@ -1,28 +0,0 @@
-import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
-import { Index, MeiliSearch } from "meilisearch";
-import {
-  MatchOptions,
-  MatchResult,
-  Memory,
-  MemoryChunk,
-  MemoryScopeType,
-} from "../types";
-
-export class PersistentMemory {
-  private client: MeiliSearch;
-  private readonly INDEX_PREFIX: string;
-  private textSplitter: RecursiveCharacterTextSplitter;
-
-  constructor(options: PersistentMemoryOptions);
-
-  private _getIndexName(scope: MemoryScopeType, userId?: string): string;
-  private _getOrCreateIndex(indexName: string): Promise<Index>;
-  private _processContent(content: string): Promise<MemoryChunk[]>;
-
-  storeMemory(memory: Memory): Promise<void>;
-  findBestMatches(
-    query: string,
-    options?: MatchOptions
-  ): Promise<MatchResult[]>;
-  deleteMemories(scope: MemoryScopeType, userId?: string): Promise<void>;
-}

package/memory/persistent.js

@@ -1,198 +0,0 @@
-import { openai } from "@ai-sdk/openai";
-import { cosineSimilarity, embed, embedMany } from "ai";
-import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
-import { MeiliSearch } from "meilisearch";
-
-const MemoryScope = {
-  GLOBAL: "global",
-  USER: "user",
-};
-
-export class PersistentMemory {
-  /**
-   * @param {Object} options
-   * @param {string} options.host - MeiliSearch host URL
-   * @param {string} options.apiKey - MeiliSearch API key
-   * @param {string} [options.indexPrefix="memory_"] - Prefix for index names
-   */
-  constructor(options) {
-    this.client = new MeiliSearch({
-      host: options.host,
-      apiKey: options.apiKey,
-    });
-    this.INDEX_PREFIX = options.indexPrefix || "memory_";
-    this.textSplitter = new RecursiveCharacterTextSplitter({
-      chunkSize: 1000,
-    });
-  }
-
-  /**
-   * Get the index name based on scope and userId
-   * @private
-   */
-  _getIndexName(scope, userId) {
-    if (scope === MemoryScope.GLOBAL) {
-      return `${this.INDEX_PREFIX}global`;
-    }
-    return `${this.INDEX_PREFIX}user_${userId}`;
-  }
-
-  /**
-   * Get or create an index with proper settings
-   * @private
-   */
-  async _getOrCreateIndex(indexName) {
-    const index = this.client.index(indexName);
-
-    try {
-      await this.client.createIndex(indexName, { primaryKey: "id" });
-      await index.updateSettings({
-        searchableAttributes: ["query", "purpose", "chunks.content"],
-        sortableAttributes: ["createdAt"],
-      });
-    } catch (error) {
-      // Index might already exist, which is fine
-      if (!error.message.includes("already exists")) {
-        throw error;
-      }
-    }
-
-    return index;
-  }
-
-  /**
-   * Process content into chunks with embeddings
-   * @private
-   */
-  async _processContent(content) {
-    // Split content into chunks
-    const chunks = await this.textSplitter.createDocuments([content]);
-
-    // Generate embeddings for all chunks
-    const { embeddings } = await embedMany({
-      model: openai.embedding("text-embedding-3-small"),
-      values: chunks.map((chunk) => chunk.pageContent),
-    });
-
-    // Create processed chunks with embeddings
-    return chunks.map((chunk, i) => ({
-      content: chunk.pageContent,
-      embedding: embeddings[i],
-    }));
-  }
-
-  /**
-   * Store a memory in the database
-   * @param {Object} memory - Memory object to store
-   */
-  async storeMemory(memory) {
-    const indexName = this._getIndexName(memory.scope, memory.userId);
-    const index = await this._getOrCreateIndex(indexName);
-
-    // Process the query into chunks with embeddings
-    const chunks = await this._processContent(memory.query);
-
-    const result = await index.addDocuments([
-      {
-        ...memory,
-        chunks,
-        createdAt: memory.createdAt.toISOString(),
-      },
-    ]);
-    return result;
-  }
-
-  /**
-   * Find best matching memories using cosine similarity
-   * @param {string} query - Search query
-   * @param {Object} options - Search options
-   */
-  async findBestMatches(query, options = {}) {
-    console.log("\n🔍 Searching in persistent memory:", query);
-
-    // Generate embedding for the query
-    const { embedding: queryEmbedding } = await embed({
-      model: openai.embedding("text-embedding-3-small"),
-      value: query,
-    });
-    const searchResults = [];
-
-    // Search in global memories
-    if (!options.scope || options.scope === MemoryScope.GLOBAL) {
-      const globalIndex = await this._getOrCreateIndex(
-        this._getIndexName(MemoryScope.GLOBAL)
-      );
-      const globalResults = await globalIndex.search(query, {
-        limit: options.maxResults || 10,
-      });
-      searchResults.push(...globalResults.hits);
-    }
-
-    // Search in user memories
-    if (
-      options.userId &&
-      (!options.scope || options.scope === MemoryScope.USER)
-    ) {
-      const userIndex = await this._getOrCreateIndex(
-        this._getIndexName(MemoryScope.USER, options.userId)
-      );
-      const userResults = await userIndex.search(query, {
-        limit: options.maxResults || 10,
-      });
-      searchResults.push(...userResults.hits);
-    }
-
-    // Process and filter results using cosine similarity
-    const results = searchResults
-      .flatMap((hit) => {
-        // Calculate similarities for each chunk
-        console.log(hit);
-        const chunkSimilarities = hit.chunks.map((chunk) => ({
-          data: hit.data,
-          purpose: hit.purpose,
-          chunk: chunk.content,
-          similarityPercentage:
-            (cosineSimilarity(queryEmbedding, chunk.embedding) + 1) * 50,
-        }));
-        console.log({ chunkSimilarities });
-        // Return the chunk with highest similarity
-        return chunkSimilarities.reduce(
-          (best, current) =>
-            current.similarityPercentage > best.similarityPercentage
-              ? current
-              : best,
-          chunkSimilarities[0]
-        );
-      })
-      .filter(
-        (match) =>
-          match.similarityPercentage >= (options.similarityThreshold || 70)
-      )
-      .sort((a, b) => b.similarityPercentage - a.similarityPercentage);
-
-    // Log results
-    if (results.length > 0) {
-      console.log("\n✨ Best matches found:");
-      results.forEach((match) => {
-        console.log(
-          `- ${match.purpose} (${match.similarityPercentage.toFixed(2)}%)`
-        );
-        console.log(`  Matching chunk: "${match.chunk}"`);
-      });
-    } else {
-      console.log("No matches found");
-    }
-
-    return results;
-  }
-
-  /**
-   * Delete memories for a given scope and user
-   * @param {string} scope - Memory scope
-   * @param {string} [userId] - User ID for user-specific memories
-   */
-  async deleteMemories(scope, userId) {
-    const indexName = this._getIndexName(scope, userId);
-    await this.client.deleteIndex(indexName);
-  }
-}