@ai.ntellect/core 0.3.1 → 0.4.0

package/memory/cache.ts

@@ -1,5 +1,5 @@
 import { openai } from "@ai-sdk/openai";
-import { cosineSimilarity, embed } from "ai";
+import { cosineSimilarity, embed, EmbeddingModel } from "ai";
 import { createClient } from "redis";
 import {
   CacheMemoryOptions,
@@ -7,16 +7,16 @@ import {
   CreateMemoryInput,
   MatchOptions,
   MemoryScope,
-  MemoryType,
-  QueueResult,
 } from "../types";
 
 export class CacheMemory {
   private redis;
   private readonly CACHE_PREFIX: string;
   private readonly CACHE_TTL: number;
+  private readonly embeddingModel: EmbeddingModel<string>;
 
-  constructor(options: CacheMemoryOptions = {}) {
+  constructor(options: CacheMemoryOptions) {
+    this.embeddingModel = options.embeddingModel;
     const ttlInHours = options.cacheTTL ?? 1;
     this.CACHE_TTL = ttlInHours * 60 * 60;
     this.CACHE_PREFIX = options.cachePrefix ?? "memory:";
@@ -44,18 +44,11 @@ export class CacheMemory {
     }
   }
 
-  private getMemoryKey(scope: MemoryScope, userId?: string): string {
-    if (scope === MemoryScope.GLOBAL) {
-      return `${this.CACHE_PREFIX}global:`;
-    }
-    return `${this.CACHE_PREFIX}user:${userId}:`;
-  }
-
-  private async storeMemory(memory: CacheMemoryType) {
-    const prefix = this.getMemoryKey(memory.scope, memory.userId);
+  private async storeMemory(memory: CacheMemoryType, ttl?: number) {
+    const prefix = this.CACHE_PREFIX;
     const key = `${prefix}${memory.id}`;
     const result = await this.redis.set(key, JSON.stringify(memory), {
-      EX: this.CACHE_TTL,
+      EX: ttl || this.CACHE_TTL,
     });
     console.log("💾 Cache memory created:", result);
   }
@@ -65,9 +58,9 @@ export class CacheMemory {
     options: MatchOptions & { userId?: string; scope?: MemoryScope } = {}
   ): Promise<
     {
-      actions: QueueResult[];
-      similarityPercentage: number;
+      data: any;
       query: string;
+      createdAt: Date;
     }[]
   > {
     console.log("\n🔍 Searching in cache");
@@ -79,7 +72,7 @@ export class CacheMemory {
       value: query,
     });
 
-    const memories = await this.getAllMemories(options.scope, options.userId);
+    const memories = await this.getAllMemories();
     console.log(`\n📚 Found ${memories.length} cached queries to compare`);
 
     const matches = memories
@@ -87,7 +80,7 @@ export class CacheMemory {
         const similarity = cosineSimilarity(embedding, memory.embedding);
         const similarityPercentage = (similarity + 1) * 50;
         return {
-          actions: memory.data,
+          data: memory.data,
           query: memory.query,
           similarityPercentage,
           createdAt: memory.createdAt,
@@ -110,6 +103,7 @@ export class CacheMemory {
       results.forEach((match, index) => {
         console.log(`\n${index + 1}. Match Details:`);
         console.log(`   Query: ${match.query}`);
+        console.log(`   Data: ${JSON.stringify(match.data)}`);
         console.log(`   Similarity: ${match.similarityPercentage.toFixed(2)}%`);
         console.log("─".repeat(50));
       });
@@ -117,30 +111,20 @@ export class CacheMemory {
       console.log("\n❌ No similar queries found in cache");
     }
 
-    return results;
+    return results.map((match) => {
+      return {
+        data: match.data,
+        query: match.query,
+        createdAt: match.createdAt,
+      };
+    });
   }
 
-  async getAllMemories(
-    scope?: MemoryScope,
-    userId?: string
-  ): Promise<CacheMemoryType[]> {
-    let patterns: CacheMemoryType[] = [];
-
-    if (!scope || scope === MemoryScope.GLOBAL) {
-      const globalPrefix = this.getMemoryKey(MemoryScope.GLOBAL);
-      const globalKeys = await this.redis.keys(`${globalPrefix}*`);
-      const globalPatterns = await this.getMemoriesFromKeys(globalKeys);
-      patterns = patterns.concat(globalPatterns);
-    }
+  async getAllMemories(): Promise<CacheMemoryType[]> {
+    const keys = await this.redis.keys(`${this.CACHE_PREFIX}*`);
+    const memories = await this.getMemoriesFromKeys(keys);
 
-    if (userId && (!scope || scope === MemoryScope.USER)) {
-      const userPrefix = this.getMemoryKey(MemoryScope.USER, userId);
-      const userKeys = await this.redis.keys(`${userPrefix}*`);
-      const userPatterns = await this.getMemoriesFromKeys(userKeys);
-      patterns = patterns.concat(userPatterns);
-    }
-
-    return patterns;
+    return memories;
   }
 
   private async getMemoriesFromKeys(
@@ -160,11 +144,10 @@ export class CacheMemory {
     input: CreateMemoryInput
   ): Promise<CacheMemoryType | undefined> {
     console.log("\n📝 Processing new memory creation");
-    console.log("Content:", input.content);
-    console.log("Type:", input.type);
-    console.log("Scope:", input.scope);
+    console.log("Content:", input.query);
+    console.log("TTL:", input.ttl ? `${input.ttl} seconds` : "default");
 
-    const existingPattern = await this.findSimilarActions(input.content, {
+    const existingPattern = await this.findSimilarActions(input.query, {
       similarityThreshold: 95,
       userId: input.userId,
       scope: input.scope,
@@ -176,7 +159,8 @@ export class CacheMemory {
       existingPattern.forEach((match, index) => {
         console.log(`\n${index + 1}. Existing Match:`);
         console.log(`   Query: ${match.query}`);
-        console.log(`   Similarity: ${match.similarityPercentage.toFixed(2)}%`);
+        console.log(`   Data: ${JSON.stringify(match.data)}`);
+        console.log(`   Created At: ${match.createdAt}`);
       });
       console.log("\n⏭️  Skipping creation of new memory");
       return;
@@ -186,11 +170,11 @@ export class CacheMemory {
 
     const memory = await this.createSingleMemory({
       id: crypto.randomUUID(),
-      content: input.content,
-      type: input.type,
+      query: input.query,
       data: input.data,
       userId: input.userId,
       scope: input.scope,
+      ttl: input.ttl,
     });
 
     return memory;
@@ -198,28 +182,27 @@ export class CacheMemory {
 
   private async createSingleMemory(params: {
     id: string;
-    content: string;
-    type: MemoryType;
+    query: string;
     data: any;
     userId?: string;
     scope?: MemoryScope;
+    ttl?: number;
   }): Promise<CacheMemoryType> {
     console.log("\n🏗️  Creating new cache memory");
     console.log("ID:", params.id);
-    console.log("Content:", params.content);
+    console.log("Content:", params.query);
 
     console.log("\n🧮 Generating embedding...");
     const { embedding } = await embed({
-      model: openai.embedding("text-embedding-3-small"),
-      value: params.content,
+      model: this.embeddingModel,
+      value: params.query,
     });
     console.log("✅ Embedding generated successfully");
 
     const memory: CacheMemoryType = {
       id: params.id,
-      type: params.type,
       data: params.data,
-      query: params.content,
+      query: params.query,
       embedding,
       userId: params.userId,
       scope:
@@ -227,8 +210,11 @@ export class CacheMemory {
       createdAt: new Date(),
     };
 
-    await this.storeMemory(memory);
-    console.log("✅ Memory created and stored successfully");
+    await this.storeMemory(memory, params.ttl);
+    console.log("✅ Short-term memory created and stored successfully", {
+      ...memory,
+      ttl: params.ttl || this.CACHE_TTL,
+    });
 
     return memory;
   }

package/memory/persistent.ts

@@ -1,7 +1,6 @@
-import { openai } from "@ai-sdk/openai";
-import { cosineSimilarity, embed, embedMany } from "ai";
+import { cosineSimilarity, embed, EmbeddingModel, embedMany } from "ai";
 import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
-import { Memory, MemoryScope } from "../types";
+import { LongTermMemory, MemoryScope } from "../types";
 
 interface SearchOptions {
   scope?: MemoryScope;
@@ -28,18 +27,6 @@ interface MeilisearchResponse {
   }>;
 }
 
-interface SearchParams {
-  q?: string;
-  offset?: number;
-  limit?: number;
-  filter?: string | string[];
-  facets?: string[];
-  attributesToRetrieve?: string[];
-  attributesToSearchOn?: string[];
-  sort?: string[];
-  matchingStrategy?: "last" | "all" | "frequency";
-}
-
 interface ProcessedChunk {
   content: string;
   embedding: number[];
@@ -52,22 +39,31 @@ export class PersistentMemory {
   private readonly host: string;
   private readonly apiKey: string;
   private readonly INDEX_PREFIX: string;
-
-  constructor(options: { host: string; apiKey: string; indexPrefix?: string }) {
+  private readonly embeddingModel: EmbeddingModel<string>;
+  constructor(options: {
+    host: string;
+    apiKey: string;
+    indexPrefix?: string;
+    embeddingModel: EmbeddingModel<string>;
+  }) {
     this.host = options.host;
     this.apiKey = options.apiKey;
-    this.INDEX_PREFIX = options.indexPrefix || "memory_";
+    this.INDEX_PREFIX = options.indexPrefix || "memory";
+    this.embeddingModel = options.embeddingModel;
   }
 
   /**
    * Initialize indexes
    */
   async init() {
-    // Create global index
-    await this._getOrCreateIndex(this._getIndexName(MemoryScope.GLOBAL));
-
-    // Create user index
-    await this._getOrCreateIndex(this._getIndexName(MemoryScope.USER));
+    try {
+      // Create or get main index
+      await this._getOrCreateIndex(this.INDEX_PREFIX);
+      console.log(`✅ Index '${this.INDEX_PREFIX}' initialized successfully`);
+    } catch (error) {
+      console.error(`❌ Failed to initialize index: ${error}`);
+      throw error;
+    }
   }
 
   /**
@@ -78,6 +74,7 @@ export class PersistentMemory {
     options: RequestInit = {}
   ): Promise<T> {
     const url = `${this.host}${path}`;
+    console.log("🔍 Making request to Meilisearch:", url);
     const response = await fetch(url, {
       ...options,
       headers: {
@@ -95,29 +92,26 @@ export class PersistentMemory {
     return response.json() as Promise<T>;
   }
 
-  /**
-   * Get index name based on scope and userId
-   */
-  private _getIndexName(scope: MemoryScope, userId?: string): string {
-    if (scope === "global") {
-      return `${this.INDEX_PREFIX}global`;
-    }
-    return `${this.INDEX_PREFIX}user_${userId}`;
-  }
-
   /**
    * Get or create an index with proper settings
    */
   private async _getOrCreateIndex(indexName: string) {
     try {
-      // Try to create index
-      await this._makeRequest("/indexes", {
-        method: "POST",
-        body: JSON.stringify({
-          uid: indexName,
-          primaryKey: "id",
-        }),
-      });
+      // Check if index exists first
+      const indexExists = await this._makeRequest(`/indexes/${indexName}`, {
+        method: "GET",
+      }).catch(() => false);
+
+      if (!indexExists) {
+        console.log(`Creating new index: ${indexName}`);
+        await this._makeRequest("/indexes", {
+          method: "POST",
+          body: JSON.stringify({
+            uid: indexName,
+            primaryKey: "id",
+          }),
+        });
+      }
 
       // Update index settings
       const settings: MeilisearchSettings = {
@@ -129,11 +123,11 @@ export class PersistentMemory {
         method: "PATCH",
         body: JSON.stringify(settings),
       });
+
+      console.log(`Index ${indexName} configured successfully`);
     } catch (error: any) {
-      // Index might already exist, which is fine
-      if (!error.message.includes("already exists")) {
-        throw error;
-      }
+      console.error(`Failed to configure index ${indexName}:`, error);
+      throw error;
     }
   }
 
@@ -146,7 +140,7 @@ export class PersistentMemory {
 
     // Generate embeddings for all chunks
     const { embeddings } = await embedMany({
-      model: openai.embedding("text-embedding-3-small"),
+      model: this.embeddingModel,
       values: chunks.map((chunk) => chunk.pageContent),
     });
 
@@ -160,133 +154,111 @@ export class PersistentMemory {
   /**
    * Store a memory in the database
    */
-  async createMemory(memory: Memory) {
-    const indexName = this._getIndexName(memory.scope, memory.userId);
-    await this._getOrCreateIndex(indexName);
+  async createMemory(memory: LongTermMemory) {
+    try {
+      console.log(`📝 Creating memory in index: ${this.INDEX_PREFIX}`);
 
-    const chunks = await this.processContent(memory.data);
+      // Process content into chunks with embeddings
+      const chunks = await this.processContent(memory.data);
 
-    const document = {
-      ...memory,
-      chunks,
-      createdAt: memory.createdAt.toISOString(),
-    };
+      // Generate unique ID if not provided
+      const id = memory.id || crypto.randomUUID();
 
-    const response = await this._makeRequest(
-      `/indexes/${indexName}/documents`,
-      {
-        method: "POST",
-        body: JSON.stringify([document]),
-      }
-    );
-    console.log("Stored persistent memory response:", response);
-    return response;
+      const document = {
+        ...memory,
+        chunks,
+        createdAt: memory.createdAt.toISOString(),
+      };
+
+      const response = await this._makeRequest(
+        `/indexes/${this.INDEX_PREFIX}/documents`,
+        {
+          method: "POST",
+          body: JSON.stringify([document]),
+        }
+      );
+
+      console.log("✅ Memory created successfully", { id });
+      return response;
+    } catch (error) {
+      console.error("❌ Failed to create memory:", error);
+      throw error;
+    }
   }
 
   /**
    * Find best matching memories
    */
   async findRelevantDocuments(query: string, options: SearchOptions = {}) {
-    console.log("\n🔍 Searching in persistent memory");
+    console.log(`\n🔍 Searching in index: ${this.INDEX_PREFIX}`);
     console.log("Query:", query);
-    console.log("Options:", JSON.stringify(options, null, 2));
 
-    // Generate embedding for the query
-    const { embedding: queryEmbedding } = await embed({
-      model: openai.embedding("text-embedding-3-small"),
-      value: query,
-    });
-
-    const searchResults = [];
-
-    // Search in global memories
-    if (!options.scope || options.scope === "global") {
-      const globalIndex = this._getIndexName(MemoryScope.GLOBAL);
-      console.log("\n📚 Searching in global index:", globalIndex);
-      try {
-        const globalResults = await this._makeRequest<MeilisearchResponse>(
-          `/indexes/${globalIndex}/search`,
-          {
-            method: "POST",
-            body: JSON.stringify({ q: query }),
-          }
-        );
-        if (globalResults?.hits) {
-          searchResults.push(...globalResults.hits);
-        }
-      } catch (error) {
-        console.error("❌ Error searching global index:", error);
-      }
-    }
+    try {
+      // Generate embedding for the query
+      const { embedding: queryEmbedding } = await embed({
+        model: this.embeddingModel,
+        value: query,
+      });
 
-    // Search in user memories
-    if (
-      options.userId &&
-      (!options.scope || options.scope === MemoryScope.USER)
-    ) {
-      const userIndex = this._getIndexName(MemoryScope.USER, options.userId);
-      const userResults = await this._makeRequest<MeilisearchResponse>(
-        `/indexes/${userIndex}/search`,
+      // Search in the index
+      const searchResults = await this._makeRequest<MeilisearchResponse>(
+        `/indexes/${this.INDEX_PREFIX}/search`,
         {
           method: "POST",
-          body: JSON.stringify({ q: query }),
+          body: JSON.stringify({
+            q: query,
+            limit: options.maxResults || 10,
+          }),
         }
       );
-      if (userResults.hits) {
-        searchResults.push(...userResults.hits);
-      }
-    }
-
-    const totalResults = searchResults.length;
-    console.log(`\n📊 Found ${totalResults} total matches`);
 
-    // Process and filter results using cosine similarity
-    const results = searchResults
-      .flatMap((hit) => {
-        const chunkSimilarities = hit.chunks.map((chunk) => ({
-          query: hit.query,
-          data: hit.data,
-          similarityPercentage:
-            (cosineSimilarity(queryEmbedding, chunk.embedding) + 1) * 50,
-          createdAt: hit.createdAt,
-        }));
-
-        return chunkSimilarities.reduce(
-          (best, current) =>
-            current.similarityPercentage > best.similarityPercentage
-              ? current
-              : best,
-          chunkSimilarities[0]
-        );
-      })
-      .filter(
-        (match) =>
-          match.similarityPercentage >= (options.similarityThreshold || 70)
-      )
-      .sort((a, b) => b.similarityPercentage - a.similarityPercentage);
-
-    // Log filtered results in a more structured way
-    if (results.length > 0) {
-      console.log("\n✨ Relevant matches found:");
-      console.log("─".repeat(50));
+      if (!searchResults?.hits?.length) {
+        console.log("❌ No matches found");
+        return [];
+      }
 
-      results.forEach((match, index) => {
-        console.log(`\n${index + 1}. Match Details:`);
-        console.log(`   Query: ${match.query}`);
-      });
-    } else {
-      console.log("\n❌ No relevant matches found");
+      // Process and filter results using cosine similarity
+      const results = searchResults.hits
+        .flatMap((hit) => {
+          const chunkSimilarities = hit.chunks.map((chunk) => ({
+            query: hit.query,
+            data: hit.data,
+            similarityPercentage:
+              (cosineSimilarity(queryEmbedding, chunk.embedding) + 1) * 50,
+            createdAt: hit.createdAt,
+          }));
+
+          return chunkSimilarities.reduce(
+            (best, current) =>
+              current.similarityPercentage > best.similarityPercentage
+                ? current
+                : best,
+            chunkSimilarities[0]
+          );
+        })
+        .filter(
+          (match) =>
+            match.similarityPercentage >= (options.similarityThreshold || 70)
+        )
+        .sort((a, b) => b.similarityPercentage - a.similarityPercentage);
+
+      console.log(`✨ Found ${results.length} relevant matches`);
+      return results.map((result) => ({
+        query: result.query,
+        data: result.data,
+        createdAt: result.createdAt,
+      }));
+    } catch (error) {
+      console.error("❌ Search failed:", error);
+      return [];
     }
-
-    return results;
   }
 
   /**
    * Delete memories for a given scope and user
    */
-  async deleteMemories(scope: MemoryScope, userId?: string) {
-    const indexName = this._getIndexName(scope, userId);
-    return this._makeRequest(`/indexes/${indexName}`, {
+  async deleteMemories() {
+    return this._makeRequest(`/indexes/${this.INDEX_PREFIX}`, {
       method: "DELETE",
     });
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai.ntellect/core",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "",
   "main": "dist/index.js",
   "scripts": {
@@ -12,18 +12,24 @@
   "author": "Lorcann Rauzduel",
   "license": "ISC",
   "dependencies": {
+    "@ai-sdk/deepseek": "^0.1.0",
     "@ai-sdk/openai": "1.0.6",
+    "@types/node-cron": "^3.0.11",
     "ai": "^3.0.0",
     "ethers": "^6.13.5",
     "langchain": "^0.3.11",
+    "node-cron": "^3.0.3",
+    "readline": "^1.3.0",
     "redis": "^4.7.0",
     "rss-parser": "^3.13.0",
+    "ws": "^8.18.0",
     "zod": "^3.24.1"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
     "@types/chai": "^4.3.20",
     "@types/mocha": "^10.0.0",
+    "@types/ws": "^8.5.14",
     "chai": "^4.5.0",
     "mocha": "^10.0.0",
     "ts-node": "^10.9.0",
@@ -37,5 +43,8 @@
   "bugs": {
     "url": "https://github.com/ai-ntellect/core/issues"
   },
-  "homepage": "https://github.com/ai-ntellect/core#readme"
+  "homepage": "https://github.com/ai-ntellect/core#readme",
+  "bin": {
+    "wallet-assistant": "./dist/examples/index.js"
+  }
 }