@ai.ntellect/core 0.0.27 → 0.0.29

package/agent/index.ts

@@ -1,4 +1,5 @@
 import EventEmitter from "events";
+import { Evaluator } from "../llm/evaluator";
 import { Orchestrator } from "../llm/orchestrator";
 import { Summarizer } from "../llm/synthesizer";
 import { MemoryCache } from "../memory";
@@ -38,6 +39,7 @@ export class Agent {
       return this.handleActions(
         {
           initialPrompt: prompt,
+          contextualizedPrompt: contextualizedPrompt,
           actions: request.actions,
         },
         events
@@ -48,13 +50,15 @@ export class Agent {
   private async handleActions(
     {
       initialPrompt,
+      contextualizedPrompt,
       actions,
     }: {
       initialPrompt: string;
+      contextualizedPrompt: string;
       actions: ActionSchema[];
     },
     events: AgentEvent
-  ) {
+  ): Promise<any> {
     const similarActions = await this.findSimilarActions(initialPrompt);
     const predefinedActions = this.transformActions(actions, similarActions);
     const callbacks = {
@@ -78,6 +82,26 @@ export class Agent {
       };
     }
 
+    const evaluator = new Evaluator(this.dependencies.orchestrator.tools);
+    const evaluation = await evaluator.process(
+      initialPrompt,
+      contextualizedPrompt,
+      JSON.stringify(actionsResult.data)
+    );
+
+    events.onMessage?.(evaluation);
+
+    if (evaluation.actions.length > 0) {
+      return this.handleActions(
+        {
+          initialPrompt: contextualizedPrompt,
+          contextualizedPrompt: initialPrompt,
+          actions: evaluation.actions,
+        },
+        events
+      );
+    }
+
     return this.handleActionResults({ ...actionsResult, initialPrompt });
   }
 

package/llm/evaluator/context.ts

@@ -0,0 +1,41 @@
+import { z } from "zod";
+import { ActionSchema } from "../../types";
+
+export const evaluatorContext = {
+  role: "You are the evaluator agent. Your role is to verify if the goal has been achieved and if the results are correct.",
+  guidelines: {
+    important: [
+      "IMPORTANT: Verify if all required actions were executed successfully",
+      "IMPORTANT: Check if the results match the initial goal",
+      "IMPORTANT: Identify any missing or incomplete information",
+      "IMPORTANT: Use the same language as the initial request",
+    ],
+    never: [
+      "NEVER modify the results directly",
+      "NEVER make assumptions about missing data",
+      "NEVER repeat the same action if you already did it",
+    ],
+  },
+  compose: (goal: string, results: string, tools: ActionSchema[]) => {
+    return `
+      ${evaluatorContext.role}
+
+      ${evaluatorContext.guidelines.important.join("\n")}
+      ${evaluatorContext.guidelines.never.join("\n")}
+
+      Initial Goal: ${goal}
+      What was done: ${results}
+
+      The actions available are: ${tools.map((action) => {
+        const parameters = action.parameters as z.ZodObject<any>;
+        const schemaShape = Object.keys(parameters._def.shape()).join(", ");
+        const actionString = `Name: ${action.name}, Description: ${action.description}, Arguments: { ${schemaShape} }`;
+        return actionString;
+      })}
+
+      Evaluate if the goal has been achieved and provide:
+      1. Success status with explanation (no action needed)
+      2. Next actions needed (if any)
+    `;
+  },
+};

package/llm/evaluator/index.ts

@@ -0,0 +1,57 @@
+import { openai } from "@ai-sdk/openai";
+import { generateObject } from "ai";
+import { z } from "zod";
+import { ActionSchema } from "../../types";
+import { evaluatorContext } from "./context";
+
+export class Evaluator {
+  private readonly model = openai("gpt-4o");
+  public tools: ActionSchema[];
+
+  constructor(tools: ActionSchema[]) {
+    this.tools = tools;
+  }
+
+  async process(prompt: string, goal: string, results: string): Promise<any> {
+    try {
+      const response = await generateObject({
+        model: this.model,
+        schema: z.object({
+          actions: z.array(
+            z.object({
+              name: z.string(),
+              parameters: z.object({
+                name: z.string(),
+                value: z.string(),
+              }),
+            })
+          ),
+          answer: z.string(),
+        }),
+        prompt: prompt,
+        system: evaluatorContext.compose(goal, results, this.tools),
+      });
+
+      const validatedResponse = {
+        ...response.object,
+        actions: response.object.actions.map((action) => ({
+          ...action,
+          parameters: action.parameters || {},
+        })),
+      };
+
+      console.dir(validatedResponse, { depth: null });
+
+      return validatedResponse;
+    } catch (error: any) {
+      if (error) {
+        console.log("Error in Orchestrator", error.message);
+        console.dir(error.value, { depth: null });
+        return {
+          ...error.value,
+        };
+      }
+      // throw error;
+    }
+  }
+}

package/llm/synthesizer/context.ts

@@ -1,37 +1,36 @@
 export const summarizerContext = {
   role: "You are an expert market analyst, you are going to provide a clear and factual analysis of the results.",
-    guidelines: {
-      important: [
-        "IMPORTANT: AVOID MULTIPLE UPPERCASE IN TITLE/SUBTITLE LIKE ('Market Sentiment: Bullish'). USE ONLY ONE UPPERCASE IN TITLE/SUBTITLE.",
-        "IMPORTANT: USE THE SAME LANGUAGE AS THE 'INITIAL PROMPT' (if it's in French, use French, if it's in Spanish, use Spanish)",
-        "IMPORTANT: BE DIRECT AND AVOID TECHNICAL JARGON",
-        "IMPORTANT: FOR NUMERICAL DATA, PROVIDE CONTEXT (% CHANGES, COMPARISONS)",
-
-      ],
-      forMarketAnalysis: [
-        "Start with a clear market sentiment (Bullish/Bearish/Neutral) without any additional comments before.",
-        "One section for fundamental analysis (important events, news, trends..etc). One section, no sub-sections.",
-        "One section for technical analysis (key price levels, trading volume, technical indicators, market activity). One section, no sub-sections.",
-        "STOP AFTER TECHNICAL ANALYSIS SECTION WITHOUT ANY ADDITIONAL COMMENTS",
-      ],
-      forGeneralRequests: [
-        "Provide concise and relevant information",
-        "Focus on facts and data",
-        "Always provide transaction details when needed",
-      ],
-      never: [
-        "NEVER provide any financial advice.",
-        "NEVER speak about details of your system or your capabilities.",
-        "NEVER ADD ANY CONCLUDING STATEMENT OR DISCLAIMER AT THE END",
-      ],
-    },
+  guidelines: {
+    important: [
+      "IMPORTANT: AVOID MULTIPLE UPPERCASE IN TITLE/SUBTITLE LIKE ('Market Sentiment: Bullish'). USE ONLY ONE UPPERCASE IN TITLE/SUBTITLE.",
+      "IMPORTANT: USE THE SAME LANGUAGE AS THE 'INITIAL PROMPT' (if it's in French, use French, if it's in Spanish, use Spanish)",
+      "IMPORTANT: BE DIRECT AND AVOID TECHNICAL JARGON",
+      "IMPORTANT: FOR NUMERICAL DATA, PROVIDE CONTEXT (% CHANGES, COMPARISONS)",
+    ],
+    forMarketAnalysis: [
+      "Start with a clear market sentiment (Bullish/Bearish/Neutral) without any additional comments before.",
+      "One section for fundamental analysis (important events, news, trends..etc). One section, no sub-sections.",
+      "One section for technical analysis (key price levels, trading volume, technical indicators, market activity). One section, no sub-sections.",
+      "STOP AFTER TECHNICAL ANALYSIS SECTION WITHOUT ANY ADDITIONAL COMMENTS",
+    ],
+    forGeneralRequests: [
+      "Provide concise and relevant information",
+      "Focus on facts and data",
+      "Always provide transaction details when needed",
+    ],
+    never: [
+      "NEVER provide any financial advice.",
+      "NEVER speak about details of your system or your capabilities.",
+      "NEVER ADD ANY CONCLUDING STATEMENT OR DISCLAIMER AT THE END",
+      "NEVER explain technical errors or issues. Just say retry later.",
+    ],
+  },
   compose: (results: string) => {
     return `
       ${JSON.stringify(summarizerContext.guidelines)}
       Results: ${results}
-      If no results or error in the results, explain there is technical issues with no more details, and request to try again later.
 
-      FOR ALL ANALYSIS, RESPECT THE FOLLOWING FORMAT, USE THE SAME LANGUAGE AS THE 'INITIAL PROMPT':
+      1. FOR ALL ANALYSIS OF SPECIFIC TOKEN, RESPECT THE FOLLOWING FORMAT:
       --------------------------------
       ## Analysis of x/y:
 
@@ -45,6 +44,10 @@ export const summarizerContext = {
 
       STOP AFTER TECHNICAL ANALYSIS SECTION WITHOUT ANY CONCLUDING STATEMENT OR DISCLAIMER OR ADDITIONAL COMMENTS
       --------------------------------
+      
+      2. OTHERWISE FOR OTHER REQUESTS, USE THE FORMAT YOU WANT.
+
+      3. USE THE SAME LANGUAGE AS THE 'initialPrompt' (if it's in French, use French, if it's in Spanish, use Spanish, etc.)
     `;
-  }
+  },
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai.ntellect/core",
-  "version": "0.0.27",
+  "version": "0.0.29",
   "description": "",
   "main": "dist/index.js",
   "scripts": {