@ai.ntellect/core 0.0.28 → 0.0.29

package/agent/index.ts

@@ -1,4 +1,5 @@
 import EventEmitter from "events";
+import { Evaluator } from "../llm/evaluator";
 import { Orchestrator } from "../llm/orchestrator";
 import { Summarizer } from "../llm/synthesizer";
 import { MemoryCache } from "../memory";
@@ -38,6 +39,7 @@ export class Agent {
       return this.handleActions(
         {
           initialPrompt: prompt,
+          contextualizedPrompt: contextualizedPrompt,
           actions: request.actions,
         },
         events
@@ -48,13 +50,15 @@ export class Agent {
   private async handleActions(
     {
       initialPrompt,
+      contextualizedPrompt,
       actions,
     }: {
       initialPrompt: string;
+      contextualizedPrompt: string;
       actions: ActionSchema[];
     },
     events: AgentEvent
-  ) {
+  ): Promise<any> {
     const similarActions = await this.findSimilarActions(initialPrompt);
     const predefinedActions = this.transformActions(actions, similarActions);
     const callbacks = {
@@ -78,6 +82,26 @@ export class Agent {
       };
     }
 
+    const evaluator = new Evaluator(this.dependencies.orchestrator.tools);
+    const evaluation = await evaluator.process(
+      initialPrompt,
+      contextualizedPrompt,
+      JSON.stringify(actionsResult.data)
+    );
+
+    events.onMessage?.(evaluation);
+
+    if (evaluation.actions.length > 0) {
+      return this.handleActions(
+        {
+          initialPrompt: contextualizedPrompt,
+          contextualizedPrompt: initialPrompt,
+          actions: evaluation.actions,
+        },
+        events
+      );
+    }
+
     return this.handleActionResults({ ...actionsResult, initialPrompt });
   }
 

package/llm/evaluator/context.ts

@@ -0,0 +1,41 @@
+import { z } from "zod";
+import { ActionSchema } from "../../types";
+
+export const evaluatorContext = {
+  role: "You are the evaluator agent. Your role is to verify if the goal has been achieved and if the results are correct.",
+  guidelines: {
+    important: [
+      "IMPORTANT: Verify if all required actions were executed successfully",
+      "IMPORTANT: Check if the results match the initial goal",
+      "IMPORTANT: Identify any missing or incomplete information",
+      "IMPORTANT: Use the same language as the initial request",
+    ],
+    never: [
+      "NEVER modify the results directly",
+      "NEVER make assumptions about missing data",
+      "NEVER repeat the same action if you already did it",
+    ],
+  },
+  compose: (goal: string, results: string, tools: ActionSchema[]) => {
+    return `
+      ${evaluatorContext.role}
+
+      ${evaluatorContext.guidelines.important.join("\n")}
+      ${evaluatorContext.guidelines.never.join("\n")}
+
+      Initial Goal: ${goal}
+      What was done: ${results}
+
+      The actions available are: ${tools.map((action) => {
+        const parameters = action.parameters as z.ZodObject<any>;
+        const schemaShape = Object.keys(parameters._def.shape()).join(", ");
+        const actionString = `Name: ${action.name}, Description: ${action.description}, Arguments: { ${schemaShape} }`;
+        return actionString;
+      })}
+
+      Evaluate if the goal has been achieved and provide:
+      1. Success status with explanation (no action needed)
+      2. Next actions needed (if any)
+    `;
+  },
+};

package/llm/evaluator/index.ts

@@ -0,0 +1,57 @@
+import { openai } from "@ai-sdk/openai";
+import { generateObject } from "ai";
+import { z } from "zod";
+import { ActionSchema } from "../../types";
+import { evaluatorContext } from "./context";
+
+export class Evaluator {
+  private readonly model = openai("gpt-4o");
+  public tools: ActionSchema[];
+
+  constructor(tools: ActionSchema[]) {
+    this.tools = tools;
+  }
+
+  async process(prompt: string, goal: string, results: string): Promise<any> {
+    try {
+      const response = await generateObject({
+        model: this.model,
+        schema: z.object({
+          actions: z.array(
+            z.object({
+              name: z.string(),
+              parameters: z.object({
+                name: z.string(),
+                value: z.string(),
+              }),
+            })
+          ),
+          answer: z.string(),
+        }),
+        prompt: prompt,
+        system: evaluatorContext.compose(goal, results, this.tools),
+      });
+
+      const validatedResponse = {
+        ...response.object,
+        actions: response.object.actions.map((action) => ({
+          ...action,
+          parameters: action.parameters || {},
+        })),
+      };
+
+      console.dir(validatedResponse, { depth: null });
+
+      return validatedResponse;
+    } catch (error: any) {
+      if (error) {
+        console.log("Error in Orchestrator", error.message);
+        console.dir(error.value, { depth: null });
+        return {
+          ...error.value,
+        };
+      }
+      // throw error;
+    }
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai.ntellect/core",
-  "version": "0.0.28",
+  "version": "0.0.29",
   "description": "",
   "main": "dist/index.js",
   "scripts": {