@exulu/backend 1.27.2 → 1.28.0

package/dist/index.cjs

@@ -40,14 +40,15 @@ __export(index_exports, {
   ExuluDefaultAgents: () => ExuluDefaultAgents,
   ExuluDefaultContexts: () => ExuluDefaultContexts,
   ExuluEmbedder: () => ExuluEmbedder,
-  ExuluEval: () => ExuluEval,
+  ExuluEval: () => ExuluEval2,
   ExuluJobs: () => ExuluJobs,
   ExuluOtel: () => ExuluOtel,
   ExuluQueues: () => queues,
   ExuluTool: () => ExuluTool2,
   ExuluUtils: () => ExuluUtils,
   ExuluVariables: () => ExuluVariables,
-  db: () => db2
+  db: () => db2,
+  logMetadata: () => logMetadata
 });
 module.exports = __toCommonJS(index_exports);
 var import_config = require("dotenv/config");
@@ -150,7 +151,17 @@ async function ensureDatabaseExists() {
       database: "postgres",
       // Connect to default database
       password: process.env.POSTGRES_DB_PASSWORD,
-      ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
+      ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false,
+      connectionTimeoutMillis: 1e4
+    },
+    pool: {
+      min: 2,
+      max: 4,
+      acquireTimeoutMillis: 3e4,
+      createTimeoutMillis: 3e4,
+      idleTimeoutMillis: 3e4,
+      reapIntervalMillis: 1e3,
+      createRetryIntervalMillis: 200
     }
   });
   try {
@@ -195,7 +206,17 @@ async function postgresClient() {
           user: process.env.POSTGRES_DB_USER,
           database: dbName,
           password: process.env.POSTGRES_DB_PASSWORD,
-          ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
+          ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false,
+          connectionTimeoutMillis: 1e4
+        },
+        pool: {
+          min: 2,
+          max: 20,
+          acquireTimeoutMillis: 3e4,
+          createTimeoutMillis: 3e4,
+          idleTimeoutMillis: 3e4,
+          reapIntervalMillis: 1e3,
+          createRetryIntervalMillis: 200
         }
       });
       try {
@@ -243,7 +264,9 @@ var bullmqDecorator = async ({
   workflow,
   item,
   context,
-  retries
+  retries,
+  backoff,
+  timeoutInSeconds
 }) => {
   const types = [
     embedder,
@@ -267,30 +290,35 @@ var bullmqDecorator = async ({
   if (embedder) {
     type = "embedder";
   }
+  const jobData = {
+    label,
+    type: `${type}`,
+    timeoutInSeconds: timeoutInSeconds || 180,
+    // 3 minutes default
+    inputs,
+    ...user && { user },
+    ...role && { role },
+    ...trigger && { trigger },
+    ...workflow && { workflow },
+    ...embedder && { embedder },
+    ...processor && { processor },
+    ...evaluation && { evaluation },
+    ...item && { item },
+    ...context && { context }
+  };
   const redisId = (0, import_uuid.v4)();
   const job = await queue.add(
     `${embedder || workflow || processor || evaluation}`,
-    {
-      label,
-      type: `${type}`,
-      inputs,
-      ...user && { user },
-      ...role && { role },
-      ...trigger && { trigger },
-      ...workflow && { workflow },
-      ...embedder && { embedder },
-      ...processor && { processor },
-      ...evaluation && { evaluation },
-      ...item && { item },
-      ...context && { context }
-    },
+    jobData,
     {
       jobId: redisId,
       // Setting it to 3 as a sensible default, as
       // many AI services are quite unstable.
       attempts: retries || 3,
       // todo make this configurable?
-      backoff: {
+      removeOnComplete: 5e3,
+      removeOnFail: 1e4,
+      backoff: backoff || {
         type: "exponential",
         delay: 2e3
       }
@@ -735,6 +763,17 @@ var requestValidators = {
 // src/registry/utils/graphql.ts
 var import_bcryptjs3 = __toESM(require("bcryptjs"), 1);
 
+// types/enums/jobs.ts
+var JOB_STATUS_ENUM = {
+  completed: "completed",
+  failed: "failed",
+  delayed: "delayed",
+  active: "active",
+  waiting: "waiting",
+  paused: "paused",
+  stuck: "stuck"
+};
+
 // src/postgres/core-schema.ts
 var agentMessagesSchema = {
   type: "agent_messages",
@@ -1169,6 +1208,45 @@ var evalSetsSchema = {
     }
   ]
 };
+var jobResultsSchema = {
+  type: "job_results",
+  name: {
+    plural: "job_results",
+    singular: "job_result"
+  },
+  fields: [
+    {
+      name: "job_id",
+      type: "text"
+    },
+    {
+      name: "state",
+      type: "text"
+    },
+    {
+      name: "error",
+      type: "json"
+    },
+    {
+      name: "label",
+      type: "text",
+      index: true
+    },
+    {
+      name: "tries",
+      type: "number",
+      default: 0
+    },
+    {
+      name: "result",
+      type: "json"
+    },
+    {
+      name: "metadata",
+      type: "json"
+    }
+  ]
+};
 var evalRunsSchema = {
   type: "eval_runs",
   name: {
@@ -1177,6 +1255,15 @@ var evalRunsSchema = {
   },
   RBAC: true,
   fields: [
+    {
+      name: "name",
+      type: "text"
+    },
+    {
+      name: "timeout_in_seconds",
+      type: "number",
+      default: 180
+    },
     {
       name: "eval_set_id",
       type: "uuid",
@@ -1188,7 +1275,7 @@ var evalRunsSchema = {
       required: true
     },
     {
-      name: "eval_function_ids",
+      name: "eval_functions",
       type: "json",
       required: true
     },
@@ -1199,7 +1286,7 @@ var evalRunsSchema = {
     {
       name: "scoring_method",
       type: "enum",
-      enumValues: ["mean", "sum", "average"],
+      enumValues: ["median", "sum", "average"],
       required: true
     },
     {
@@ -1323,7 +1410,8 @@ var coreSchemas = {
       variablesSchema: () => addCoreFields(variablesSchema),
       rbacSchema: () => addCoreFields(rbacSchema),
       workflowTemplatesSchema: () => addCoreFields(workflowTemplatesSchema),
-      platformConfigurationsSchema: () => addCoreFields(platformConfigurationsSchema)
+      platformConfigurationsSchema: () => addCoreFields(platformConfigurationsSchema),
+      jobResultsSchema: () => addCoreFields(jobResultsSchema)
     };
   }
 };
@@ -1386,11 +1474,11 @@ var bullmq = {
     if (!data.inputs) {
       throw new Error(`Missing property "inputs" in data for job ${id}.`);
     }
-    if (data.type !== "embedder" && data.type !== "workflow") {
-      throw new Error(`Property "type" in data for job ${id} must be of value "embedder" or "workflow".`);
+    if (data.type !== "embedder" && data.type !== "workflow" && data.type !== "processor" && data.type !== "eval_run" && data.type !== "eval_function") {
+      throw new Error(`Property "type" in data for job ${id} must be of value "embedder", "workflow", "processor", "eval_run" or "eval_function".`);
     }
-    if (!data.workflow && !data.embedder) {
-      throw new Error(`Either a workflow or embedder must be set for job ${id}.`);
+    if (!data.workflow && !data.embedder && !data.processor && !data.eval_run_id && !data.eval_functions?.length) {
+      throw new Error(`Either a workflow, embedder, processor, eval_run or eval_functions must be set for job ${id}.`);
     }
   }
 };
@@ -1594,6 +1682,7 @@ var generateApiKey = async (name, email) => {
 };
 
 // src/registry/utils/graphql.ts
+var import_uuid2 = require("uuid");
 var GraphQLDate = new import_graphql2.GraphQLScalarType({
   name: "Date",
   description: "Date custom scalar type",
@@ -1689,7 +1778,6 @@ ${enumValues}
     fields.push("  maxContextLength: Int");
     fields.push("  provider: String");
     fields.push("  slug: String");
-    fields.push("  evals: [AgentEvalFunction]");
   }
   const rbacField = table.RBAC ? "  RBAC: RBACData" : "";
   const typeDef = `
@@ -1901,11 +1989,6 @@ function createMutations(table, agents, contexts, tools, config) {
       if (!record) {
         throw new Error("Record not found");
       }
-      if (tableNamePlural === "jobs") {
-        if (!user.super_admin && record.created_by !== user.id) {
-          throw new Error("You are not authorized to edit this record");
-        }
-      }
       if (record.rights_mode === "public") {
         return true;
       }
@@ -2328,10 +2411,6 @@ function createMutations(table, agents, contexts, tools, config) {
 }
 var applyAccessControl = (table, user, query) => {
   const tableNamePlural = table.name.plural.toLowerCase();
-  if (!user.super_admin && table.name.plural === "jobs") {
-    query = query.where("created_by", user.id);
-    return query;
-  }
   if (table.name.plural !== "agent_sessions" && user.super_admin === true) {
     return query;
   }
@@ -2466,14 +2545,6 @@ var addAgentFields = async (requestedFields, agents, result, tools, user) => {
   if (requestedFields.includes("provider")) {
     result.provider = backend?.provider || "";
   }
-  if (requestedFields.includes("evals")) {
-    result.evals = backend?.evals?.map((evalFunc) => ({
-      id: evalFunc.id,
-      name: evalFunc.name,
-      description: evalFunc.description,
-      config: evalFunc.config || []
-    })) || [];
-  }
   if (!requestedFields.includes("backend")) {
     delete result.backend;
   }
@@ -3176,7 +3247,7 @@ var contextToTableDefinition = (context) => {
   });
   return addCoreFields(definition);
 };
-function createSDL(tables, contexts, agents, tools, config) {
+function createSDL(tables, contexts, agents, tools, config, evals, queues3) {
   const contextSchemas = contexts.map((context) => contextToTableDefinition(context));
   tables.forEach((table) => {
     if (!table.fields.some((field) => field.name === "createdAt")) {
@@ -3371,15 +3442,39 @@ type PageInfo {
   typeDefs += `
    providers: ProviderPaginationResult
     `;
+  typeDefs += `
+    queue(queue: QueueEnum!): QueueResult
+    `;
+  typeDefs += `
+    evals: EvalPaginationResult
+    `;
   typeDefs += `
     contexts: ContextPaginationResult
     `;
   typeDefs += `
     contextById(id: ID!): Context
     `;
+  mutationDefs += `
+    runEval(id: ID!, test_case_ids: [ID!]): RunEvalReturnPayload
+    `;
+  mutationDefs += `
+    drainQueue(queue: QueueEnum!): JobActionReturnPayload
+    `;
+  mutationDefs += `
+    pauseQueue(queue: QueueEnum!): JobActionReturnPayload
+    `;
+  mutationDefs += `
+    resumeQueue(queue: QueueEnum!): JobActionReturnPayload
+    `;
+  mutationDefs += `
+    deleteJob(queue: QueueEnum!, id: String!): JobActionReturnPayload
+    `;
   typeDefs += `
    tools: ToolPaginationResult
     `;
+  typeDefs += `
+   jobs(queue: QueueEnum!, statusses: [JobStateEnum!]): JobPaginationResult
+    `;
   resolvers.Query["providers"] = async (_, args, context, info) => {
     const requestedFields = getRequestedFields(info);
     return {
@@ -3392,6 +3487,227 @@ type PageInfo {
       })
     };
   };
+  resolvers.Query["queue"] = async (_, args, context, info) => {
+    if (!args.queue) {
+      throw new Error("Queue name is required");
+    }
+    const queue = queues.list.get(args.queue);
+    if (!queue) {
+      throw new Error("Queue not found");
+    }
+    const config2 = await queue.use();
+    return {
+      name: config2.queue.name,
+      concurrency: config2.concurrency,
+      ratelimit: config2.ratelimit,
+      isMaxed: await config2.queue.isMaxed(),
+      isPaused: await config2.queue.isPaused(),
+      jobs: {
+        paused: await config2.queue.isPaused(),
+        completed: await config2.queue.getJobCountByTypes("completed"),
+        failed: await config2.queue.getJobCountByTypes("failed"),
+        waiting: await config2.queue.getJobCountByTypes("waiting"),
+        active: await config2.queue.getJobCountByTypes("active"),
+        delayed: await config2.queue.getJobCountByTypes("delayed")
+      }
+    };
+  };
+  resolvers.Mutation["runEval"] = async (_, args, context, info) => {
+    console.log("[EXULU] /evals/run/:id", args.id);
+    const user = context.user;
+    const eval_run_id = args.id;
+    if (!user.super_admin && (!user.role || user.role.evals !== "write")) {
+      throw new Error("You don't have permission to run evals. Required: super_admin or evals write access.");
+    }
+    const { db: db3 } = await postgresClient();
+    const evalRun = await db3.from("eval_runs").where({ id: eval_run_id }).first();
+    if (!evalRun) {
+      throw new Error("Eval run not found in database.");
+    }
+    const hasAccessToEvalRun = await checkRecordAccess(evalRun, "write", user);
+    if (!hasAccessToEvalRun) {
+      throw new Error("You don't have access to this eval run.");
+    }
+    let testCaseIds = evalRun.test_case_ids ? typeof evalRun.test_case_ids === "string" ? JSON.parse(evalRun.test_case_ids) : evalRun.test_case_ids : [];
+    const eval_functions = evalRun.eval_functions ? typeof evalRun.eval_functions === "string" ? JSON.parse(evalRun.eval_functions) : evalRun.eval_functions : [];
+    if (!testCaseIds || testCaseIds.length === 0) {
+      throw new Error("No test cases selected for this eval run.");
+    }
+    if (!eval_functions || eval_functions.length === 0) {
+      throw new Error("No eval functions selected for this eval run.");
+    }
+    if (args.test_case_ids) {
+      testCaseIds = testCaseIds.filter((testCase) => args.test_case_ids.includes(testCase));
+    }
+    console.log("[EXULU] test cases ids filtered", testCaseIds);
+    const testCases = await db3.from("test_cases").whereIn("id", testCaseIds);
+    if (testCases.length === 0) {
+      throw new Error("No test cases found for eval run.");
+    }
+    const agentInstance = await loadAgent(evalRun.agent_id);
+    if (!agentInstance) {
+      throw new Error("Agent instance not found for eval run.");
+    }
+    const evalQueue = await queues.register("eval_runs", 1, 1).use();
+    const jobIds = [];
+    for (const testCase of testCases) {
+      const jobData = {
+        label: `Eval Run ${eval_run_id} - Test Case ${testCase.id}`,
+        trigger: "api",
+        timeoutInSeconds: evalRun.timeout_in_seconds || 180,
+        // default to 3 minutes
+        type: "eval_run",
+        eval_run_id,
+        eval_run_name: evalRun.name,
+        test_case_id: testCase.id,
+        test_case_name: testCase.name,
+        eval_functions,
+        // Array of eval function IDs - worker will create child jobs for these
+        agent_id: evalRun.agent_id,
+        inputs: testCase.inputs,
+        expected_output: testCase.expected_output,
+        expected_tools: testCase.expected_tools,
+        expected_knowledge_sources: testCase.expected_knowledge_sources,
+        expected_agent_tools: testCase.expected_agent_tools,
+        config: evalRun.config,
+        scoring_method: evalRun.scoring_method,
+        pass_threshold: evalRun.pass_threshold,
+        user: user.id,
+        role: user.role?.id
+      };
+      const redisId = (0, import_uuid2.v4)();
+      const job = await evalQueue.queue.add("eval_run", jobData, {
+        jobId: redisId,
+        // Setting it to 3 as a sensible default, as
+        // many AI services are quite unstable.
+        attempts: evalQueue.retries || 1,
+        removeOnComplete: 5e3,
+        removeOnFail: 1e4,
+        backoff: evalQueue.backoff || {
+          type: "exponential",
+          delay: 2e3
+        }
+      });
+      jobIds.push(job.id);
+    }
+    const response = {
+      jobs: jobIds,
+      count: jobIds.length
+    };
+    const requestedFields = getRequestedFields(info);
+    const mapped = {};
+    requestedFields.forEach((field) => {
+      mapped[field] = response[field];
+    });
+    return mapped;
+  };
+  resolvers.Mutation["drainQueue"] = async (_, args, context, info) => {
+    if (!args.queue) {
+      throw new Error("Queue name is required");
+    }
+    const queue = queues.list.get(args.queue);
+    if (!queue) {
+      throw new Error("Queue not found");
+    }
+    const config2 = await queue.use();
+    await config2.queue.drain();
+    return { success: true };
+  };
+  resolvers.Mutation["pauseQueue"] = async (_, args, context, info) => {
+    if (!args.queue) {
+      throw new Error("Queue name is required");
+    }
+    const queue = queues.list.get(args.queue);
+    if (!queue) {
+      throw new Error("Queue not found");
+    }
+    const config2 = await queue.use();
+    await config2.queue.pause();
+    return { success: true };
+  };
+  resolvers.Mutation["resumeQueue"] = async (_, args, context, info) => {
+    if (!args.queue) {
+      throw new Error("Queue name is required");
+    }
+    const queue = queues.list.get(args.queue);
+    if (!queue) {
+      throw new Error("Queue not found");
+    }
+    const config2 = await queue.use();
+    await config2.queue.resume();
+    return { success: true };
+  };
+  resolvers.Mutation["deleteJob"] = async (_, args, context, info) => {
+    if (!args.id) {
+      throw new Error("Job ID is required");
+    }
+    if (!args.queue) {
+      throw new Error("Queue name is required");
+    }
+    const queue = queues.list.get(args.queue);
+    if (!queue) {
+      throw new Error("Queue not found");
+    }
+    const config2 = await queue.use();
+    await config2.queue.remove(args.id);
+    return { success: true };
+  };
+  resolvers.Query["evals"] = async (_, args, context, info) => {
+    const requestedFields = getRequestedFields(info);
+    return {
+      items: evals.map((_eval) => {
+        const object = {};
+        requestedFields.forEach((field) => {
+          object[field] = _eval[field];
+        });
+        return object;
+      })
+    };
+  };
+  resolvers.Query["jobs"] = async (_, args, context, info) => {
+    if (!args.queue) {
+      throw new Error("Queue name is required");
+    }
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      throw new Error("Redis client not created properly");
+    }
+    const {
+      jobs,
+      count
+    } = await getJobsByQueueAndName(
+      args.queue,
+      args.statusses,
+      args.page || 1,
+      args.limit || 100
+    );
+    console.log("[EXULU] jobs", jobs.map((job) => job.name));
+    const requestedFields = getRequestedFields(info);
+    return {
+      items: await Promise.all(jobs.map(async (job) => {
+        const object = {};
+        for (const field of requestedFields) {
+          if (field === "data") {
+            object[field] = job[field];
+          } else if (field === "timestamp") {
+            object[field] = new Date(job[field]).toISOString();
+          } else if (field === "state") {
+            object[field] = await job.getState();
+          } else {
+            object[field] = job[field];
+          }
+        }
+        return object;
+      })),
+      pageInfo: {
+        pageCount: Math.ceil(count / (args.limit || 100)),
+        itemCount: count,
+        currentPage: args.page || 1,
+        hasPreviousPage: args.page && args.page > 1 ? true : false,
+        hasNextPage: args.page && args.page < Math.ceil(jobs.length / (args.limit || 100)) ? true : false
+      }
+    };
+  };
   resolvers.Query["contexts"] = async (_, args, context, info) => {
     const data = contexts.map((context2) => ({
       id: context2.id,
@@ -3497,7 +3813,32 @@ type PageInfo {
   };
   modelDefs += `
     type ProviderPaginationResult {
-    items: [Provider]!
+        items: [Provider]!
+    }
+    `;
+  modelDefs += `
+    type QueueResult {
+        name: String!
+        concurrency: Int!
+        ratelimit: Int!
+        isMaxed: Boolean!
+        isPaused: Boolean!
+        jobs: QueueJobsCounts
+    }
+    `;
+  modelDefs += `
+    type QueueJobsCounts {
+        paused: Int!
+        completed: Int!
+        failed: Int!
+        waiting: Int!
+        active: Int!
+        delayed: Int!
+    }
+    `;
+  modelDefs += `
+    type EvalPaginationResult {
+    items: [Eval]!
     }
     `;
   modelDefs += `
@@ -3510,6 +3851,12 @@ type PageInfo {
     items: [Tool]!
     }
     `;
+  modelDefs += `
+    type JobPaginationResult {
+        items: [Job]!
+        pageInfo: PageInfo!
+    }
+    `;
   typeDefs += "}\n";
   mutationDefs += "}\n";
   const genericTypes = `
@@ -3567,6 +3914,19 @@ type Provider {
   type: EnumProviderType!
 }
 
+type Eval {
+    id: ID!
+    name: String!
+    description: String!
+    llm: Boolean!
+    config: [EvalConfig!]
+}
+
+type EvalConfig {
+    name: String!
+    description: String!
+}
+
 type Context {
     id: ID!
     name: String!
@@ -3578,6 +3938,15 @@ type Context {
     configuration: JSON
 }
 
+type RunEvalReturnPayload {
+    jobs: [String!]!
+    count: Int!
+}
+
+type JobActionReturnPayload {
+    success: Boolean!
+}
+
 type ContextField {
     name: String!
     type: String!
@@ -3593,10 +3962,35 @@ type Tool {
   config: JSON
 }
 
+type Job {
+  id: String!
+  name: String!
+  returnvalue: JSON
+  stacktrace: [String]
+  failedReason: String
+  state: String!
+  data: JSON
+  timestamp: Date
+}
+
 enum EnumProviderType {
   agent
 }
 
+enum QueueEnum {
+  ${queues.list.keys().toArray().join("\n")}
+}
+
+enum JobStateEnum {
+  ${JOB_STATUS_ENUM.active}
+  ${JOB_STATUS_ENUM.waiting}
+  ${JOB_STATUS_ENUM.delayed}
+  ${JOB_STATUS_ENUM.failed}
+  ${JOB_STATUS_ENUM.completed}
+  ${JOB_STATUS_ENUM.paused}
+  ${JOB_STATUS_ENUM.stuck}
+}
+
 type StatisticsResult {
   group: String!
   count: Int!
@@ -3626,6 +4020,25 @@ var validateCreateOrRemoveSuperAdminPermission = async (tableNamePlural, input,
     }
   }
 };
+async function getJobsByQueueAndName(queueName, statusses, page, limit) {
+  const queue = queues.list.get(queueName);
+  if (!queue) {
+    throw new Error(`Queue ${queueName} not found`);
+  }
+  const config = await queue.use();
+  const startIndex = (page || 1) - 1;
+  const endIndex = startIndex + (limit || 100);
+  const jobs = await config.queue.getJobs(statusses || [], startIndex, endIndex, false);
+  const counts = await config.queue.getJobCounts(...statusses || []);
+  let total = 0;
+  if (counts) {
+    total = Object.keys(counts).reduce((acc, key) => acc + (counts[key] || 0), 0);
+  }
+  return {
+    jobs,
+    count: total
+  };
+}
 
 // src/registry/classes.ts
 var import_client_s32 = require("@aws-sdk/client-s3");
@@ -3675,6 +4088,7 @@ var addPrefixToKey = (keyPath, config) => {
   return `${prefix}/${keyPath}`;
 };
 var uploadFile = async (user, file, key, config, options = {}) => {
+  console.log("[EXULU] Uploading file to S3", key);
   const client2 = getS3Client(config);
   let folder = `${user}/`;
   const fullKey = addPrefixToKey(!key.includes(folder) ? folder + key : key, config);
@@ -4288,27 +4702,34 @@ function errorHandler(error) {
   }
   return JSON.stringify(error);
 }
-var ExuluEval = class {
+var ExuluEval2 = class {
   id;
   name;
   description;
+  llm;
   execute;
   config;
   queue;
-  constructor({ id, name, description, execute: execute2, config, queue }) {
+  constructor({ id, name, description, execute: execute2, config, queue, llm }) {
     this.id = id;
     this.name = name;
     this.description = description;
     this.execute = execute2;
     this.config = config;
+    this.llm = llm;
     this.queue = queue;
   }
-  async run(messages, metadata, config) {
-    const score = await this.execute({ messages, metadata, config });
-    if (score < 0 || score > 100) {
-      throw new Error(`Eval function ${this.name} must return a score between 0 and 100, got ${score}`);
+  async run(agent, backend, testCase, messages, config) {
+    try {
+      const score = await this.execute({ agent, backend, testCase, messages, config });
+      if (score < 0 || score > 100) {
+        throw new Error(`Eval function ${this.name} must return a score between 0 and 100, got ${score}`);
+      }
+      return score;
+    } catch (error) {
+      console.error(`[EXULU] error running eval function ${this.name}.`, error);
+      throw new Error(`Error running eval function ${this.name}: ${error instanceof Error ? error.message : String(error)}`);
     }
-    return score;
   }
 };
 var ExuluAgent2 = class {
@@ -4323,13 +4744,13 @@ var ExuluAgent2 = class {
   type;
   streaming = false;
   maxContextLength;
+  queue;
   rateLimit;
   config;
-  evals;
   // private memory: Memory | undefined; // TODO do own implementation
   model;
   capabilities;
-  constructor({ id, name, description, config, rateLimit, capabilities, type, maxContextLength, evals, provider }) {
+  constructor({ id, name, description, config, rateLimit, capabilities, type, maxContextLength, provider, queue }) {
     this.id = id;
     this.name = name;
     this.description = description;
@@ -4338,7 +4759,7 @@ var ExuluAgent2 = class {
     this.config = config;
     this.type = type;
     this.maxContextLength = maxContextLength;
-    this.evals = evals;
+    this.queue = queue;
     this.capabilities = capabilities || {
       text: false,
       images: [],
@@ -4437,7 +4858,7 @@ var ExuluAgent2 = class {
     prompt,
     user,
     session,
-    message,
+    inputMessages,
     currentTools,
     allExuluTools,
     statistics,
@@ -4455,10 +4876,10 @@ var ExuluAgent2 = class {
     if (!this.config) {
       throw new Error("Config is required for generating.");
     }
-    if (prompt && message) {
+    if (prompt && inputMessages?.length) {
       throw new Error("Message and prompt cannot be provided at the same time.");
     }
-    if (!prompt && !message) {
+    if (!prompt && !inputMessages?.length) {
       throw new Error("Prompt or message is required for generating.");
     }
     if (outputSchema && !prompt) {
@@ -4468,18 +4889,18 @@ var ExuluAgent2 = class {
       apiKey: providerapikey
     });
     console.log("[EXULU] Model for agent: " + this.name, " created for generating sync.");
-    let messages = [];
-    if (message && session && user) {
+    let messages = inputMessages || [];
+    if (messages && session && user) {
       const previousMessages = await getAgentMessages({
         session,
         user: user.id,
         limit: 50,
         page: 1
       });
-      const previousMessagesContent = previousMessages.map((message2) => JSON.parse(message2.content));
+      const previousMessagesContent = previousMessages.map((message) => JSON.parse(message.content));
       messages = await (0, import_ai.validateUIMessages)({
         // append the new message to the previous messages:
-        messages: [...previousMessagesContent, message]
+        messages: [...previousMessagesContent, ...messages]
       });
     }
     console.log("[EXULU] Message count for agent: " + this.name, "loaded for generating sync.", messages.length);
@@ -4615,13 +5036,12 @@ var ExuluAgent2 = class {
     return "";
   };
   generateStream = async ({
-    express: express3,
     user,
     session,
     message,
+    previousMessages,
     currentTools,
     allExuluTools,
-    statistics,
     toolConfigs,
     providerapikey,
     contexts,
@@ -4629,27 +5049,34 @@ var ExuluAgent2 = class {
     instructions
   }) => {
     if (!this.model) {
+      console.error("[EXULU] Model is required for streaming.");
       throw new Error("Model is required for streaming.");
     }
     if (!this.config) {
+      console.error("[EXULU] Config is required for streaming.");
       throw new Error("Config is required for generating.");
     }
     if (!message) {
+      console.error("[EXULU] Message is required for streaming.");
       throw new Error("Message is required for streaming.");
     }
     const model = this.model.create({
       apiKey: providerapikey
     });
     let messages = [];
-    const previousMessages = await getAgentMessages({
-      session,
-      user: user.id,
-      limit: 50,
-      page: 1
-    });
-    const previousMessagesContent = previousMessages.map(
-      (message2) => JSON.parse(message2.content)
-    );
+    let previousMessagesContent = previousMessages || [];
+    if (session) {
+      console.log("[EXULU] loading previous messages from session: " + session);
+      const previousMessages2 = await getAgentMessages({
+        session,
+        user: user.id,
+        limit: 50,
+        page: 1
+      });
+      previousMessagesContent = previousMessages2.map(
+        (message2) => JSON.parse(message2.content)
+      );
+    }
     messages = await (0, import_ai.validateUIMessages)({
       // append the new message to the previous messages:
       messages: [...previousMessagesContent, message]
@@ -4682,80 +5109,17 @@ var ExuluAgent2 = class {
         user,
         exuluConfig
       ),
-      onError: (error) => console.error("[EXULU] chat stream error.", error)
-      // stopWhen: [stepCountIs(1)],
-    });
-    result.consumeStream();
-    result.pipeUIMessageStreamToResponse(express3.res, {
-      messageMetadata: ({ part }) => {
-        if (part.type === "finish") {
-          return {
-            totalTokens: part.totalUsage.totalTokens,
-            reasoningTokens: part.totalUsage.reasoningTokens,
-            inputTokens: part.totalUsage.inputTokens,
-            outputTokens: part.totalUsage.outputTokens,
-            cachedInputTokens: part.totalUsage.cachedInputTokens
-          };
-        }
-      },
-      originalMessages: messages,
-      sendReasoning: true,
-      sendSources: true,
       onError: (error) => {
-        console.error("[EXULU] chat response error.", error);
-        return errorHandler(error);
-      },
-      generateMessageId: (0, import_ai.createIdGenerator)({
-        prefix: "msg_",
-        size: 16
-      }),
-      onFinish: async ({ messages: messages2, isContinuation, isAborted, responseMessage }) => {
-        if (session) {
-          await saveChat({
-            session,
-            user: user.id,
-            messages: messages2.filter((x) => !previousMessagesContent.find((y) => y.id === x.id))
-          });
-        }
-        const metadata = messages2[messages2.length - 1]?.metadata;
-        console.log("[EXULU] Finished streaming", metadata);
-        console.log("[EXULU] Statistics", statistics);
-        if (statistics) {
-          await Promise.all([
-            updateStatistic({
-              name: "count",
-              label: statistics.label,
-              type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-              trigger: statistics.trigger,
-              count: 1,
-              user: user.id,
-              role: user?.role?.id
-            }),
-            ...metadata?.inputTokens ? [
-              updateStatistic({
-                name: "inputTokens",
-                label: statistics.label,
-                type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-                trigger: statistics.trigger,
-                count: metadata?.inputTokens,
-                user: user.id,
-                role: user?.role?.id
-              })
-            ] : [],
-            ...metadata?.outputTokens ? [
-              updateStatistic({
-                name: "outputTokens",
-                label: statistics.label,
-                type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-                trigger: statistics.trigger,
-                count: metadata?.outputTokens
-              })
-            ] : []
-          ]);
-        }
+        console.error("[EXULU] chat stream error.", error);
+        throw new Error(`Chat stream error: ${error instanceof Error ? error.message : String(error)}`);
       }
+      // stopWhen: [stepCountIs(1)],
     });
-    return;
+    return {
+      stream: result,
+      originalMessages: messages,
+      previousMessages: previousMessagesContent
+    };
   };
 };
 var getAgentMessages = async ({ session, user, limit, page }) => {
@@ -4946,16 +5310,21 @@ var ExuluContext = class {
     if (queue?.queue.name) {
       console.log("[EXULU] processor is in queue mode, scheduling job.");
       const job = await bullmqDecorator({
+        timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 180,
         label: `${this.name} ${field.name} data processor`,
         processor: `${this.id}-${field.name}`,
         context: this.id,
         inputs: item,
         item: item.id,
         queue: queue.queue,
+        backoff: queue.backoff || {
+          type: "exponential",
+          delay: 2e3
+        },
+        retries: queue.retries || 2,
         user,
         role,
-        trigger,
-        retries: 2
+        trigger
       });
       return {
         result: "",
@@ -5025,12 +5394,14 @@ var ExuluContext = class {
       role
     );
     await db3.from(getChunksTableName(this.id)).where({ source }).delete();
-    await db3.from(getChunksTableName(this.id)).insert(chunks.map((chunk) => ({
-      source,
-      content: chunk.content,
-      chunk_index: chunk.index,
-      embedding: import_knex5.default.toSql(chunk.vector)
-    })));
+    if (chunks?.length) {
+      await db3.from(getChunksTableName(this.id)).insert(chunks.map((chunk) => ({
+        source,
+        content: chunk.content,
+        chunk_index: chunk.index,
+        embedding: import_knex5.default.toSql(chunk.vector)
+      })));
+    }
     await db3.from(getTableName(this.id)).where({ id: item.id }).update({
       embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
     }).returning("id");
@@ -5159,9 +5530,15 @@ var ExuluContext = class {
         if (queue?.queue.name) {
           console.log("[EXULU] embedder is in queue mode, scheduling job.");
           const job = await bullmqDecorator({
+            timeoutInSeconds: queue.timeoutInSeconds || 180,
             label: `${this.embedder.name}`,
             embedder: this.embedder.id,
             context: this.id,
+            backoff: queue.backoff || {
+              type: "exponential",
+              delay: 2e3
+            },
+            retries: queue.retries || 2,
             inputs: item,
             item: item.id,
             queue: queue.queue,
@@ -5413,6 +5790,8 @@ var ExuluQueues = class {
   constructor() {
     this.queues = [];
   }
+  list = /* @__PURE__ */ new Map();
+  // list of queue names
   queue(name) {
     return this.queues.find((x) => x.queue?.name === name);
   }
@@ -5424,40 +5803,54 @@ var ExuluQueues = class {
   // method of ExuluQueues we need to store the desired rate limit on the queue
   // here so we can use the value when creating workers for the queue instance
   // as there is no way to store a rate limit value natively on a bullm queue.
-  use = async (name, concurrency = 1, ratelimit = 1) => {
-    const existing = this.queues.find((x) => x.queue?.name === name);
-    if (existing) {
-      const globalConcurrency = await existing.queue.getGlobalConcurrency();
-      if (globalConcurrency !== concurrency) {
-        await existing.queue.setGlobalConcurrency(concurrency);
+  register = (name, concurrency = 1, ratelimit = 1) => {
+    const use = async () => {
+      const existing = this.queues.find((x) => x.queue?.name === name);
+      if (existing) {
+        const globalConcurrency = await existing.queue.getGlobalConcurrency();
+        if (globalConcurrency !== concurrency) {
+          await existing.queue.setGlobalConcurrency(concurrency);
+        }
+        return {
+          queue: existing.queue,
+          ratelimit,
+          concurrency
+        };
       }
-      return {
-        queue: existing.queue,
-        ratelimit,
-        concurrency
-      };
-    }
-    if (!redisServer.host?.length || !redisServer.port?.length) {
-      console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or embedder (look for ExuluQueues.use() ).`);
-      throw new Error(`[EXULU] no redis server configured.`);
-    }
-    const newQueue = new import_bullmq4.Queue(
-      `${name}`,
-      {
-        connection: redisServer,
-        telemetry: new import_bullmq_otel.BullMQOtel("simple-guide")
+      if (!redisServer.host?.length || !redisServer.port?.length) {
+        console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or embedder (look for ExuluQueues.register().use() ).`);
+        throw new Error(`[EXULU] no redis server configured.`);
       }
-    );
-    await newQueue.setGlobalConcurrency(concurrency);
-    this.queues.push({
-      queue: newQueue,
+      const newQueue = new import_bullmq4.Queue(
+        `${name}`,
+        {
+          connection: {
+            ...redisServer,
+            enableOfflineQueue: false
+          },
+          telemetry: new import_bullmq_otel.BullMQOtel("simple-guide")
+        }
+      );
+      await newQueue.setGlobalConcurrency(concurrency);
+      this.queues.push({
+        queue: newQueue,
+        ratelimit,
+        concurrency
+      });
+      return {
+        queue: newQueue,
+        ratelimit,
+        concurrency
+      };
+    };
+    this.list.set(name, {
+      name,
+      concurrency,
       ratelimit,
-      concurrency
+      use
     });
     return {
-      queue: newQueue,
-      ratelimit,
-      concurrency
+      use
     };
   };
 };
@@ -5476,8 +5869,6 @@ var import_openai = __toESM(require("openai"), 1);
 var import_fs = __toESM(require("fs"), 1);
 var import_node_crypto3 = require("crypto");
 var import_api = require("@opentelemetry/api");
-var import_ai2 = require("ai");
-var import_express_http_proxy = __toESM(require("express-http-proxy"), 1);
 var import_sdk = __toESM(require("@anthropic-ai/sdk"), 1);
 
 // src/registry/utils/claude-messages.ts
@@ -5503,13 +5894,15 @@ var CLAUDE_MESSAGES = {
 };
 
 // src/registry/routes.ts
+var import_ai2 = require("ai");
 var REQUEST_SIZE_LIMIT = "50mb";
 var global_queues = {
-  logs_cleaner: "logs-cleaner"
+  eval_runs: "eval_runs"
 };
 var {
   agentsSchema: agentsSchema2,
   projectsSchema: projectsSchema2,
+  jobResultsSchema: jobResultsSchema2,
   testCasesSchema: testCasesSchema2,
   evalSetsSchema: evalSetsSchema2,
   evalRunsSchema: evalRunsSchema2,
@@ -5523,38 +5916,7 @@ var {
   rbacSchema: rbacSchema2,
   statisticsSchema: statisticsSchema2
 } = coreSchemas.get();
-var createRecurringJobs = async () => {
-  console.log("[EXULU] creating recurring jobs.");
-  const recurringJobSchedulersLogs = [];
-  const queue = await queues.use(global_queues.logs_cleaner);
-  recurringJobSchedulersLogs.push({
-    name: global_queues.logs_cleaner,
-    pattern: "0 10 * * * *",
-    ttld: "30 days",
-    opts: {
-      backoff: 3,
-      attempts: 5,
-      removeOnFail: 1e3
-    }
-  });
-  await queue.queue.upsertJobScheduler(
-    "logs-cleaner-scheduler",
-    { pattern: "0 10 * * * *" },
-    // every 10 minutes
-    {
-      name: global_queues.logs_cleaner,
-      data: { ttld: 30 },
-      // time to live in days
-      opts: {
-        backoff: 3,
-        attempts: 5,
-        removeOnFail: 1e3
-      }
-    }
-  );
-  return queue;
-};
-var createExpressRoutes = async (app, agents, tools, contexts, config, tracer) => {
+var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer, queues3) => {
   var corsOptions = {
     origin: "*",
     exposedHeaders: "*",
@@ -5573,19 +5935,15 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, tracer) =
     \u2588\u2588\u2554\u2550\u2550\u255D   \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2554\u255D \u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D
     \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D 
-    Intelligence Management Platform
+    Intelligence Management Platform - Server
 
     `);
-  if (redisServer.host?.length && redisServer.port?.length) {
-    await createRecurringJobs();
-  } else {
-    console.log("[EXULU] no redis server configured, not setting up recurring jobs.");
-  }
   const schema = createSDL([
     usersSchema2(),
     rolesSchema2(),
     agentsSchema2(),
     projectsSchema2(),
+    jobResultsSchema2(),
     evalRunsSchema2(),
     platformConfigurationsSchema2(),
     evalSetsSchema2(),
@@ -5596,7 +5954,7 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, tracer) =
     workflowTemplatesSchema2(),
     statisticsSchema2(),
     rbacSchema2()
-  ], contexts ?? [], agents, tools, config);
+  ], contexts ?? [], agents, tools, config, evals, queues3 || []);
   const server = new import_server3.ApolloServer({
     cache: new import_utils3.InMemoryLRUCache(),
     schema,
@@ -5727,103 +6085,6 @@ Mood: friendly and intelligent.
       image: `${process.env.BACKEND}/${uuid}.png`
     });
   });
-  app.post("/evals/run/:id", async (req, res) => {
-    console.log("[EXULU] /evals/run/:id", req.params.id);
-    const authenticationResult = await requestValidators.authenticate(req);
-    if (!authenticationResult.user?.id) {
-      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-      return;
-    }
-    const user = authenticationResult.user;
-    const evalRunId = req.params.id;
-    if (!user.super_admin && (!user.role || user.role.evals !== "write")) {
-      res.status(403).json({
-        message: "You don't have permission to run evals. Required: super_admin or evals write access."
-      });
-      return;
-    }
-    const { db: db3 } = await postgresClient();
-    const evalRun = await db3.from("eval_runs").where({ id: evalRunId }).first();
-    if (!evalRun) {
-      res.status(404).json({
-        message: "Eval run not found."
-      });
-      return;
-    }
-    const hasAccessToEvalRun = await checkRecordAccess(evalRun, "write", user);
-    if (!hasAccessToEvalRun) {
-      res.status(403).json({
-        message: "You don't have access to this eval run."
-      });
-      return;
-    }
-    const testCaseIds = evalRun.test_case_ids ? typeof evalRun.test_case_ids === "string" ? JSON.parse(evalRun.test_case_ids) : evalRun.test_case_ids : [];
-    const evalFunctionIds = evalRun.eval_function_ids ? typeof evalRun.eval_function_ids === "string" ? JSON.parse(evalRun.eval_function_ids) : evalRun.eval_function_ids : [];
-    if (!testCaseIds || testCaseIds.length === 0) {
-      res.status(400).json({
-        message: "No test cases selected for this eval run."
-      });
-      return;
-    }
-    if (!evalFunctionIds || evalFunctionIds.length === 0) {
-      res.status(400).json({
-        message: "No eval functions selected for this eval run."
-      });
-      return;
-    }
-    const testCases = await db3.from("test_cases").whereIn("id", testCaseIds);
-    if (testCases.length === 0) {
-      res.status(404).json({
-        message: "No test cases found."
-      });
-      return;
-    }
-    const agentInstance = await loadAgent(evalRun.agentId);
-    if (!agentInstance) {
-      res.status(404).json({
-        message: "Agent instance not found."
-      });
-      return;
-    }
-    const evalQueue = await queues.use("evals");
-    const jobIds = [];
-    for (const testCase of testCases) {
-      const existingJobs = await evalQueue.queue.getJobs(["waiting", "active", "delayed", "paused"]);
-      const duplicateJob = existingJobs.find(
-        (job2) => job2.data.evalRunId === evalRunId && job2.data.testCaseId === testCase.id && job2.data.type === "eval"
-      );
-      if (duplicateJob) {
-        console.log(`[EXULU] Skipping duplicate job for eval run ${evalRunId} and test case ${testCase.id}`);
-        continue;
-      }
-      const job = await evalQueue.queue.add(`eval-${testCase.id}`, {
-        type: "eval",
-        evalRunId,
-        testCaseId: testCase.id,
-        evalFunctionIds,
-        // Array of eval function IDs - worker will create child jobs for these
-        agentId: evalRun.agentId,
-        inputs: testCase.inputs,
-        expected_output: testCase.expected_output,
-        expected_tools: testCase.expected_tools,
-        expected_knowledge_sources: testCase.expected_knowledge_sources,
-        expected_agent_tools: testCase.expected_agent_tools,
-        config: evalRun.config,
-        scoring_method: evalRun.scoring_method,
-        pass_threshold: evalRun.pass_threshold,
-        user: user.id,
-        role: user.role?.id
-      });
-      jobIds.push(job.id);
-    }
-    res.status(200).json({
-      message: `Created ${jobIds.length} eval jobs.`,
-      jobIds,
-      evalRunId,
-      testCaseCount: testCases.length,
-      evalFunctionCount: evalFunctionIds.length
-    });
-  });
   app.get("/ping", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -5952,11 +6213,11 @@ Mood: friendly and intelligent.
         providerapikey = bytes.toString(import_crypto_js3.default.enc.Utf8);
       }
       if (!!headers.stream) {
-        await agent.generateStream({
-          express: {
-            res,
-            req
-          },
+        const statistics = {
+          label: agent.name,
+          trigger: "agent"
+        };
+        const result = await agent.generateStream({
           contexts,
           user,
           instructions: agentInstance.instructions,
@@ -5966,10 +6227,79 @@ Mood: friendly and intelligent.
           allExuluTools: tools,
           providerapikey,
           toolConfigs: agentInstance.tools,
-          exuluConfig: config,
-          statistics: {
-            label: agent.name,
-            trigger: "agent"
+          exuluConfig: config
+        });
+        result.stream.consumeStream();
+        result.stream.pipeUIMessageStreamToResponse(res, {
+          messageMetadata: ({ part }) => {
+            if (part.type === "finish") {
+              return {
+                totalTokens: part.totalUsage.totalTokens,
+                reasoningTokens: part.totalUsage.reasoningTokens,
+                inputTokens: part.totalUsage.inputTokens,
+                outputTokens: part.totalUsage.outputTokens,
+                cachedInputTokens: part.totalUsage.cachedInputTokens
+              };
+            }
+          },
+          originalMessages: result.originalMessages,
+          sendReasoning: true,
+          sendSources: true,
+          onError: (error) => {
+            console.error("[EXULU] chat response error.", error);
+            return errorHandler(error);
+          },
+          generateMessageId: (0, import_ai2.createIdGenerator)({
+            prefix: "msg_",
+            size: 16
+          }),
+          onFinish: async ({ messages, isContinuation, isAborted, responseMessage }) => {
+            if (headers.session) {
+              await saveChat({
+                session: headers.session,
+                user: user.id,
+                messages: messages.filter((x) => !result.previousMessages.find((y) => y.id === x.id))
+              });
+            }
+            const metadata = messages[messages.length - 1]?.metadata;
+            console.log("[EXULU] Finished streaming", metadata);
+            console.log("[EXULU] Statistics", {
+              label: agent.name,
+              trigger: "agent"
+            });
+            if (statistics) {
+              await Promise.all([
+                updateStatistic({
+                  name: "count",
+                  label: statistics.label,
+                  type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+                  trigger: statistics.trigger,
+                  count: 1,
+                  user: user.id,
+                  role: user?.role?.id
+                }),
+                ...metadata?.inputTokens ? [
+                  updateStatistic({
+                    name: "inputTokens",
+                    label: statistics.label,
+                    type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+                    trigger: statistics.trigger,
+                    count: metadata?.inputTokens,
+                    user: user.id,
+                    role: user?.role?.id
+                  })
+                ] : [],
+                ...metadata?.outputTokens ? [
+                  updateStatistic({
+                    name: "outputTokens",
+                    label: statistics.label,
+                    type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+                    trigger: statistics.trigger,
+                    count: metadata?.outputTokens
+                  })
+                ] : []
+              ]);
+            }
           }
         });
         return;
@@ -5978,7 +6308,7 @@ Mood: friendly and intelligent.
           user,
           instructions: agentInstance.instructions,
           session: headers.session,
-          message: req.body.message,
+          inputMessages: [req.body.message],
           contexts,
           currentTools: enabledTools,
           allExuluTools: tools,
@@ -6000,91 +6330,6 @@ Mood: friendly and intelligent.
   } else {
     console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in ExuluApp instance.");
   }
-  app.use("/xxx/anthropic/:id", import_express4.default.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }), (0, import_express_http_proxy.default)(
-    (req, res, next) => {
-      return "https://api.anthropic.com";
-    },
-    {
-      limit: "50mb",
-      memoizeHost: false,
-      preserveHostHdr: true,
-      secure: false,
-      reqAsBuffer: true,
-      proxyReqBodyDecorator: function(bodyContent, srcReq) {
-        return bodyContent;
-      },
-      userResDecorator: function(proxyRes, proxyResData, userReq, userRes) {
-        console.log("[EXULU] Proxy response!", proxyResData);
-        proxyResData = proxyResData.toString();
-        console.log("[EXULU] Proxy response string!", proxyResData);
-        return proxyResData;
-      },
-      proxyReqPathResolver: (req) => {
-        const prefix = `/gateway/anthropic/${req.params.id}`;
-        let path = req.url.startsWith(prefix) ? req.url.slice(prefix.length) : req.url;
-        if (!path.startsWith("/")) path = "/" + path;
-        console.log("[EXULU] Provider path:", path);
-        return path;
-      },
-      proxyReqOptDecorator: function(proxyReqOpts, srcReq) {
-        return new Promise(async (resolve, reject) => {
-          try {
-            const authenticationResult = await requestValidators.authenticate(srcReq);
-            if (!authenticationResult.user?.id) {
-              console.log("[EXULU] failed authentication result", authenticationResult);
-              reject(authenticationResult.message);
-              return;
-            }
-            console.log("[EXULU] Authenticated call", authenticationResult.user?.email);
-            const { db: db3 } = await postgresClient();
-            let query = db3("agents");
-            query.select("*");
-            query = applyAccessControl(agentsSchema2(), authenticationResult.user, query);
-            query.where({ id: srcReq.params.id });
-            const agent = await query.first();
-            if (!agent) {
-              reject(new Error("Agent with id " + srcReq.params.id + " not found."));
-              return;
-            }
-            console.log("[EXULU] Agent loaded", agent.name);
-            const backend = agents.find((x) => x.id === agent.backend);
-            if (!process.env.NEXTAUTH_SECRET) {
-              reject(new Error("Missing NEXTAUTH_SECRET"));
-              return;
-            }
-            if (!agent.providerapikey) {
-              reject(new Error("API Key not set for agent"));
-              return;
-            }
-            const variableName = agent.providerapikey;
-            const variable = await db3.from("variables").where({ name: variableName }).first();
-            console.log("[EXULU] Variable loaded", variable);
-            let providerapikey = variable.value;
-            if (!variable.encrypted) {
-              reject(new Error("API Key not encrypted for agent"));
-              return;
-            }
-            if (variable.encrypted) {
-              const bytes = import_crypto_js3.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-              providerapikey = bytes.toString(import_crypto_js3.default.enc.Utf8);
-            }
-            console.log("[EXULU] Provider API key", providerapikey);
-            proxyReqOpts.headers["x-api-key"] = providerapikey;
-            proxyReqOpts.rejectUnauthorized = false;
-            delete proxyReqOpts.headers["provider"];
-            const url = new URL("https://api.anthropic.com");
-            proxyReqOpts.headers["host"] = url.host;
-            proxyReqOpts.headers["anthropic-version"] = "2023-06-01";
-            console.log("[EXULU] Proxy request headers", proxyReqOpts.headers);
-            resolve(proxyReqOpts);
-          } catch (error) {
-            console.error("[EXULU] Proxy error", error);
-            reject(error);
-          }
-        });
-      }
-    }
-  ));
   app.get("/config", async (req, res) => {
     res.status(200).json({
       message: "Config fetched successfully.",
@@ -6258,8 +6503,34 @@ var createCustomAnthropicStreamingMessage = (message) => {
 var import_ioredis = __toESM(require("ioredis"), 1);
 var import_bullmq5 = require("bullmq");
 var import_api2 = require("@opentelemetry/api");
+var import_uuid3 = require("uuid");
+var import_ai3 = require("ai");
+var import_crypto_js4 = __toESM(require("crypto-js"), 1);
+
+// src/registry/log-metadata.ts
+function logMetadata(id, additionalMetadata) {
+  return {
+    __logMetadata: true,
+    id,
+    ...additionalMetadata
+  };
+}
+
+// src/registry/workers.ts
 var redisConnection;
-var createWorkers = async (queues2, config, contexts, tracer) => {
+var createWorkers = async (agents, queues3, config, contexts, evals, tools, tracer) => {
+  console.log(`
+        \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2557      \u2588\u2588\u2557   \u2588\u2588\u2557
+        \u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u255A\u2588\u2588\u2557\u2588\u2588\u2554\u255D\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+        \u2588\u2588\u2588\u2588\u2588\u2557   \u255A\u2588\u2588\u2588\u2554\u255D \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+        \u2588\u2588\u2554\u2550\u2550\u255D   \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+        \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2554\u255D \u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D
+        \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D   
+        Intelligence Management Platform - Workers
+    
+        `);
+  console.log("[EXULU] creating workers for " + queues3?.length + " queues.");
+  console.log("[EXULU] queues", queues3.map((q) => q.queue.name));
   if (!redisServer.host || !redisServer.port) {
     console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
     throw new Error("No redis server configured in the environment, so cannot start worker.");
@@ -6267,80 +6538,333 @@ var createWorkers = async (queues2, config, contexts, tracer) => {
   if (!redisConnection) {
     redisConnection = new import_ioredis.default({
       ...redisServer,
+      enableOfflineQueue: true,
+      retryStrategy: function(times) {
+        return Math.max(Math.min(Math.exp(times), 2e4), 1e3);
+      },
       maxRetriesPerRequest: null
     });
   }
-  const workers = queues2.map((queue) => {
-    console.log(`[EXULU] creating worker for queue ${queue}.`);
+  const workers = queues3.map((queue) => {
+    console.log(`[EXULU] creating worker for queue ${queue.queue.name}.`);
     const worker = new import_bullmq5.Worker(
-      `${queue}`,
+      `${queue.queue.name}`,
       async (bullmqJob) => {
+        console.log("[EXULU] starting execution for job", logMetadata(bullmqJob.name, {
+          name: bullmqJob.name,
+          status: await bullmqJob.getState(),
+          type: bullmqJob.data.type
+        }));
         const { db: db3 } = await postgresClient();
-        try {
-          const data = bullmqJob.data;
-          bullmq.validate(bullmqJob.id, data);
-          if (data.type === "embedder") {
-            const context = contexts.find((context2) => context2.id === data.context);
-            if (!context) {
-              throw new Error(`Context ${data.context} not found in the registry.`);
-            }
-            if (!data.embedder) {
-              throw new Error(`No embedder set for embedder job.`);
-            }
-            const embedder = contexts.find((context2) => context2.embedder?.id === data.embedder);
-            if (!embedder) {
-              throw new Error(`Embedder ${data.embedder} not found in the registry.`);
-            }
-            const result = await context.createAndUpsertEmbeddings(data.inputs, config, data.user, {
-              label: embedder.name,
-              trigger: data.trigger
-            }, data.role, bullmqJob.id);
-            return result;
-          }
-          if (data.type === "processor") {
-            const context = contexts.find((context2) => context2.id === data.context);
-            if (!context) {
-              throw new Error(`Context ${data.context} not found in the registry.`);
-            }
-            const field = context.fields.find((field2) => field2.name === data.inputs.field);
-            if (!field) {
-              throw new Error(`Field ${data.inputs.field} not found in the context ${data.context}.`);
-            }
-            if (!field.processor) {
-              throw new Error(`Processor not set for field ${data.inputs.field} in the context ${data.context}.`);
+        const data = bullmqJob.data;
+        const timeoutMs = data.timeoutInSeconds * 1e3;
+        const timeoutPromise = new Promise((_, reject) => {
+          setTimeout(() => {
+            reject(new Error(`Timeout for job ${bullmqJob.id} reached after ${data.timeoutInSeconds}s`));
+          }, timeoutMs);
+        });
+        const workPromise = (async () => {
+          try {
+            bullmq.validate(bullmqJob.id, data);
+            if (data.type === "embedder") {
+              console.log("[EXULU] running an embedder job.", logMetadata(bullmqJob.name));
+              const label = `embedder-${bullmqJob.name}`;
+              await db3.from("job_results").insert({
+                job_id: bullmqJob.id,
+                label,
+                state: await bullmqJob.getState(),
+                result: null,
+                metadata: {}
+              });
+              const context = contexts.find((context2) => context2.id === data.context);
+              if (!context) {
+                throw new Error(`Context ${data.context} not found in the registry.`);
+              }
+              if (!data.embedder) {
+                throw new Error(`No embedder set for embedder job.`);
+              }
+              const embedder = contexts.find((context2) => context2.embedder?.id === data.embedder);
+              if (!embedder) {
+                throw new Error(`Embedder ${data.embedder} not found in the registry.`);
+              }
+              const result = await context.createAndUpsertEmbeddings(data.inputs, config, data.user, {
+                label: embedder.name,
+                trigger: data.trigger
+              }, data.role, bullmqJob.id);
+              return {
+                result,
+                metadata: {}
+              };
             }
-            const exuluStorage = new ExuluStorage({ config });
-            if (!data.user) {
-              throw new Error(`User not set for processor job.`);
+            if (data.type === "processor") {
+              console.log("[EXULU] running a processor job.", logMetadata(bullmqJob.name));
+              const label = `processor-${bullmqJob.name}`;
+              await db3.from("job_results").insert({
+                job_id: bullmqJob.id,
+                label,
+                state: await bullmqJob.getState(),
+                result: null,
+                metadata: {}
+              });
+              const context = contexts.find((context2) => context2.id === data.context);
+              if (!context) {
+                throw new Error(`Context ${data.context} not found in the registry.`);
+              }
+              const field = context.fields.find((field2) => field2.name === data.inputs.field);
+              if (!field) {
+                throw new Error(`Field ${data.inputs.field} not found in the context ${data.context}.`);
+              }
+              if (!field.processor) {
+                throw new Error(`Processor not set for field ${data.inputs.field} in the context ${data.context}.`);
+              }
+              const exuluStorage = new ExuluStorage({ config });
+              if (!data.user) {
+                throw new Error(`User not set for processor job.`);
+              }
+              if (!data.role) {
+                throw new Error(`Role not set for processor job.`);
+              }
+              const result = await field.processor.execute({
+                item: data.inputs,
+                user: data.user,
+                role: data.role,
+                utils: {
+                  storage: exuluStorage,
+                  items: {
+                    update: context.updateItem,
+                    create: context.createItem,
+                    delete: context.deleteItem
+                  }
+                },
+                config
+              });
+              return {
+                result,
+                metadata: {}
+              };
             }
-            if (!data.role) {
-              throw new Error(`Role not set for processor job.`);
+            if (data.type === "eval_run") {
+              console.log("[EXULU] running an eval run job.", logMetadata(bullmqJob.name));
+              const label = `eval-run-${data.eval_run_id}-${data.test_case_id}`;
+              const existingResult = await db3.from("job_results").where({ label }).first();
+              if (existingResult) {
+                await db3.from("job_results").where({ label }).update({
+                  job_id: bullmqJob.id,
+                  label,
+                  state: await bullmqJob.getState(),
+                  result: null,
+                  metadata: {},
+                  tries: existingResult.tries + 1
+                });
+              } else {
+                await db3.from("job_results").insert({
+                  job_id: bullmqJob.id,
+                  label,
+                  state: await bullmqJob.getState(),
+                  result: null,
+                  metadata: {},
+                  tries: 1
+                });
+              }
+              const {
+                agentInstance,
+                backend: agentBackend,
+                user,
+                evalRun,
+                testCase,
+                messages: inputMessages
+              } = await validateEvalPayload(data, agents);
+              const retries = 3;
+              let attempts = 0;
+              const promise = new Promise(async (resolve, reject) => {
+                while (attempts < retries) {
+                  try {
+                    const messages2 = await processUiMessagesFlow({
+                      agents,
+                      agentInstance,
+                      agentBackend,
+                      inputMessages,
+                      contexts,
+                      user,
+                      tools,
+                      config
+                    });
+                    resolve(messages2);
+                    break;
+                  } catch (error) {
+                    console.error(`[EXULU] error processing UI messages flow for agent ${agentInstance.name} (${agentInstance.id}).`, logMetadata(bullmqJob.name, {
+                      error: error instanceof Error ? error.message : String(error)
+                    }));
+                    attempts++;
+                    if (attempts >= retries) {
+                      reject(error);
+                    }
+                    await new Promise((resolve2) => setTimeout(resolve2, 2e3));
+                  }
+                }
+              });
+              const result = await promise;
+              const messages = result.messages;
+              const metadata = result.metadata;
+              const evalFunctions = evalRun.eval_functions;
+              let evalFunctionResults = [];
+              for (const evalFunction of evalFunctions) {
+                const evalMethod = evals.find((e) => e.id === evalFunction.id);
+                if (!evalMethod) {
+                  throw new Error(`Eval function ${evalFunction.id} not found in the registry, check your code and make sure the eval function is registered correctly.`);
+                }
+                let result2;
+                if (evalMethod.queue) {
+                  const queue2 = await evalMethod.queue;
+                  const jobData = {
+                    ...data,
+                    type: "eval_function",
+                    eval_functions: [{
+                      id: evalFunction.id,
+                      config: evalFunction.config || {}
+                    }],
+                    // updating the input messages with the messages we want to run the eval 
+                    // function on, which are the output messages from the agent.
+                    inputs: messages
+                  };
+                  const redisId = (0, import_uuid3.v4)();
+                  const job = await queue2.queue.add("eval_function", jobData, {
+                    jobId: redisId,
+                    // Setting it to 3 as a sensible default, as
+                    // many AI services are quite unstable.
+                    attempts: queue2.retries || 3,
+                    // todo make this configurable?
+                    removeOnComplete: 5e3,
+                    removeOnFail: 1e4,
+                    backoff: queue2.backoff || {
+                      type: "exponential",
+                      delay: 2e3
+                    }
+                  });
+                  if (!job.id) {
+                    throw new Error(`Tried to add job to queue ${queue2.queue.name} but failed to get the job ID.`);
+                  }
+                  result2 = await pollJobResult({ queue: queue2, jobId: job.id });
+                  const evalFunctionResult = {
+                    test_case_id: testCase.id,
+                    eval_run_id: evalRun.id,
+                    eval_function_id: evalFunction.id,
+                    result: result2 || 0
+                  };
+                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata(bullmqJob.name, {
+                    result: result2 || 0
+                  }));
+                  evalFunctionResults.push(evalFunctionResult);
+                } else {
+                  result2 = await evalMethod.run(
+                    agentInstance,
+                    agentBackend,
+                    testCase,
+                    messages,
+                    evalFunction.config || {}
+                  );
+                  const evalFunctionResult = {
+                    test_case_id: testCase.id,
+                    eval_run_id: evalRun.id,
+                    eval_function_id: evalFunction.id,
+                    result: result2 || 0
+                  };
+                  evalFunctionResults.push(evalFunctionResult);
+                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata(bullmqJob.name, {
+                    result: result2 || 0
+                  }));
+                }
+              }
+              const scores = evalFunctionResults.map((result2) => result2.result);
+              let score = 0;
+              switch (data.scoring_method) {
+                case "median":
+                  score = getMedian(scores);
+                  break;
+                case "average":
+                  score = getAverage(scores);
+                  break;
+                case "sum":
+                  score = getSum(scores);
+                  break;
+              }
+              return {
+                result: score,
+                metadata: {
+                  ...evalFunctionResults,
+                  ...metadata
+                }
+              };
             }
-            const result = await field.processor.execute({
-              item: data.inputs,
-              user: data.user,
-              role: data.role,
-              utils: {
-                storage: exuluStorage,
-                items: {
-                  update: context.updateItem
+            if (data.type === "eval_function") {
+              console.log("[EXULU] running an eval function job.", logMetadata(bullmqJob.name));
+              if (data.eval_functions?.length !== 1) {
+                throw new Error(`Expected 1 eval function for eval function job, got ${data.eval_functions?.length}.`);
+              }
+              const label = `eval-function-${data.eval_run_id}-${data.test_case_id}-${data.eval_functions?.[0]?.id}`;
+              const existingResult = await db3.from("job_results").where({ label }).first();
+              if (existingResult) {
+                await db3.from("job_results").where({ label }).update({
+                  job_id: bullmqJob.id,
+                  label,
+                  state: await bullmqJob.getState(),
+                  result: null,
+                  metadata: {},
+                  tries: existingResult.tries + 1
+                });
+              } else {
+                await db3.from("job_results").insert({
+                  job_id: bullmqJob.id,
+                  label,
+                  state: await bullmqJob.getState(),
+                  result: null,
+                  metadata: {},
+                  tries: 1
+                });
+              }
+              const {
+                evalRun,
+                agentInstance,
+                backend,
+                testCase,
+                messages: inputMessages
+              } = await validateEvalPayload(data, agents);
+              const evalFunctions = evalRun.eval_functions;
+              let result;
+              for (const evalFunction of evalFunctions) {
+                const evalMethod = evals.find((e) => e.id === evalFunction.id);
+                if (!evalMethod) {
+                  throw new Error(`Eval function ${evalFunction.id} not found in the registry, check your code and make sure the eval function is registered correctly.`);
                 }
-              },
-              config
-            });
-            return result;
+                result = await evalMethod.run(
+                  agentInstance,
+                  backend,
+                  testCase,
+                  inputMessages,
+                  evalFunction.config || {}
+                );
+                console.log(`[EXULU] eval function ${evalFunction.id} result: ${result}`, logMetadata(bullmqJob.name, {
+                  result: result || 0
+                }));
+              }
+              return {
+                result,
+                metadata: {}
+              };
+            }
+            throw new Error(`Invalid job type: ${data.type} for job ${bullmqJob.name}.`);
+          } catch (error) {
+            console.error(`[EXULU] job failed.`, error instanceof Error ? error.message : String(error));
+            throw error;
           }
-        } catch (error) {
-          await db3.from("jobs").where({ redis: bullmqJob.id }).update({
-            status: "failed",
-            finishedAt: /* @__PURE__ */ new Date(),
-            error: error instanceof Error ? error.message : String(error)
-          });
-          throw new Error(error instanceof Error ? error.message : String(error));
-        }
+        })();
+        return Promise.race([workPromise, timeoutPromise]);
       },
       {
+        autorun: true,
         connection: redisConnection,
+        removeOnComplete: { count: 1e3 },
+        removeOnFail: { count: 5e3 },
         ...queue.ratelimit && {
           limiter: {
             max: queue.ratelimit,
@@ -6349,22 +6873,312 @@ var createWorkers = async (queues2, config, contexts, tracer) => {
         }
       }
     );
-    worker.on("completed", (job, returnvalue) => {
-      console.log(`[EXULU] completed job ${job.id}.`, returnvalue);
+    worker.on("completed", async (job, returnvalue) => {
+      console.log(`[EXULU] completed job ${job.id}.`, logMetadata(job.name, {
+        result: returnvalue
+      }));
+      const { db: db3 } = await postgresClient();
+      await db3.from("job_results").where({ job_id: job.id }).update({
+        state: JOB_STATUS_ENUM.completed,
+        result: returnvalue.result,
+        metadata: returnvalue.metadata
+      });
     });
-    worker.on("failed", (job, error, prev) => {
+    worker.on("failed", async (job, error, prev) => {
       if (job?.id) {
-        console.error(`[EXULU] failed job ${job.id}.`);
+        const { db: db3 } = await postgresClient();
+        console.error(`[EXULU] failed job ${job.id}.`, error);
+        await db3.from("job_results").where({ job_id: job.id }).update({
+          state: JOB_STATUS_ENUM.failed,
+          error
+        });
+        return;
       }
-      console.error(`[EXULU] job error.`, error);
+      console.error(`[EXULU] job failed.`, job?.name ? logMetadata(job.name, {
+        error: error instanceof Error ? error.message : String(error)
+      }) : error);
+    });
+    worker.on("error", (error) => {
+      console.error(`[EXULU] worker error.`, error);
     });
     worker.on("progress", (job, progress) => {
-      console.log(`[EXULU] job progress ${job.id}.`, progress);
+      console.log(`[EXULU] job progress ${job.id}.`, logMetadata(job.name, {
+        progress
+      }));
     });
+    const gracefulShutdown = async (signal) => {
+      console.log(`Received ${signal}, closing server...`);
+      await worker.close();
+      process.exit(0);
+    };
+    process.on("SIGINT", () => gracefulShutdown("SIGINT"));
+    process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
     return worker;
   });
   return workers;
 };
+var validateEvalPayload = async (data, agents) => {
+  if (!data.eval_run_id) {
+    throw new Error(`No eval run ID set for eval job.`);
+  }
+  if (!data.test_case_id) {
+    throw new Error(`No test case ID set for eval job.`);
+  }
+  if (!data.user) {
+    throw new Error(`No user set for eval job.`);
+  }
+  if (!data.role) {
+    throw new Error(`No role set for eval job.`);
+  }
+  if (!data.agent_id) {
+    throw new Error(`No agent ID set for eval job.`);
+  }
+  if (!data.inputs?.length) {
+    throw new Error(`No inputs set for eval job, expected array of UIMessage objects.`);
+  }
+  const { db: db3 } = await postgresClient();
+  const evalRun = await db3.from("eval_runs").where({ id: data.eval_run_id }).first();
+  if (!evalRun) {
+    throw new Error(`Eval run ${data.eval_run_id} not found in the database.`);
+  }
+  const agentInstance = await loadAgent(evalRun.agent_id);
+  if (!agentInstance) {
+    throw new Error(`Agent ${evalRun.agent_id} not found in the database.`);
+  }
+  const backend = agents.find((a) => a.id === agentInstance.backend);
+  if (!backend) {
+    throw new Error(`Backend ${agentInstance.backend} not found in the database.`);
+  }
+  const user = await db3.from("users").where({ id: data.user }).first();
+  if (!user) {
+    throw new Error(`User ${data.user} not found in the database.`);
+  }
+  const testCase = await db3.from("test_cases").where({ id: data.test_case_id }).first();
+  if (!testCase) {
+    throw new Error(`Test case ${data.test_case_id} not found in the database.`);
+  }
+  return {
+    agentInstance,
+    backend,
+    user,
+    testCase,
+    evalRun,
+    messages: data.inputs
+  };
+};
+var pollJobResult = async ({ queue, jobId }) => {
+  let attempts = 0;
+  let timeoutInSeconds = queue.timeoutInSeconds || 180;
+  const startTime = Date.now();
+  let result;
+  while (true) {
+    attempts++;
+    const job = await import_bullmq5.Job.fromId(queue.queue, jobId);
+    if (!job) {
+      await new Promise((resolve) => setTimeout(resolve, 2e3));
+      continue;
+    }
+    const elapsedTime = Date.now() - startTime;
+    if (elapsedTime > timeoutInSeconds * 1e3) {
+      throw new Error(`Job ${job.id} timed out after ${timeoutInSeconds} seconds for job eval function job ${job.name}.`);
+    }
+    console.log(`[EXULU] polling eval function job ${job.name} for state... (attempt ${attempts})`);
+    const jobState = await job.getState();
+    console.log(`[EXULU] eval function job ${job.name} state: ${jobState}`);
+    if (jobState === "failed") {
+      throw new Error(`Job ${job.name} (${job.id}) failed with error: ${job.failedReason}.`);
+    }
+    if (jobState === "completed") {
+      console.log(`[EXULU] eval function job ${job.name} completed, getting result from database...`);
+      const { db: db3 } = await postgresClient();
+      const entry = await db3.from("job_results").where({ job_id: job.id }).first();
+      result = entry?.result;
+      if (result === void 0 || result === null || result === "") {
+        throw new Error(`Eval function ${job.id} result not found in database for job eval function job ${job.name}.`);
+      }
+      console.log(`[EXULU] eval function ${job.id} result: ${result}`);
+      break;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 2e3));
+  }
+  return result;
+};
+var processUiMessagesFlow = async ({
+  agents,
+  agentInstance,
+  agentBackend,
+  inputMessages,
+  contexts,
+  user,
+  tools,
+  config
+}) => {
+  console.log("[EXULU] processing UI messages flow for agent.");
+  console.log("[EXULU] input messages", inputMessages);
+  console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
+  const disabledTools = [];
+  let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
+  console.log("[EXULU] enabled tools", enabledTools?.map((x) => x.name + " (" + x.id + ")"));
+  const variableName = agentInstance.providerapikey;
+  const { db: db3 } = await postgresClient();
+  const variable = await db3.from("variables").where({ name: variableName }).first();
+  if (!variable) {
+    throw new Error(`Provider API key variable not found for agent ${agentInstance.name} (${agentInstance.id}).`);
+  }
+  let providerapikey = variable.value;
+  if (!variable.encrypted) {
+    throw new Error(`Provider API key variable not encrypted for agent ${agentInstance.name} (${agentInstance.id}), for security reasons you are only allowed to use encrypted variables for provider API keys.`);
+  }
+  if (variable.encrypted) {
+    const bytes = import_crypto_js4.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+    providerapikey = bytes.toString(import_crypto_js4.default.enc.Utf8);
+  }
+  const messagesWithoutPlaceholder = inputMessages.filter(
+    (message) => message.metadata?.type !== "placeholder"
+  );
+  console.log("[EXULU] messages without placeholder", messagesWithoutPlaceholder);
+  let index = 0;
+  let messageHistory = {
+    messages: [],
+    metadata: {
+      tokens: {
+        totalTokens: 0,
+        reasoningTokens: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        cachedInputTokens: 0
+      },
+      duration: 0
+    }
+  };
+  for (const currentMessage of messagesWithoutPlaceholder) {
+    console.log("[EXULU] running through the conversation");
+    console.log("[EXULU] current index", index);
+    console.log("[EXULU] current message", currentMessage);
+    console.log("[EXULU] message history", messageHistory);
+    const statistics = {
+      label: agentInstance.name,
+      trigger: "agent"
+    };
+    messageHistory = await new Promise(async (resolve, reject) => {
+      const startTime = Date.now();
+      try {
+        const result = await agentBackend.generateStream({
+          contexts,
+          user,
+          instructions: agentInstance.instructions,
+          session: void 0,
+          previousMessages: messageHistory.messages,
+          message: currentMessage,
+          currentTools: enabledTools,
+          allExuluTools: tools,
+          providerapikey,
+          toolConfigs: agentInstance.tools,
+          exuluConfig: config
+        });
+        console.log("[EXULU] consuming stream for agent.");
+        const stream = result.stream.toUIMessageStream({
+          messageMetadata: ({ part }) => {
+            console.log("[EXULU] part", part.type);
+            if (part.type === "finish") {
+              return {
+                totalTokens: part.totalUsage.totalTokens,
+                reasoningTokens: part.totalUsage.reasoningTokens,
+                inputTokens: part.totalUsage.inputTokens,
+                outputTokens: part.totalUsage.outputTokens,
+                cachedInputTokens: part.totalUsage.cachedInputTokens
+              };
+            }
+          },
+          originalMessages: result.originalMessages,
+          sendReasoning: true,
+          sendSources: true,
+          onError: (error) => {
+            console.error("[EXULU] Ui message stream error.", error);
+            reject(error);
+            return `Ui message stream error: ${error instanceof Error ? error.message : String(error)}`;
+          },
+          onFinish: async ({ messages, isContinuation, responseMessage }) => {
+            const metadata = messages[messages.length - 1]?.metadata;
+            console.log("[EXULU] Stream finished with messages:", messages);
+            console.log("[EXULU] Stream metadata", metadata);
+            await Promise.all([
+              updateStatistic({
+                name: "count",
+                label: statistics.label,
+                type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+                trigger: statistics.trigger,
+                count: 1,
+                user: user.id,
+                role: user?.role?.id
+              }),
+              ...metadata?.inputTokens ? [
+                updateStatistic({
+                  name: "inputTokens",
+                  label: statistics.label,
+                  type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+                  trigger: statistics.trigger,
+                  count: metadata?.inputTokens,
+                  user: user.id,
+                  role: user?.role?.id
+                })
+              ] : [],
+              ...metadata?.outputTokens ? [
+                updateStatistic({
+                  name: "outputTokens",
+                  label: statistics.label,
+                  type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+                  trigger: statistics.trigger,
+                  count: metadata?.outputTokens
+                })
+              ] : []
+            ]);
+            resolve({
+              messages,
+              metadata: {
+                tokens: {
+                  totalTokens: messageHistory.metadata.tokens.totalTokens + metadata?.totalTokens,
+                  reasoningTokens: messageHistory.metadata.tokens.reasoningTokens + metadata?.reasoningTokens,
+                  inputTokens: messageHistory.metadata.tokens.inputTokens + metadata?.inputTokens,
+                  outputTokens: messageHistory.metadata.tokens.outputTokens + metadata?.outputTokens,
+                  cachedInputTokens: messageHistory.metadata.tokens.cachedInputTokens + metadata?.cachedInputTokens
+                },
+                duration: messageHistory.metadata.duration + (Date.now() - startTime)
+              }
+            });
+          }
+        });
+        for await (const message of stream) {
+          console.log("[EXULU] message", message);
+        }
+      } catch (error) {
+        console.error(`[EXULU] error generating stream for agent ${agentInstance.name} (${agentInstance.id}).`, error);
+        reject(error);
+      }
+    });
+    index++;
+  }
+  console.log("[EXULU] finished processing UI messages flow for agent, messages result", messageHistory);
+  return messageHistory;
+};
+function getMedian(arr) {
+  if (arr.length === 0) return 0;
+  const sortedArr = arr.slice().sort((a, b) => a - b);
+  const mid = Math.floor(sortedArr.length / 2);
+  if (sortedArr.length % 2 !== 0) {
+    return sortedArr[mid];
+  } else {
+    return (sortedArr[mid - 1] + sortedArr[mid]) / 2;
+  }
+}
+function getSum(arr) {
+  if (arr.length === 0) return 0;
+  return arr.reduce((a, b) => a + b, 0);
+}
+function getAverage(arr) {
+  if (arr.length === 0) return 0;
+  return arr.reduce((a, b) => a + b, 0) / arr.length;
+}
 
 // src/mcp/index.ts
 var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
@@ -6517,7 +7331,6 @@ var claudeOpus4Agent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 2e5,
   config: {
     name: `CLAUDE-OPUS-4`,
@@ -6552,7 +7365,6 @@ var claudeSonnet4Agent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 2e5,
   config: {
     name: `CLAUDE-SONNET-4`,
@@ -6580,7 +7392,6 @@ var claudeSonnet45Agent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 2e5,
   config: {
     name: `CLAUDE-SONNET-4.5`,
@@ -6611,7 +7422,6 @@ var gpt5proAgent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 4e5,
   config: {
     name: `GPT-5-PRO`,
@@ -6639,7 +7449,6 @@ var gpt5CodexAgent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 4e5,
   config: {
     name: `GPT-5-CODEX`,
@@ -6667,7 +7476,6 @@ var gpt5MiniAgent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 4e5,
   config: {
     name: `GPT-5-MINI`,
@@ -6702,7 +7510,6 @@ var gpt5agent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 4e5,
   config: {
     name: `GPT-5`,
@@ -6737,7 +7544,6 @@ var gpt5NanoAgent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 4e5,
   config: {
     name: `GPT-5-NANO`,
@@ -6765,7 +7571,6 @@ var gpt41Agent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 1047576,
   config: {
     name: `GPT-4.1`,
@@ -6793,7 +7598,6 @@ var gpt41MiniAgent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 1047576,
   config: {
     name: `GPT-4.1-MINI`,
@@ -6821,7 +7625,6 @@ var gpt4oAgent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 128e3,
   config: {
     name: `Default agent`,
@@ -6849,7 +7652,6 @@ var gpt4oMiniAgent = new ExuluAgent2({
     audio: [],
     video: []
   },
-  evals: [],
   maxContextLength: 128e3,
   config: {
     name: `GPT-4O-MINI`,
@@ -6940,6 +7742,55 @@ var outputsContext = new ExuluContext({
 // src/registry/index.ts
 var import_winston2 = __toESM(require("winston"), 1);
 var import_util = __toESM(require("util"), 1);
+
+// src/templates/evals/index.ts
+var import_zod3 = require("zod");
+var llmAsJudgeEval = new ExuluEval2({
+  id: "llm_as_judge",
+  name: "LLM as Judge",
+  description: "Evaluate the output of the LLM as a judge.",
+  execute: async ({ agent, backend, messages, testCase, config }) => {
+    console.log("[EXULU] running llm as judge eval", { agent, backend, messages, testCase, config });
+    let prompt = config?.prompt;
+    if (!prompt) {
+      console.error("[EXULU] prompt is required.");
+      throw new Error("Prompt is required.");
+    }
+    const lastMessage = messages[messages.length - 1]?.parts?.filter((part) => part.type === "text").map((part) => part.text).join("\n");
+    console.log("[EXULU] last message", lastMessage);
+    if (!lastMessage) {
+      return 0;
+    }
+    prompt = prompt.replace("{actual_output}", lastMessage);
+    prompt = prompt.replace("{expected_output}", testCase.expected_output);
+    if (!agent.providerapikey) {
+      throw new Error(`Provider API key for agent ${agent.name} is required, variable name is ${agent.providerapikey} but it is not set.`);
+    }
+    const providerapikey = await ExuluVariables.get(agent.providerapikey);
+    console.log("[EXULU] prompt", prompt);
+    const response = await backend.generateSync({
+      prompt,
+      outputSchema: import_zod3.z.object({
+        score: import_zod3.z.number().min(0).max(100).describe("The score between 0 and 100.")
+      }),
+      providerapikey
+    });
+    console.log("[EXULU] response", response);
+    const score = parseFloat(response.score);
+    if (isNaN(score)) {
+      throw new Error(`Generated score from llm as a judge eval is not a number: ${response.score}`);
+    }
+    return score;
+  },
+  config: [{
+    name: "prompt",
+    description: "The prompt to send to the LLM as a judge, make sure to instruct the LLM to output a numerical score between 0 and 100. Add {actual_output} to the prompt to replace with the last message content, and {expected_output} to replace with the expected output."
+  }],
+  queue: queues.register("llm_as_judge", 1, 1).use(),
+  llm: true
+});
+
+// src/registry/index.ts
 var isDev = process.env.NODE_ENV !== "production";
 var consoleTransport = new import_winston2.default.transports.Console({
   format: isDev ? import_winston2.default.format.combine(
@@ -6950,6 +7801,20 @@ var consoleTransport = new import_winston2.default.transports.Console({
     })
   ) : import_winston2.default.format.json()
 });
+var formatArg = (arg) => typeof arg === "object" ? import_util.default.inspect(arg, { depth: null, colors: isDev }) : String(arg);
+var createLogMethod = (logger, logLevel) => {
+  return (...args) => {
+    const lastArg = args[args.length - 1];
+    let metadata = void 0;
+    let messageArgs = args;
+    if (lastArg && typeof lastArg === "object" && lastArg.__logMetadata === true) {
+      metadata = lastArg;
+      messageArgs = args.slice(0, -1);
+    }
+    const message = messageArgs.map(formatArg).join(" ");
+    logger[logLevel](message, metadata);
+  };
+};
 var isValidPostgresName = (id) => {
   const regex = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
   const isValid = regex.test(id);
@@ -6959,6 +7824,7 @@ var isValidPostgresName = (id) => {
 var ExuluApp = class {
   _agents = [];
   _config;
+  _evals = [];
   _queues = [];
   _contexts = {};
   _tools = [];
@@ -6967,7 +7833,11 @@ var ExuluApp = class {
   }
   // Factory function so we can async 
   // initialize the MCP server if needed.
-  create = async ({ contexts, agents, config, tools }) => {
+  create = async ({ contexts, agents, config, tools, evals }) => {
+    this._evals = [
+      llmAsJudgeEval,
+      ...evals ?? []
+    ];
     this._contexts = {
       ...contexts,
       codeStandardsContext,
@@ -7019,11 +7889,12 @@ var ExuluApp = class {
       console.error(`%c[EXULU] Invalid ID found for a context, tool or agent: ${invalid.map((x) => x.id).join(", ")}. An ID must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or underscores and be a max length of 80 characters and at least 5 characters long.`, "color: orange; font-weight: bold; \n \n");
       throw new Error(`Invalid ID found for a context, tool or agent: ${invalid.map((x) => x.id).join(", ")}. An ID must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or underscores and be a max length of 80 characters and at least 5 characters long.`);
     }
-    const contextsArray = Object.values(contexts || {});
     const queueSet = /* @__PURE__ */ new Set();
-    for (const context of contextsArray) {
-      if (context.embedder?.queue) {
-        queueSet.add(await context.embedder.queue);
+    if (redisServer.host?.length && redisServer.port?.length) {
+      queues.register(global_queues.eval_runs, 1, 1);
+      for (const queue of queues.list.values()) {
+        const config2 = await queue.use();
+        queueSet.add(config2);
       }
     }
     this._queues = [...new Set(queueSet.values())];
@@ -7099,7 +7970,7 @@ var ExuluApp = class {
   };
   bullmq = {
     workers: {
-      create: async () => {
+      create: async (queues3) => {
         if (!this._config) {
           throw new Error("Config not initialized, make sure to call await ExuluApp.create() first when starting your server.");
         }
@@ -7112,16 +7983,22 @@ var ExuluApp = class {
           enableOtel: this._config?.workers?.telemetry?.enabled ?? false,
           transports
         });
-        const formatArg = (arg) => typeof arg === "object" ? import_util.default.inspect(arg, { depth: null, colors: isDev }) : String(arg);
-        console.log = (...args) => logger.info(args.map(formatArg).join(" "));
-        console.info = (...args) => logger.info(args.map(formatArg).join(" "));
-        console.warn = (...args) => logger.warn(args.map(formatArg).join(" "));
-        console.error = (...args) => logger.error(args.map(formatArg).join(" "));
-        console.debug = (...args) => logger.debug(args.map(formatArg).join(" "));
+        console.log = createLogMethod(logger, "info");
+        console.info = createLogMethod(logger, "info");
+        console.warn = createLogMethod(logger, "warn");
+        console.error = createLogMethod(logger, "error");
+        console.debug = createLogMethod(logger, "debug");
+        let filteredQueues = this._queues;
+        if (queues3) {
+          filteredQueues = filteredQueues.filter((q) => queues3.includes(q.queue.name));
+        }
         return await createWorkers(
-          this._queues,
+          this._agents,
+          filteredQueues,
           this._config,
           Object.values(this._contexts ?? {}),
+          this._evals,
+          this._tools,
           tracer
         );
       }
@@ -7138,16 +8015,16 @@ var ExuluApp = class {
         if (this._config?.telemetry?.enabled) {
           tracer = import_api4.trace.getTracer("exulu", "1.0.0");
         }
+        const transports = this._config?.logger?.winston?.transports ?? [consoleTransport];
         const logger = logger_default({
           enableOtel: this._config?.telemetry?.enabled ?? false,
-          transports: this._config?.logger?.winston?.transports ?? [consoleTransport]
+          transports
         });
-        const formatArg = (arg) => typeof arg === "object" ? import_util.default.inspect(arg, { depth: null, colors: isDev }) : String(arg);
-        console.log = (...args) => logger.info(args.map(formatArg).join(" "));
-        console.info = (...args) => logger.info(args.map(formatArg).join(" "));
-        console.warn = (...args) => logger.warn(args.map(formatArg).join(" "));
-        console.error = (...args) => logger.error(args.map(formatArg).join(" "));
-        console.debug = (...args) => logger.debug(args.map(formatArg).join(" "));
+        console.log = createLogMethod(logger, "info");
+        console.info = createLogMethod(logger, "info");
+        console.warn = createLogMethod(logger, "warn");
+        console.error = createLogMethod(logger, "error");
+        console.debug = createLogMethod(logger, "debug");
         if (!this._config) {
           throw new Error("Config not initialized, make sure to call await ExuluApp.create() first when starting your server.");
         }
@@ -7157,7 +8034,9 @@ var ExuluApp = class {
           this._tools,
           Object.values(this._contexts ?? {}),
           this._config,
-          tracer
+          this._evals,
+          tracer,
+          this._queues
         );
         if (this._config?.MCP.enabled) {
           const mcp = new ExuluMCP();
@@ -8386,7 +9265,8 @@ var {
   variablesSchema: variablesSchema3,
   workflowTemplatesSchema: workflowTemplatesSchema3,
   rbacSchema: rbacSchema3,
-  projectsSchema: projectsSchema3
+  projectsSchema: projectsSchema3,
+  jobResultsSchema: jobResultsSchema3
 } = coreSchemas.get();
 var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
   for (const field of fields) {
@@ -8420,6 +9300,7 @@ var up = async function(knex) {
     platformConfigurationsSchema3(),
     statisticsSchema3(),
     projectsSchema3(),
+    jobResultsSchema3(),
     rbacSchema3(),
     agentsSchema3(),
     variablesSchema3(),
@@ -8604,20 +9485,7 @@ var create = ({
 };
 
 // src/index.ts
-var import_crypto_js4 = __toESM(require("crypto-js"), 1);
-
-// types/enums/jobs.ts
-var JOB_STATUS_ENUM = {
-  completed: "completed",
-  failed: "failed",
-  delayed: "delayed",
-  active: "active",
-  waiting: "waiting",
-  paused: "paused",
-  stuck: "stuck"
-};
-
-// src/index.ts
+var import_crypto_js5 = __toESM(require("crypto-js"), 1);
 var ExuluJobs = {
   redis: redisClient,
   jobs: {
@@ -8654,8 +9522,8 @@ var ExuluVariables = {
       throw new Error(`Variable ${name} not found.`);
     }
     if (variable.encrypted) {
-      const bytes = import_crypto_js4.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-      variable.value = bytes.toString(import_crypto_js4.default.enc.Utf8);
+      const bytes = import_crypto_js5.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+      variable.value = bytes.toString(import_crypto_js5.default.enc.Utf8);
     }
     return variable.value;
   }
@@ -8791,5 +9659,6 @@ var ExuluChunkers = {
   ExuluTool,
   ExuluUtils,
   ExuluVariables,
-  db
+  db,
+  logMetadata
 });