@exulu/backend 0.3.13 → 1.2.0

package/dist/index.js

@@ -102,6 +102,47 @@ async function postgresClient() {
 }
 
 // src/postgres/core-schema.ts
+var agentMessagesSchema = {
+  name: {
+    plural: "agent_messages",
+    singular: "agent_message"
+  },
+  fields: [
+    {
+      name: "content",
+      type: "text"
+    },
+    {
+      name: "title",
+      type: "text"
+    },
+    {
+      name: "session",
+      type: "text"
+    }
+  ]
+};
+var agentSessionsSchema = {
+  name: {
+    plural: "agent_sessions",
+    singular: "agent_session"
+  },
+  fields: [
+    {
+      name: "agent",
+      type: "uuid"
+    },
+    {
+      name: "user",
+      // next auth stores users with id type SERIAL, so we need to use number
+      type: "number"
+    },
+    {
+      name: "title",
+      type: "text"
+    }
+  ]
+};
 var usersSchema = {
   name: {
     plural: "users",
@@ -159,14 +200,13 @@ var usersSchema = {
       name: "last_used",
       type: "date"
     },
+    {
+      name: "anthropic_token",
+      type: "text"
+    },
     {
       name: "role",
-      type: "reference",
-      references: {
-        table: "roles",
-        field: "id",
-        onDelete: "CASCADE"
-      }
+      type: "uuid"
     }
   ]
 };
@@ -212,10 +252,6 @@ var statisticsSchema = {
     {
       name: "total",
       type: "number"
-    },
-    {
-      name: "timeseries",
-      type: "json"
     }
   ]
 };
@@ -271,11 +307,11 @@ var evalResultsSchema = {
     },
     {
       name: "agent_id",
-      type: "text"
+      type: "uuid"
     },
     {
       name: "workflow_id",
-      type: "text"
+      type: "uuid"
     },
     {
       name: "eval_type",
@@ -291,50 +327,6 @@ var evalResultsSchema = {
     }
   ]
 };
-var threadsSchema = {
-  name: {
-    plural: "threads",
-    singular: "thread"
-  },
-  fields: [
-    {
-      name: "resourceId",
-      type: "text"
-    },
-    {
-      name: "title",
-      type: "text"
-    },
-    {
-      name: "metadata",
-      type: "text"
-    }
-  ]
-};
-var messagesSchema = {
-  name: {
-    plural: "messages",
-    singular: "message"
-  },
-  fields: [
-    {
-      name: "thread_id",
-      type: "text"
-    },
-    {
-      name: "content",
-      type: "text"
-    },
-    {
-      name: "role",
-      type: "text"
-    },
-    {
-      name: "type",
-      type: "text"
-    }
-  ]
-};
 var jobsSchema = {
   name: {
     plural: "jobs",
@@ -367,19 +359,20 @@ var jobsSchema = {
     },
     {
       name: "agent",
-      type: "text"
+      type: "uuid"
     },
     {
       name: "workflow",
-      type: "text"
+      type: "uuid"
     },
     {
       name: "user",
-      type: "text"
+      // next auth stores users with id type SERIAL, so we need to use number
+      type: "number"
     },
     {
       name: "item",
-      type: "text"
+      type: "uuid"
     },
     {
       name: "steps",
@@ -473,7 +466,7 @@ var mapType = (t, type, name, defaultValue) => {
     return;
   }
   if (type === "date") {
-    t.date(name);
+    t.timestamp(name);
     return;
   }
   if (type === "uuid") {
@@ -543,23 +536,44 @@ var generateApiKey = async (name, email) => {
 
 // src/postgres/init-db.ts
 var up = async function(knex) {
+  if (!await knex.schema.hasTable("agent_sessions")) {
+    await knex.schema.createTable("agent_sessions", (table) => {
+      table.uuid("id").primary().defaultTo(knex.fn.uuid());
+      table.timestamp("createdAt").defaultTo(knex.fn.now());
+      table.timestamp("updatedAt").defaultTo(knex.fn.now());
+      for (const field of agentSessionsSchema.fields) {
+        const { type, name, default: defaultValue } = field;
+        if (!type || !name) {
+          continue;
+        }
+        mapType(table, type, sanitizeName(name), defaultValue);
+      }
+    });
+  }
+  if (!await knex.schema.hasTable("agent_messages")) {
+    await knex.schema.createTable("agent_messages", (table) => {
+      table.uuid("id").primary().defaultTo(knex.fn.uuid());
+      table.timestamp("createdAt").defaultTo(knex.fn.now());
+      table.timestamp("updatedAt").defaultTo(knex.fn.now());
+      for (const field of agentMessagesSchema.fields) {
+        const { type, name, default: defaultValue } = field;
+        if (!type || !name) {
+          continue;
+        }
+        mapType(table, type, sanitizeName(name), defaultValue);
+      }
+    });
+  }
   if (!await knex.schema.hasTable("roles")) {
     await knex.schema.createTable("roles", (table) => {
       table.uuid("id").primary().defaultTo(knex.fn.uuid());
-      table.date("createdAt").defaultTo(knex.fn.now());
-      table.date("updatedAt").defaultTo(knex.fn.now());
+      table.timestamp("createdAt").defaultTo(knex.fn.now());
+      table.timestamp("updatedAt").defaultTo(knex.fn.now());
       for (const field of rolesSchema.fields) {
-        const { type, name, references, default: defaultValue } = field;
+        const { type, name, default: defaultValue } = field;
         if (!type || !name) {
           continue;
         }
-        if (type === "reference") {
-          if (!references) {
-            throw new Error("Field with type reference must have a reference definition.");
-          }
-          table.uuid(name).references(references.field).inTable(references.table);
-          return;
-        }
         mapType(table, type, sanitizeName(name), defaultValue);
       }
     });
@@ -567,20 +581,13 @@ var up = async function(knex) {
   if (!await knex.schema.hasTable("eval_results")) {
     await knex.schema.createTable("eval_results", (table) => {
       table.uuid("id").primary().defaultTo(knex.fn.uuid());
-      table.date("createdAt").defaultTo(knex.fn.now());
-      table.date("updatedAt").defaultTo(knex.fn.now());
+      table.timestamp("createdAt").defaultTo(knex.fn.now());
+      table.timestamp("updatedAt").defaultTo(knex.fn.now());
       for (const field of evalResultsSchema.fields) {
-        const { type, name, references, default: defaultValue } = field;
+        const { type, name, default: defaultValue } = field;
         if (!type || !name) {
           continue;
         }
-        if (type === "reference") {
-          if (!references) {
-            throw new Error("Field with type reference must have a reference definition.");
-          }
-          table.uuid(name).references(references.field).inTable(references.table);
-          return;
-        }
         mapType(table, type, sanitizeName(name), defaultValue);
       }
     });
@@ -588,62 +595,41 @@ var up = async function(knex) {
   if (!await knex.schema.hasTable("statistics")) {
     await knex.schema.createTable("statistics", (table) => {
       table.uuid("id").primary().defaultTo(knex.fn.uuid());
-      table.date("createdAt").defaultTo(knex.fn.now());
-      table.date("updatedAt").defaultTo(knex.fn.now());
+      table.timestamp("createdAt").defaultTo(knex.fn.now());
+      table.timestamp("updatedAt").defaultTo(knex.fn.now());
       for (const field of statisticsSchema.fields) {
-        const { type, name, references, default: defaultValue } = field;
+        const { type, name, default: defaultValue } = field;
         if (!type || !name) {
           continue;
         }
-        if (type === "reference") {
-          if (!references) {
-            throw new Error("Field with type reference must have a reference definition.");
-          }
-          table.uuid(name).references(references.field).inTable(references.table);
-          return;
-        }
         mapType(table, type, sanitizeName(name), defaultValue);
       }
     });
   }
   if (!await knex.schema.hasTable("jobs")) {
     await knex.schema.createTable("jobs", (table) => {
-      table.increments("id").primary();
-      table.date("createdAt").defaultTo(knex.fn.now());
-      table.date("updatedAt").defaultTo(knex.fn.now());
+      table.uuid("id").primary().defaultTo(knex.fn.uuid());
+      table.timestamp("createdAt").defaultTo(knex.fn.now());
+      table.timestamp("updatedAt").defaultTo(knex.fn.now());
       for (const field of jobsSchema.fields) {
-        const { type, name, references, default: defaultValue } = field;
+        const { type, name, default: defaultValue } = field;
         if (!type || !name) {
           continue;
         }
-        if (type === "reference") {
-          if (!references) {
-            throw new Error("Field with type reference must have a reference definition.");
-          }
-          table.uuid(name).references(references.field).inTable(references.table);
-          return;
-        }
         mapType(table, type, sanitizeName(name), defaultValue);
       }
     });
   }
   if (!await knex.schema.hasTable("agents")) {
     await knex.schema.createTable("agents", (table) => {
-      table.increments("id").primary();
-      table.date("createdAt").defaultTo(knex.fn.now());
-      table.date("updatedAt").defaultTo(knex.fn.now());
+      table.uuid("id").primary().defaultTo(knex.fn.uuid());
+      table.timestamp("createdAt").defaultTo(knex.fn.now());
+      table.timestamp("updatedAt").defaultTo(knex.fn.now());
       for (const field of agentsSchema.fields) {
-        const { type, name, references, default: defaultValue } = field;
+        const { type, name, default: defaultValue } = field;
         if (!type || !name) {
           continue;
         }
-        if (type === "reference") {
-          if (!references) {
-            throw new Error("Field with type reference must have a reference definition.");
-          }
-          table.uuid(name).references(references.field).inTable(references.table);
-          return;
-        }
         mapType(table, type, sanitizeName(name), defaultValue);
       }
     });
@@ -659,8 +645,8 @@ var up = async function(knex) {
   if (!await knex.schema.hasTable("users")) {
     await knex.schema.createTable("users", (table) => {
       table.increments("id").primary();
-      table.date("createdAt").defaultTo(knex.fn.now());
-      table.date("updatedAt").defaultTo(knex.fn.now());
+      table.timestamp("createdAt").defaultTo(knex.fn.now());
+      table.timestamp("updatedAt").defaultTo(knex.fn.now());
       table.string("name", 255);
       table.string("password", 255);
       table.string("email", 255);
@@ -668,20 +654,13 @@ var up = async function(knex) {
       table.text("image");
       for (const field of usersSchema.fields) {
         console.log("[EXULU] field", field);
-        const { type, name, references, default: defaultValue } = field;
+        const { type, name, default: defaultValue } = field;
         if (name === "id" || name === "name" || name === "email" || name === "emailVerified" || name === "image") {
           continue;
         }
         if (!type || !name) {
           continue;
         }
-        if (type === "reference") {
-          if (!references) {
-            throw new Error("Field with type reference must have a reference definition.");
-          }
-          table.uuid(name).references(references.field).inTable(references.table);
-          return;
-        }
         mapType(table, type, sanitizeName(name), defaultValue);
       }
     });
@@ -702,14 +681,6 @@ var up = async function(knex) {
       table.text("token_type");
     });
   }
-  if (!await knex.schema.hasTable("sessions")) {
-    await knex.schema.createTable("sessions", (table) => {
-      table.increments("id").primary();
-      table.integer("userId").notNullable();
-      table.timestamp("expires", { useTz: true }).notNullable();
-      table.string("sessionToken", 255).notNullable();
-    });
-  }
 };
 var execute = async () => {
   console.log("[EXULU] Initializing database.");
@@ -960,6 +931,35 @@ var ExuluEvalUtils = {
 };
 
 // src/registry/classes.ts
+function sanitizeToolName(name) {
+  if (typeof name !== "string") return "";
+  let sanitized = name.replace(/[^a-zA-Z0-9_-]+/g, "_");
+  sanitized = sanitized.replace(/^_+|_+$/g, "");
+  if (sanitized.length > 128) {
+    sanitized = sanitized.substring(0, 128);
+  }
+  return sanitized;
+}
+var convertToolsArrayToObject = (tools) => {
+  if (!tools) return {};
+  const sanitizedTools = tools ? tools.map((tool2) => ({
+    ...tool2,
+    name: sanitizeToolName(tool2.name)
+  })) : [];
+  const askForConfirmation = {
+    description: "Ask the user for confirmation.",
+    parameters: z.object({
+      message: z.string().describe("The message to ask for confirmation.")
+    })
+  };
+  return {
+    ...sanitizedTools?.reduce(
+      (prev, cur) => ({ ...prev, [cur.name]: cur.tool }),
+      {}
+    ),
+    askForConfirmation
+  };
+};
 function generateSlug(name) {
   const normalized = name.normalize("NFKD").replace(/[\u0300-\u036f]/g, "");
   const lowercase = normalized.toLowerCase();
@@ -986,54 +986,119 @@ var ExuluAgent = class {
   name;
   description = "";
   slug = "";
+  type;
   streaming = false;
   rateLimit;
   config;
   // private memory: Memory | undefined; // TODO do own implementation
-  tools;
   evals;
   model;
   capabilities;
-  constructor({ id, name, description, config, rateLimit, capabilities, tools, evals }) {
+  constructor({ id, name, description, config, rateLimit, capabilities, type, evals }) {
     this.id = id;
     this.name = name;
     this.evals = evals;
     this.description = description;
     this.rateLimit = rateLimit;
-    this.tools = tools;
     this.config = config;
-    this.capabilities = capabilities;
+    this.type = type;
+    this.capabilities = capabilities || {
+      images: [],
+      files: [],
+      audio: [],
+      video: []
+    };
     this.slug = `/agents/${generateSlug(this.name)}/run`;
-    this.model = this.config.model;
+    this.model = this.config?.model;
   }
-  generate = async ({ prompt, stream }) => {
+  // Exports the agent as a tool that can be used by another agent
+  // todo test this
+  tool = () => {
+    return new ExuluTool({
+      id: this.id,
+      name: `${this.name} agent`,
+      type: "agent",
+      inputSchema: z.object({
+        prompt: z.string()
+      }),
+      description: `A function that calls an AI agent named: ${this.name}. The agent does the following: ${this.description}.`,
+      execute: async ({ prompt }) => {
+        return await this.generateSync({
+          prompt,
+          statistics: {
+            label: "",
+            trigger: "tool"
+          }
+        });
+      }
+    });
+  };
+  generateSync = async ({ messages, prompt, tools, statistics }) => {
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
-    if (this.config.outputSchema) {
-      if (stream) {
-      }
-      const { object } = await generateObject({
-        model: this.model,
-        schema: this.config.outputSchema,
-        prompt
-      });
-      return object;
+    if (!this.config) {
+      throw new Error("Config is required for generating.");
     }
-    if (stream) {
-      const result = streamText({
-        model: this.model,
-        prompt
-      });
-      const text2 = await result.text;
-      return text2;
+    if (prompt && messages) {
+      throw new Error("Prompt and messages cannot be provided at the same time.");
     }
     const { text } = await generateText({
       model: this.model,
-      prompt
+      system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
+      messages,
+      prompt,
+      maxRetries: 2,
+      tools: convertToolsArrayToObject(tools),
+      maxSteps: 5
     });
+    if (statistics) {
+      await updateStatistic({
+        name: "count",
+        label: statistics.label,
+        type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+        trigger: statistics.trigger,
+        count: 1
+      });
+    }
     return text;
   };
+  generateStream = ({ messages, prompt, tools, statistics }) => {
+    if (!this.model) {
+      throw new Error("Model is required for streaming.");
+    }
+    if (!this.config) {
+      throw new Error("Config is required for generating.");
+    }
+    if (prompt && messages) {
+      throw new Error("Prompt and messages cannot be provided at the same time.");
+    }
+    return streamText({
+      model: this.model,
+      messages,
+      prompt,
+      system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
+      maxRetries: 2,
+      tools: convertToolsArrayToObject(tools),
+      maxSteps: 5,
+      onError: (error) => console.error("[EXULU] chat stream error.", error),
+      onFinish: async ({ response, usage }) => {
+        console.info(
+          "[EXULU] chat stream finished.",
+          usage
+        );
+        if (statistics) {
+          await updateStatistic({
+            name: "count",
+            label: statistics.label,
+            type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+            trigger: statistics.trigger,
+            count: 1
+          });
+        }
+      }
+    });
+  };
 };
 var ExuluEmbedder = class {
   id;
@@ -1056,6 +1121,13 @@ var ExuluEmbedder = class {
   }
   async generateFromQuery(query, statistics) {
     if (statistics) {
+      await updateStatistic({
+        name: "count",
+        label: statistics.label,
+        type: STATISTICS_TYPE_ENUM.EMBEDDER_GENERATE,
+        trigger: statistics.trigger,
+        count: 1
+      });
     }
     return await this.generateEmbeddings({
       item: {
@@ -1069,6 +1141,13 @@ var ExuluEmbedder = class {
   }
   async generateFromDocument(input, statistics) {
     if (statistics) {
+      await updateStatistic({
+        name: "count",
+        label: statistics.label,
+        type: STATISTICS_TYPE_ENUM.EMBEDDER_GENERATE,
+        trigger: statistics.trigger,
+        count: 1
+      });
     }
     if (!this.chunker) {
       throw new Error("Chunker not found for embedder " + this.name);
@@ -1257,9 +1336,8 @@ var ExuluEval = class {
               if (!data.prompt) {
                 throw new Error("Prompt is required for running an agent.");
               }
-              const result = await runner.agent.generate({
-                prompt: data.prompt,
-                stream: false
+              const result = await runner.agent.generateSync({
+                prompt: data.prompt
               });
               data.result = result;
             }
@@ -1354,18 +1432,18 @@ var ExuluTool = class {
   id;
   name;
   description;
-  parameters;
+  inputSchema;
   type;
   tool;
-  constructor({ id, name, description, parameters, type, execute: execute2 }) {
+  constructor({ id, name, description, inputSchema, type, execute: execute2 }) {
     this.id = id;
     this.name = name;
     this.description = description;
-    this.parameters = parameters;
+    this.inputSchema = inputSchema;
     this.type = type;
     this.tool = tool({
       description,
-      parameters,
+      parameters: inputSchema || z.object({}),
       execute: execute2
     });
   }
@@ -1421,7 +1499,7 @@ var ExuluContext = class {
     }
     const { db: db2 } = await postgresClient();
     Object.keys(item).forEach((key) => {
-      if (key === "name" || key === "description" || key === "external_id" || key === "tags" || key === "source" || key === "textLength" || key === "upsert") {
+      if (key === "name" || key === "description" || key === "external_id" || key === "tags" || key === "source" || key === "textLength" || key === "upsert" || key === "archived") {
         return;
       }
       const field = this.fields.find((field2) => field2.name === key);
@@ -1468,6 +1546,9 @@ var ExuluContext = class {
         chunk_index: chunk.index,
         embedding: pgvector2.toSql(chunk.vector)
       })));
+      await db2.from(this.getTableName()).where({ id }).update({
+        embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
+      }).returning("id");
     }
     return {
       id: result[0].id,
@@ -1497,7 +1578,7 @@ var ExuluContext = class {
       }
     }
     Object.keys(item).forEach((key) => {
-      if (key === "name" || key === "description" || key === "external_id" || key === "tags" || key === "source" || key === "textLength" || key === "upsert") {
+      if (key === "name" || key === "description" || key === "external_id" || key === "tags" || key === "source" || key === "textLength" || key === "upsert" || key === "archived") {
         return;
       }
       const field = this.fields.find((field2) => field2.name === key);
@@ -1548,6 +1629,9 @@ var ExuluContext = class {
         chunk_index: chunk.index,
         embedding: pgvector2.toSql(chunk.vector)
       })));
+      await db2.from(this.getTableName()).where({ id: result[0].id }).update({
+        embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
+      }).returning("id");
     }
     return {
       id: result[0].id,
@@ -1557,6 +1641,8 @@ var ExuluContext = class {
   getItems = async ({
     statistics,
     limit,
+    sort,
+    order,
     page,
     name,
     archived,
@@ -1577,6 +1663,9 @@ var ExuluContext = class {
     const columns = await db2(mainTable).columnInfo();
     const totalQuery = db2.count("* as count").from(mainTable).first();
     const itemsQuery = db2.select(Object.keys(columns).map((column) => mainTable + "." + column)).from(mainTable).offset(offset).limit(limit);
+    if (sort) {
+      itemsQuery.orderBy(sort, order === "desc" ? "desc" : "asc");
+    }
     if (typeof name === "string") {
       itemsQuery.whereILike("name", `%${name}%`);
       totalQuery.whereILike("name", `%${name}%`);
@@ -1620,7 +1709,7 @@ var ExuluContext = class {
       }
       itemsQuery.limit(limit * 5);
       if (statistics) {
-        updateStatistic({
+        await updateStatistic({
           name: "count",
           label: statistics.label,
           type: STATISTICS_TYPE_ENUM.CONTEXT_RETRIEVE,
@@ -1753,6 +1842,7 @@ var ExuluContext = class {
       table.text("external_id");
       table.integer("textLength");
       table.text("source");
+      table.timestamp("embeddings_updated_at");
       for (const field of this.fields) {
         const { type, name } = field;
         if (!type || !name) {
@@ -1782,15 +1872,15 @@ var ExuluContext = class {
       id: this.id,
       name: `${this.name} context`,
       type: "context",
-      parameters: z.object({
+      inputSchema: z.object({
         query: z.string()
       }),
       description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
-      execute: async ({ context }) => {
+      execute: async ({ query }) => {
         return await this.getItems({
           page: 1,
           limit: 10,
-          query: context.query,
+          query,
           statistics: {
             label: this.name,
             trigger: "agent"
@@ -1839,7 +1929,32 @@ var ExuluSource = class {
   }
 };
 var updateStatistic = async (statistic) => {
-  return;
+  const currentDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
+  const { db: db2 } = await postgresClient();
+  const existing = await db2.from("statistics").where({
+    name: statistic.name,
+    label: statistic.label,
+    type: statistic.type,
+    createdAt: currentDate
+  }).first();
+  if (!existing) {
+    await db2.from("statistics").insert({
+      name: statistic.name,
+      label: statistic.label,
+      type: statistic.type,
+      total: statistic.count ?? 1,
+      createdAt: currentDate
+    });
+  } else {
+    await db2.from("statistics").update({
+      total: db2.raw("total + ?", [statistic.count ?? 1])
+    }).where({
+      name: statistic.name,
+      label: statistic.label,
+      type: statistic.type,
+      createdAt: currentDate
+    });
+  }
 };
 
 // src/registry/index.ts
@@ -1939,6 +2054,7 @@ var authentication = async ({
   }
   if (authtoken) {
     try {
+      console.log("[EXULU] authtoken", authtoken);
       if (!authtoken?.email) {
         return {
           error: true,
@@ -2032,8 +2148,7 @@ var requestValidators = {
     const { db: db2 } = await postgresClient();
     let authtoken = null;
     if (typeof apikey !== "string") {
-      const secret = process.env.NEXTAUTH_SECRET;
-      authtoken = await getToken(req.headers["authorization"] ?? "");
+      authtoken = await getToken((req.headers["authorization"] || req.headers["x-api-key"]) ?? "");
     }
     return await authentication({
       authtoken,
@@ -2226,6 +2341,42 @@ import "reflect-metadata";
 // src/registry/utils/graphql.ts
 import { makeExecutableSchema } from "@graphql-tools/schema";
 import GraphQLJSON from "graphql-type-json";
+import { GraphQLScalarType, Kind } from "graphql";
+import CryptoJS from "crypto-js";
+var GraphQLDate = new GraphQLScalarType({
+  name: "Date",
+  description: "Date custom scalar type",
+  serialize(value) {
+    if (value instanceof Date) {
+      return value.toISOString();
+    }
+    if (typeof value === "number") {
+      return new Date(value).toISOString();
+    }
+    if (typeof value === "string") {
+      return new Date(value).toISOString();
+    }
+    return value;
+  },
+  parseValue(value) {
+    if (typeof value === "string") {
+      return new Date(value);
+    }
+    if (typeof value === "number") {
+      return new Date(value);
+    }
+    return value;
+  },
+  parseLiteral(ast) {
+    if (ast.kind === Kind.STRING) {
+      return new Date(ast.value);
+    }
+    if (ast.kind === Kind.INT) {
+      return new Date(parseInt(ast.value, 10));
+    }
+    return null;
+  }
+});
 var map = (field) => {
   let type;
   switch (field.type) {
@@ -2245,7 +2396,7 @@ var map = (field) => {
       type = "JSON";
       break;
     case "date":
-      type = "String";
+      type = "Date";
       break;
     default:
       type = "String";
@@ -2263,8 +2414,8 @@ function createTypeDefs(table) {
   type ${table.name.singular} {
   ${fields.join("\n")}
     id: ID!
-    createdAt: String!
-    updatedAt: String!
+    createdAt: Date!
+    updatedAt: Date!
   }
   `;
   const inputDef = `
@@ -2290,6 +2441,11 @@ input FilterOperatorString {
   contains: String
 }
 
+input FilterOperatorDate {
+  lte: Date
+  gte: Date
+}
+
 input FilterOperatorFloat {
   eq: Float
   ne: Float
@@ -2342,8 +2498,10 @@ function createMutations(table) {
     [`${tableNamePlural}CreateOne`]: async (_, args, context, info) => {
       const { db: db2 } = context;
       const requestedFields = getRequestedFields(info);
+      let { input } = args;
+      input = encryptSensitiveFields(input);
       const results = await db2(tableNamePlural).insert({
-        ...args.input,
+        ...input,
         createdAt: /* @__PURE__ */ new Date(),
         updatedAt: /* @__PURE__ */ new Date()
       }).returning(requestedFields);
@@ -2351,7 +2509,8 @@ function createMutations(table) {
     },
     [`${tableNamePlural}UpdateOne`]: async (_, args, context, info) => {
       const { db: db2 } = context;
-      const { where, input } = args;
+      let { where, input } = args;
+      input = encryptSensitiveFields(input);
       await db2(tableNamePlural).where(where).update({
         ...input,
         updatedAt: /* @__PURE__ */ new Date()
@@ -2361,7 +2520,8 @@ function createMutations(table) {
       return result;
     },
     [`${tableNamePlural}UpdateOneById`]: async (_, args, context, info) => {
-      const { id, input } = args;
+      let { id, input } = args;
+      input = encryptSensitiveFields(input);
       const { db: db2 } = context;
       await db2(tableNamePlural).where({ id }).update({
         ...input,
@@ -2464,12 +2624,44 @@ function createQueries(table) {
         },
         items
       };
-    }
+    },
+    // Add jobStatistics query for jobs table
+    ...tableNamePlural === "jobs" ? {
+      jobStatistics: async (_, args, context, info) => {
+        const { user, agent, from, to } = args;
+        const { db: db2 } = context;
+        let query = db2("jobs");
+        if (user) {
+          query = query.where("user", user);
+        }
+        if (agent) {
+          query = query.where("agent", agent);
+        }
+        if (from) {
+          query = query.where("createdAt", ">=", from);
+        }
+        if (to) {
+          query = query.where("createdAt", "<=", to);
+        }
+        const completedQuery = query.clone().where("status", "completed");
+        const [{ completedCount }] = await completedQuery.count("* as completedCount");
+        const failedQuery = query.clone().where("status", "failed");
+        const [{ failedCount }] = await failedQuery.count("* as failedCount");
+        const durationQuery = query.clone().where("status", "completed").whereNotNull("duration").select(db2.raw('AVG("duration") as averageDuration'));
+        const [{ averageDuration }] = await durationQuery;
+        return {
+          completedCount: Number(completedCount),
+          failedCount: Number(failedCount),
+          averageDuration: averageDuration ? Number(averageDuration) : 0
+        };
+      }
+    } : {}
   };
 }
 function createSDL(tables) {
   let typeDefs = `
     scalar JSON
+    scalar Date
     
     type Query {
     `;
@@ -2477,7 +2669,7 @@ function createSDL(tables) {
     type Mutation {
     `;
   let modelDefs = "";
-  const resolvers = { JSON: GraphQLJSON, Query: {}, Mutation: {} };
+  const resolvers = { JSON: GraphQLJSON, Date: GraphQLDate, Query: {}, Mutation: {} };
   for (const table of tables) {
     const tableNamePlural = table.name.plural.toLowerCase();
     const tableNameSingular = table.name.singular.toLowerCase();
@@ -2486,6 +2678,7 @@ function createSDL(tables) {
       ${tableNameSingular}ById(id: ID!): ${tableNameSingular}
       ${tableNamePlural}Pagination(limit: Int, page: Int, filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingularUpperCaseFirst}PaginationResult
       ${tableNameSingular}One(filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingular}
+      ${tableNamePlural === "jobs" ? `jobStatistics(user: ID, agent: String, from: String, to: String): JobStatistics` : ""}
     `;
     mutationDefs += `
       ${tableNamePlural}CreateOne(input: ${tableNameSingular}Input!): ${tableNameSingular}
@@ -2510,6 +2703,15 @@ type PageInfo {
   hasNextPage: Boolean!
 }
 `;
+    if (tableNamePlural === "jobs") {
+      modelDefs += `
+type JobStatistics {
+  completedCount: Int!
+  failedCount: Int!
+  averageDuration: Float!
+}
+`;
+    }
     Object.assign(resolvers.Query, createQueries(table));
     Object.assign(resolvers.Mutation, createMutations(table));
   }
@@ -2546,6 +2748,15 @@ type PageInfo {
   console.log("\n");
   return schema;
 }
+var sensitiveFields = ["anthropic_token"];
+var encryptSensitiveFields = (input) => {
+  sensitiveFields.forEach((field) => {
+    if (input[field]) {
+      input[field] = CryptoJS.AES.encrypt(input[field], process.env.NEXTAUTH_SECRET).toString();
+    }
+  });
+  return input;
+};
 
 // src/registry/routes.ts
 import { expressMiddleware } from "@as-integrations/express5";
@@ -2944,6 +3155,34 @@ var createUppyRoutes = async (app) => {
 // src/registry/routes.ts
 import { InMemoryLRUCache } from "@apollo/utils.keyvaluecache";
 import bodyParser from "body-parser";
+import CryptoJS2 from "crypto-js";
+
+// src/registry/utils/claude-messages.ts
+var CLAUDE_MESSAGES = {
+  authentication_error: `
+\x1B[41m -- Authentication error please check your IMP token and try again. --
+\x1B[0m`,
+  missing_body: `
+\x1B[41m -- Missing body Anthropic response. --
+\x1B[0m`,
+  missing_nextauth_secret: `
+\x1B[41m -- Missing NEXTAUTH_SECRET in environment variables on the server. --
+\x1B[0m`,
+  not_enabled: `
+\x1B[31m
+\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2557      \u2588\u2588\u2557   \u2588\u2588\u2557
+\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u255A\u2588\u2588\u2557\u2588\u2588\u2554\u255D\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+\u2588\u2588\u2588\u2588\u2588\u2557   \u255A\u2588\u2588\u2588\u2554\u255D \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+\u2588\u2588\u2554\u2550\u2550\u255D   \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2554\u255D \u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D
+\u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D 
+Intelligence Management Platform
+\x1B[0m
+\x1B[41m -- Your account has not been enabled to use Claude Code, please contact your admin or enable Claude Code in the user settings. --
+\x1B[0m`
+};
+
+// src/registry/routes.ts
 var REQUEST_SIZE_LIMIT = "50mb";
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -2980,7 +3219,7 @@ var createRecurringJobs = async () => {
   console.table(recurringJobSchedulersLogs);
   return queue;
 };
-var createExpressRoutes = async (app, agents, embedders, tools, workflows, contexts) => {
+var createExpressRoutes = async (app, agents, tools, workflows, contexts) => {
   const routeLogs = [];
   var corsOptions = {
     origin: "*",
@@ -3000,6 +3239,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     \u2588\u2588\u2554\u2550\u2550\u255D   \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2554\u255D \u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D
     \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D 
+    Intelligence Management Platform
 
     `);
   console.log("Agents:");
@@ -3048,7 +3288,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
   } else {
     console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
   }
-  const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema, workflowSchema, evalResultsSchema, threadsSchema, messagesSchema]);
+  const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema, workflowSchema, evalResultsSchema, agentSessionsSchema, agentMessagesSchema]);
   console.log("[EXULU] graphql server");
   const server = new ApolloServer({
     cache: new InMemoryLRUCache(),
@@ -3076,10 +3316,52 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     })
   );
-  app.get(`/agents`, async (req, res) => {
+  app.get(`/providers`, async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
     res.status(200).json(agents);
   });
+  app.get(`/agents`, async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { db: db2 } = await postgresClient();
+    const agentsFromDb = await db2.from("agents").select("*");
+    res.status(200).json(agentsFromDb.map((agent) => {
+      const backend = agents.find((a) => a.id === agent.backend);
+      if (!backend) {
+        return null;
+      }
+      return {
+        name: agent.name,
+        id: agent.id,
+        description: agent.description,
+        provider: backend?.model?.provider,
+        model: backend?.model?.modelId,
+        active: agent.active,
+        public: agent.public,
+        type: agent.type,
+        slug: backend?.slug,
+        rateLimit: backend?.rateLimit,
+        streaming: backend?.streaming,
+        capabilities: backend?.capabilities,
+        // todo add contexts
+        availableTools: tools,
+        enabledTools: agent.tools
+      };
+    }).filter(Boolean));
+  });
   app.get(`/agents/:id`, async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
     const { db: db2 } = await postgresClient();
     const id = req.params.id;
     if (!id) {
@@ -3102,6 +3384,8 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
         name: agent.name,
         id: agent.id,
         description: agent.description,
+        provider: backend?.model?.provider,
+        model: backend?.model?.modelId,
         active: agent.active,
         public: agent.public,
         type: agent.type,
@@ -3110,7 +3394,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
         streaming: backend?.streaming,
         capabilities: backend?.capabilities,
         // todo add contexts
-        availableTools: backend?.tools,
+        availableTools: tools,
         enabledTools: agent.tools
       }
     });
@@ -3121,8 +3405,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       name: tool2.name,
       description: tool2.description,
       type: tool2.type || "tool",
-      inputSchema: tool2.inputSchema ? zerialize(tool2.inputSchema) : null,
-      outputSchema: tool2.outputSchema ? zerialize(tool2.outputSchema) : null
+      inputSchema: tool2.inputSchema ? zerialize(tool2.inputSchema) : null
     })));
   });
   app.get("/tools/:id", async (req, res) => {
@@ -3240,6 +3523,14 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       });
       return;
     }
+    const itemsTableExists = await context.tableExists();
+    if (!itemsTableExists) {
+      await context.createItemsTable();
+    }
+    const chunksTableExists = await db2.schema.hasTable(context.getChunksTableName());
+    if (!chunksTableExists) {
+      await context.createChunksTable();
+    }
     const item = await db2.from(context.getTableName()).where({ id: req.params.id }).select("*").first();
     if (!item) {
       res.status(404).json({
@@ -3361,6 +3652,20 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     }
     let limit = req.query.limit ? parseInt(req.query.limit) : 10;
     let page = req.query.page ? parseInt(req.query.page) : 1;
+    let sort = req.query.sort ? req.query.sort : "created_at";
+    let order = req.query.order ? req.query.order : "desc";
+    if (sort && !["created_at", "embeddings_updated_at"].includes(sort)) {
+      res.status(400).json({
+        message: "Invalid sort field, must be one of: createdAt, embeddings_updated_at"
+      });
+      return;
+    }
+    if (order && !["desc", "asc"].includes(order)) {
+      res.status(400).json({
+        message: "Invalid order, must be one of: desc, asc"
+      });
+      return;
+    }
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
@@ -3384,6 +3689,8 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       return;
     }
     const result = await context.getItems({
+      sort,
+      order,
       page,
       limit,
       archived: req.query.archived === "true",
@@ -3463,6 +3770,18 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       message: "Embedding deleted."
     });
   });
+  app.get("/ping", async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(200).json({
+        authenticated: false
+      });
+      return;
+    }
+    res.status(200).json({
+      authenticated: true
+    });
+  });
   console.log("[EXULU] statistics timeseries");
   app.post("/statistics/timeseries", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
@@ -3602,6 +3921,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
         slug: "/contexts/" + context.id,
         active: context.active,
         fields: context.fields,
+        configuration: context.configuration,
         sources: context.sources.get().map((source) => ({
           id: source.id,
           name: source.name,
@@ -3796,47 +4116,28 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
         res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
         return;
       }
+      console.log("[EXULU] agent tools", agentInstance.tools);
+      const enabledTools = agentInstance.tools.map((tool2) => tools.find(({ id }) => id === tool2)).filter(Boolean);
+      console.log("[EXULU] enabled tools", enabledTools);
       if (!!stream) {
-        const chatClient = await agent.chat(agentInstance.id);
-        if (!chatClient) {
-          res.status(500).json({
-            message: "Agent instantiation not successful."
-          });
-          return;
-        }
-        const { textStream } = await chatClient.stream(req.body.messages, {
-          threadId: `${req.body.threadId}`,
-          // conversation id
-          resourceId: `${req.body.resourceId}`,
-          // user id
-          ...agent.outputSchema && { output: agent.outputSchema },
-          maxRetries: 2,
-          // todo make part of ExuluAgent class
-          maxSteps: 5,
-          // todo make part of ExuluAgent class
-          onError: (error) => console.error("[EXULU] chat stream error.", error),
-          onFinish: ({ response, usage }) => console.info(
-            "[EXULU] chat stream finished.",
-            usage
-          )
+        const result = agent.generateStream({
+          messages: req.body.messages,
+          tools: enabledTools,
+          statistics: {
+            label: agent.name,
+            trigger: "agent"
+          }
         });
-        for await (const delta of textStream) {
-          res.write(`data: ${delta}
-
-`);
-        }
-        res.end();
+        result.pipeDataStreamToResponse(res);
         return;
       } else {
-        const response = await agent.chat.generate(req.body.messages, {
-          resourceId: `${authenticationResult.user.id}`,
-          output: agent.outputSchema,
-          threadId: `${req.body.threadId}`,
-          // conversation id
-          maxRetries: 2,
-          // todo make part of ExuluAgent class
-          maxSteps: 5
-          // todo make part of ExuluAgent class
+        const response = await agent.generateSync({
+          messages: req.body.messages,
+          tools: enabledTools.map(),
+          statistics: {
+            label: agent.name,
+            trigger: "agent"
+          }
         });
         res.status(200).json(response);
         return;
@@ -3911,7 +4212,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
   console.log("Routes:");
   console.table(routeLogs);
   const TARGET_API = "https://api.anthropic.com";
-  app.use("/gateway/anthropic", express.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }), async (req, res) => {
+  app.use("/gateway/anthropic/:id", express.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }), async (req, res) => {
     const path3 = req.url;
     const url = `${TARGET_API}${path3}`;
     console.log("[PROXY] Manual proxy to:", url);
@@ -3922,57 +4223,100 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     console.log("[PROXY] Request stream:", req.body.stream);
     console.log("[PROXY] Request messages:", req.body.messages?.length);
     try {
-      const headers = {
-        "x-api-key": process.env.ANTHROPIC_API_KEY,
-        "anthropic-version": "2023-06-01",
-        "content-type": req.headers["content-type"] || "application/json"
-      };
-      if (req.headers["accept"]) headers["accept"] = req.headers["accept"];
-      if (req.headers["user-agent"]) headers["user-agent"] = req.headers["user-agent"];
       console.log("[PROXY] Request body tools array length:", req.body.tools?.length);
       if (!req.body.tools) {
         req.body.tools = [];
       }
-      if (req.headers["x-api-key"] === "PLACEHOLDER") {
-        res.write(JSON.stringify({
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": "Hello, world!"
-          }
-        }));
-        res.end();
+      const authenticationResult = await requestValidators.authenticate(req);
+      if (!authenticationResult.user?.id) {
+        res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+        return;
+      }
+      console.log("[EXULU] authentication result", authenticationResult);
+      const { db: db2 } = await postgresClient();
+      const agent = await db2.from("agents").where({
+        id: req.params.id
+      }).first();
+      if (!agent) {
+        const arrayBuffer = createCustomAnthropicStreamingMessage(`
+\x1B[41m -- Agent ${req.params.id} not found or you do not have access to it. --
+\x1B[0m`);
+        res.setHeader("Content-Type", "application/json");
+        res.end(Buffer.from(arrayBuffer));
+        return;
+      }
+      console.log("[EXULU] agent", agent?.name);
+      if (!process.env.NEXTAUTH_SECRET) {
+        const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.missing_nextauth_secret);
+        res.setHeader("Content-Type", "application/json");
+        res.end(Buffer.from(arrayBuffer));
         return;
       }
+      if (!authenticationResult.user?.anthropic_token) {
+        const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.not_enabled);
+        res.setHeader("Content-Type", "application/json");
+        res.end(Buffer.from(arrayBuffer));
+        return;
+      }
+      const bytes = CryptoJS2.AES.decrypt(authenticationResult.user?.anthropic_token, process.env.NEXTAUTH_SECRET);
+      const anthropicApiKey = bytes.toString(CryptoJS2.enc.Utf8);
+      const headers = {
+        "x-api-key": anthropicApiKey,
+        "anthropic-version": "2023-06-01",
+        "content-type": req.headers["content-type"] || "application/json"
+      };
+      if (req.headers["accept"]) headers["accept"] = req.headers["accept"];
+      if (req.headers["user-agent"]) headers["user-agent"] = req.headers["user-agent"];
+      console.log("[EXULU] anthropic api key", anthropicApiKey);
       const response = await fetch(url, {
         method: req.method,
         headers,
         body: req.method !== "GET" ? JSON.stringify(req.body) : void 0
       });
-      console.log("[PROXY] Response status:", response.status);
+      console.log("[PROXY] Response:", response);
+      console.log("[PROXY] Response:", response.body);
+      await updateStatistic({
+        name: "count",
+        label: "Claude Code",
+        type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+        trigger: "claude-code",
+        count: 1
+      });
       response.headers.forEach((value, key) => {
         res.setHeader(key, value);
       });
       res.status(response.status);
-      if (response.headers.get("content-type")?.includes("text/event-stream")) {
+      const isStreaming = response.headers.get("content-type")?.includes("text/event-stream");
+      if (isStreaming && !response?.body) {
+        const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.missing_body);
+        res.setHeader("Content-Type", "application/json");
+        res.end(Buffer.from(arrayBuffer));
+        return;
+      }
+      if (isStreaming) {
         const reader = response.body.getReader();
         const decoder = new TextDecoder();
         while (true) {
           const { done, value } = await reader.read();
           if (done) break;
           const chunk = decoder.decode(value, { stream: true });
+          console.log("[PROXY] Chunk:", chunk);
           res.write(chunk);
         }
         res.end();
-      } else {
-        const data = await response.arrayBuffer();
-        res.end(Buffer.from(data));
+        return;
       }
+      const data = await response.arrayBuffer();
+      console.log("[PROXY] Data:", data);
+      res.end(Buffer.from(data));
     } catch (error) {
       console.error("[PROXY] Manual proxy error:", error);
       if (!res.headersSent) {
-        res.status(500).json({ error: error.message });
+        if (error?.message === "Invalid token") {
+          res.status(500).json({ error: "Authentication error, please check your IMP token and try again." });
+        } else {
+          res.status(500).json({ error: error.message });
+        }
       }
     }
   });
@@ -4020,6 +4364,20 @@ var getPresignedFileUrl = async (key) => {
   console.log(`[EXULU] presigned url for file with key: ${key}, generated: ${json.url}`);
   return json.url;
 };
+var createCustomAnthropicStreamingMessage = (message) => {
+  const responseData = {
+    type: "message",
+    content: [
+      {
+        type: "text",
+        text: message
+      }
+    ]
+  };
+  const jsonString = JSON.stringify(responseData);
+  const arrayBuffer = new TextEncoder().encode(jsonString).buffer;
+  return arrayBuffer;
+};
 
 // src/registry/workers.ts
 import IORedis from "ioredis";
@@ -4072,7 +4430,7 @@ import * as fs2 from "fs";
 import path2 from "path";
 var defaultLogsDir = path2.join(process.cwd(), "logs");
 var redisConnection;
-var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) => {
+var createWorkers = async (queues2, contexts, workflows, _logsDir) => {
   if (!redisServer.host || !redisServer.port) {
     console.error("[EXULU] you are trying to start workers, but no redis server is configured in the environment.");
     throw new Error("No redis server configured in the environment, so cannot start workers.");
@@ -4103,7 +4461,7 @@ var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) =>
             if (!bullmqJob.data.embedder) {
               throw new Error(`No embedder set for embedder job.`);
             }
-            const embedder = embedders.find((embedder2) => embedder2.id === bullmqJob.data.embedder);
+            const embedder = contexts.find((context2) => context2.embedder?.id === bullmqJob.data.embedder);
             if (!embedder) {
               throw new Error(`Embedder ${bullmqJob.data.embedder} not found in the registry.`);
             }
@@ -4143,6 +4501,9 @@ var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) =>
               duration,
               result: JSON.stringify(result)
             });
+            await db2.from((void 0).getTableName()).where({ id: result[0].id }).update({
+              embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
+            }).returning("id");
             return result;
           }
           if (bullmqJob.data.type === "workflow") {
@@ -4240,7 +4601,7 @@ var ExuluMCP = class {
   express;
   constructor() {
   }
-  create = async ({ express: express3, contexts, embedders, agents, workflows, config, tools }) => {
+  create = async ({ express: express3, contexts, agents, workflows, config, tools }) => {
     this.express = express3;
     if (!this.server) {
       console.log("[EXULU] Creating MCP server.");
@@ -4371,28 +4732,210 @@ ${code}`
 
 // src/registry/index.ts
 import express2 from "express";
+
+// src/templates/agents/claude-code.ts
+var agentId = "0832-5178-1145-2194";
+var claudeCodeAgent = new ExuluAgent({
+  id: `${agentId}-claude-code-agent`,
+  name: `Claude Code Agent`,
+  description: `Claude Code agent, enabling the creation of multiple Claude Code Agent instances with different configurations (rate limits, functions, etc).`,
+  type: "custom"
+});
+
+// src/templates/agents/claude-opus-4.ts
+import { anthropic } from "@ai-sdk/anthropic";
+import { z as z3 } from "zod";
+var agentId2 = "5434-5678-9143-2590";
+var defaultAgent = new ExuluAgent({
+  id: `${agentId2}-default-claude-4-opus-agent`,
+  name: `Default Claude 4 Opus Agent`,
+  description: `Basic agent without any defined tools, that can support MCP's.`,
+  type: "agent",
+  capabilities: {
+    tools: false,
+    images: [],
+    files: [],
+    audio: [],
+    video: []
+  },
+  evals: [],
+  config: {
+    name: `Default agent`,
+    instructions: "You are a helpful assistant.",
+    model: anthropic("claude-4-opus-20250514"),
+    // todo add a field of type string that adds a dropdown list from which the user can select the model
+    // todo for each model, check which provider is used, and require the admin to add one or multiple
+    //   API keys for the provider (which we can then auto-rotate).
+    // todo also add custom fields for rate limiting, so the admin can set custom rate limits for the agent
+    //   and allow him/her to decide if the rate limit is per user or per agent.
+    // todo finally allow switching on or off immutable audit logs on the agent. Which then enables OTEL
+    //   and stores the logs into the pre-defined storage.
+    custom: z3.object({
+      apiKey: z3.string()
+    })
+  }
+});
+
+// src/templates/tools/browserbase.ts
+import { z as z4 } from "zod";
+import { Stagehand } from "@browserbasehq/stagehand";
+import { Browserbase } from "@browserbasehq/sdk";
+var PROJECT_ID = "811444dd-6e6d-40b5-bd90-541c93e44be6";
+process.env.BROWSERBASE_PROJECT_ID = PROJECT_ID;
+var BB_API_KEY = "bb_live_LwMwNgZB5cIEKcBwMuAugrgNkFM";
+async function createContext() {
+  const bb = new Browserbase({ apiKey: BB_API_KEY });
+  const context = await bb.contexts.create({
+    projectId: PROJECT_ID
+  });
+  return context;
+}
+async function createAuthSession(contextId) {
+  const bb = new Browserbase({ apiKey: BB_API_KEY });
+  const session = await bb.sessions.create({
+    projectId: PROJECT_ID,
+    browserSettings: {
+      context: {
+        id: contextId,
+        persist: true
+      }
+    }
+    /* proxies: [{ // not included in the free tier
+        type: "browserbase",
+        geolocation: {
+            city: CITY,
+            country: COUNTRY
+        }
+    }] */
+  });
+  const liveViewLinks = await bb.sessions.debug(session.id);
+  const liveViewLink = liveViewLinks.debuggerFullscreenUrl;
+  console.log(`\u{1F50D} Live View Link: ${liveViewLink}`);
+  console.log("Session URL: https://browserbase.com/sessions/" + session.id);
+  return {
+    url: liveViewLink,
+    id: session.id
+  };
+}
+var createSession = new ExuluTool({
+  id: `1234-5178-9423-4267`,
+  type: "function",
+  name: "Create a browserbase session.",
+  description: `
+    Creates a browserbase session and returns the live view url as well as 
+    the session id as a JSON object. A browserbase session is a headless browser
+    that can be used to to visit websites and perform actions.
+    `,
+  execute: async () => {
+    const { id } = await createContext();
+    return await createAuthSession(id);
+  }
+});
+var askChatgpt = new ExuluTool({
+  id: `1234-5178-9423-4268`,
+  type: "function",
+  name: "ChatGPT browserbase operation.",
+  inputSchema: z4.object({
+    session: z4.string().describe("The session id of the browserbase session."),
+    question: z4.string().describe("The question to ask ChatGPT.")
+  }),
+  description: `Uses an existing, authenticated browserbase session to visit ChatGPT and perform actions such as asking questions.`,
+  execute: async ({ session, question }) => {
+    const stagehand = new Stagehand({
+      // With npx create-browser-app, this config is found 
+      // in a separate stagehand.config.ts file
+      env: "BROWSERBASE",
+      // set to "LOCAL" for local development
+      apiKey: BB_API_KEY,
+      // todo make this a config variable the admin can set in the UI
+      modelName: "openai/gpt-4.1-mini",
+      // todo change to claude || optionally make configurable?
+      browserbaseSessionID: session,
+      modelClientOptions: {
+        apiKey: process.env.OPENAI_API_KEY
+        // todo make this a config variable the admin can set in the UI
+      }
+    });
+    await stagehand.init();
+    const page = stagehand.page;
+    await page.goto("https://chatgpt.com");
+    await page.act(`Type in '${question}' into the search bar`);
+    const { answer } = await page.extract({
+      instruction: "The answer to the question generated by ChatGPT.",
+      schema: z4.object({
+        answer: z4.string()
+      })
+    });
+    console.log(answer);
+    await stagehand.close();
+    return {
+      answer
+    };
+  }
+});
+
+// src/templates/tools/jira.ts
+import { z as z5 } from "zod";
+var getTicket = new ExuluTool({
+  id: `1414-5179-1423-1269`,
+  name: "JIRA ticket retrieval.",
+  type: "function",
+  inputSchema: z5.object({
+    ticketId: z5.string().describe("The id of the ticket to retrieve.")
+  }),
+  description: `Retrieves a ticket from Jira.`,
+  execute: async ({ session, question }) => {
+    return {
+      name: "BYD-1234",
+      id: "12345678",
+      status: "Open",
+      description: "This is a test ticket",
+      assignee: "John Doe",
+      createdAt: "2021-01-01",
+      updatedAt: "2021-01-01",
+      dueDate: "2021-01-01",
+      priority: "High"
+    };
+  }
+});
+
+// src/registry/index.ts
 var ExuluApp = class {
   _agents = [];
   _workflows = [];
   _config;
-  _embedders = [];
   _queues = [];
   _contexts = {};
   _tools = [];
   _expressApp = null;
   constructor() {
   }
-  // Factory function so we can async initialize the
-  // MCP server if needed.
-  create = async ({ contexts, embedders, agents, workflows, config, tools }) => {
-    this._embedders = embedders ?? [];
+  // Factory function so we can async 
+  // initialize the MCP server if needed.
+  create = async ({ contexts, agents, workflows, config, tools }) => {
     this._workflows = workflows ?? [];
     this._contexts = contexts ?? {};
-    this._agents = agents ?? [];
+    this._agents = [
+      claudeCodeAgent,
+      defaultAgent,
+      ...agents ?? []
+    ];
     this._config = config;
-    this._tools = tools ?? [];
+    this._tools = [
+      ...tools ?? [],
+      // Add contexts as tools
+      ...Object.values(contexts || {}).map((context) => context.tool()),
+      // Add agents as tools
+      ...(agents || []).map((agent) => agent.tool()),
+      ...[
+        createSession,
+        askChatgpt,
+        getTicket
+      ]
+    ];
+    const contextsArray = Object.values(contexts || {});
     const queues2 = [
-      ...embedders?.length ? embedders.map((agent) => agent.queue?.name || null) : [],
+      ...contextsArray?.length ? contextsArray.map((context) => context.embedder.queue?.name || null) : [],
       ...workflows?.length ? workflows.map((workflow) => workflow.queue?.name || null) : []
     ];
     this._queues = [...new Set(queues2.filter((o) => !!o))];
@@ -4409,9 +4952,6 @@ var ExuluApp = class {
     }
     return this._expressApp;
   }
-  embedder(id) {
-    return this._embedders.find((x) => x.id === id);
-  }
   tool(id) {
     return this._tools.find((x) => x.id === id);
   }
@@ -4427,9 +4967,6 @@ var ExuluApp = class {
   workflow(id) {
     return this._workflows.find((x) => x.id === id);
   }
-  get embedders() {
-    return this._embedders;
-  }
   get contexts() {
     return Object.values(this._contexts ?? {});
   }
@@ -4445,7 +4982,6 @@ var ExuluApp = class {
         return await createWorkers(
           this._queues,
           Object.values(this._contexts ?? {}),
-          this._embedders,
           this._workflows,
           this._config?.workers?.logsDir
         );
@@ -4462,7 +4998,6 @@ var ExuluApp = class {
         await createExpressRoutes(
           app,
           this._agents,
-          this._embedders,
           this._tools,
           this._workflows,
           Object.values(this._contexts ?? {})
@@ -4472,7 +5007,6 @@ var ExuluApp = class {
           await mcp.create({
             express: app,
             contexts: this._contexts,
-            embedders: this._embedders,
             agents: this._agents,
             workflows: this._workflows,
             config: this._config,
@@ -5680,198 +6214,6 @@ var SentenceChunker = class _SentenceChunker extends BaseChunker {
   }
 };
 
-// src/cli/index.tsx
-import { useState as useState2 } from "react";
-import { Box as Box2, Text as Text4, render as render2 } from "ink";
-import { UnorderedList as UnorderedList3 } from "@inkjs/ui";
-import patchConsole from "patch-console";
-
-// src/cli/components/nav.tsx
-import { Select } from "@inkjs/ui";
-import { useApp } from "ink";
-import { jsx } from "react/jsx-runtime";
-var nav = [
-  {
-    label: "Agents",
-    value: "agents"
-  },
-  {
-    label: "Start Claude Code",
-    value: "claude-code"
-  },
-  {
-    label: "Exit",
-    value: "exit"
-  }
-];
-var Nav = ({ setView }) => {
-  const { exit } = useApp();
-  return /* @__PURE__ */ jsx(Select, { options: nav, onChange: (value) => {
-    if (value === "exit") {
-      exit();
-    }
-    setView(value);
-  } });
-};
-var nav_default = Nav;
-
-// src/cli/components/agent-selector.tsx
-import { Text as Text2 } from "ink";
-import { Select as Select2 } from "@inkjs/ui";
-import { Fragment, jsx as jsx2, jsxs } from "react/jsx-runtime";
-var AgentSelector = ({ exulu, setAgent, setEvaluations }) => {
-  const agents = exulu.agents.map((agent) => ({
-    label: agent.name,
-    value: agent.id
-  }));
-  return /* @__PURE__ */ jsxs(Fragment, { children: [
-    /* @__PURE__ */ jsx2(Text2, { children: "Please select an agent:" }),
-    /* @__PURE__ */ jsx2(Select2, { options: agents, onChange: (value) => {
-      const agent = exulu.agent(value);
-      if (!agent) {
-        console.error("Agent not found", value);
-        return;
-      }
-      setAgent(agent);
-      if (agent) {
-        setEvaluations(agent.evals || []);
-      }
-    } })
-  ] });
-};
-var agent_selector_default = AgentSelector;
-
-// src/cli/components/eval-selector.tsx
-import { Select as Select3 } from "@inkjs/ui";
-import { jsx as jsx3 } from "react/jsx-runtime";
-var EvalSelector = ({ evaluations, setEvaluation }) => {
-  return /* @__PURE__ */ jsx3(Select3, { options: evaluations.map((evaluation) => ({
-    label: evaluation.runner.name,
-    value: evaluation.runner.name
-  })), onChange: (value) => {
-    const evaluation = evaluations?.find((evaluation2) => evaluation2.runner.name === value);
-    if (evaluation) {
-      setEvaluation(evaluation);
-    }
-  } });
-};
-var eval_selector_default = EvalSelector;
-
-// src/cli/components/eval-actions.tsx
-import { useState } from "react";
-import { ProgressBar as ProgressBar2, Select as Select4, UnorderedList as UnorderedList2 } from "@inkjs/ui";
-import { Text as Text3 } from "ink";
-import { Fragment as Fragment2, jsx as jsx4, jsxs as jsxs2 } from "react/jsx-runtime";
-var EvalActions = ({ agent, evaluation, setEvaluation }) => {
-  const [progress, setProgress] = useState(0);
-  const [results, setResults] = useState([]);
-  const [running, setRunning] = useState();
-  const run = async (evaluation2) => {
-    setRunning({
-      label: evaluation2.runner.name
-    });
-    const testCases = evaluation2.runner.testcases;
-    const total = testCases.length;
-    if (!testCases) {
-      throw new Error("No test cases found");
-    }
-    let i = 0;
-    for (const testCase of testCases) {
-      i++;
-      const result = await evaluation2.runner.run({
-        data: testCase,
-        runner: {
-          agent
-        }
-      });
-      setProgress(Math.round(i / total * 100));
-      setResults([...results, {
-        name: evaluation2.runner.name,
-        prompt: testCase.prompt?.slice(0, 100) + "...",
-        score: result.score,
-        comment: result.comment
-      }]);
-    }
-    setRunning(void 0);
-  };
-  if (progress === 100) {
-    return /* @__PURE__ */ jsxs2(Fragment2, { children: [
-      /* @__PURE__ */ jsx4(Text3, { children: "Evaluations completed." }),
-      /* @__PURE__ */ jsx4(UnorderedList2, { children: results.map((result) => /* @__PURE__ */ jsx4(UnorderedList2.Item, { children: /* @__PURE__ */ jsxs2(Text3, { children: [
-        result.name,
-        ": ",
-        result.score,
-        " - ",
-        result.comment
-      ] }) })) })
-    ] });
-  }
-  if (running) {
-    return /* @__PURE__ */ jsxs2(Fragment2, { children: [
-      /* @__PURE__ */ jsxs2(Text3, { children: [
-        "Running ",
-        running.label,
-        "..."
-      ] }),
-      /* @__PURE__ */ jsx4(ProgressBar2, { value: progress })
-    ] });
-  }
-  return /* @__PURE__ */ jsx4(Select4, { options: [{
-    label: "Run evaluation",
-    value: "run"
-  }, {
-    label: "Go back",
-    value: "back"
-  }], onChange: (value) => {
-    if (value === "back") {
-      setEvaluation(void 0);
-    }
-    if (value === "run") {
-      run(evaluation);
-    }
-  } });
-};
-var eval_actions_default = EvalActions;
-
-// src/cli/index.tsx
-import { Fragment as Fragment3, jsx as jsx5, jsxs as jsxs3 } from "react/jsx-runtime";
-var Main = ({ exulu }) => {
-  patchConsole((stream, data) => {
-    setLogs([...logs, data]);
-  });
-  const [logs, setLogs] = useState2([]);
-  const [view, setView] = useState2();
-  const [agent, setAgent] = useState2();
-  const [evaluations, setEvaluations] = useState2([]);
-  const [evaluation, setEvaluation] = useState2();
-  return /* @__PURE__ */ jsxs3(Box2, { borderStyle: "round", borderColor: "cyan", padding: 1, flexDirection: "column", width: "70%", children: [
-    /* @__PURE__ */ jsx5(Text4, { children: "Logs:" }),
-    /* @__PURE__ */ jsx5(UnorderedList3, { children: logs.map((log, index) => /* @__PURE__ */ jsx5(UnorderedList3.Item, { children: /* @__PURE__ */ jsx5(Text4, { children: log }) })) }),
-    !view && /* @__PURE__ */ jsx5(nav_default, { setView }),
-    view === "agents" && !agent && /* @__PURE__ */ jsx5(agent_selector_default, { exulu, setAgent, setEvaluations }),
-    view === "agents" && agent && !evaluation && /* @__PURE__ */ jsxs3(Fragment3, { children: [
-      /* @__PURE__ */ jsxs3(Text4, { children: [
-        'Selected agent "',
-        agent.name,
-        '". Please select an evaluation:'
-      ] }),
-      /* @__PURE__ */ jsx5(eval_selector_default, { evaluations, setEvaluation })
-    ] }),
-    view === "agents" && agent && evaluation && /* @__PURE__ */ jsxs3(Fragment3, { children: [
-      /* @__PURE__ */ jsxs3(Text4, { children: [
-        "Selected evaluation: ",
-        evaluation.runner.name
-      ] }),
-      /* @__PURE__ */ jsx5(eval_actions_default, { agent, evaluation, setEvaluation })
-    ] })
-  ] });
-};
-var cli_default = {
-  run: (exulu) => {
-    render2(/* @__PURE__ */ jsx5(Main, { exulu }));
-  }
-};
-
 // src/index.ts
 var ExuluJobs = {
   redis: redisClient,
@@ -5901,7 +6243,6 @@ export {
   ExuluApp,
   authentication as ExuluAuthentication,
   ExuluChunkers,
-  cli_default as ExuluCli,
   ExuluContext,
   ExuluDatabase,
   ExuluEmbedder,