@exulu/backend 1.23.3 → 1.25.0

package/dist/index.js

@@ -63,11 +63,8 @@ var validateJob = (job) => {
 };
 
 // src/registry/classes.ts
-import "zod";
 import "bullmq";
 import { z } from "zod";
-import "fs";
-import "path";
 import { convertToModelMessages, createIdGenerator, generateObject, generateText, streamText, tool, validateUIMessages, stepCountIs } from "ai";
 
 // types/enums/statistics.ts
@@ -94,8 +91,9 @@ import "knex";
 import "pgvector/knex";
 var db = {};
 var databaseExistsChecked = false;
+var dbName = process.env.POSTGRES_DB_NAME || "exulu";
 async function ensureDatabaseExists() {
-  console.log("[EXULU] Ensuring exulu database exists...");
+  console.log(`[EXULU] Ensuring ${dbName} database exists...`);
   const defaultKnex = Knex({
     client: "pg",
     connection: {
@@ -110,15 +108,18 @@ async function ensureDatabaseExists() {
   });
   try {
     const result = await defaultKnex.raw(`
-            SELECT 1 FROM pg_database WHERE datname = 'exulu'
+            SELECT 1 FROM pg_database WHERE datname = '${dbName}'
         `);
     if (result.rows.length === 0) {
-      console.log("[EXULU] Database 'exulu' does not exist. Creating it...");
-      await defaultKnex.raw(`CREATE DATABASE exulu`);
-      console.log("[EXULU] Database 'exulu' created successfully.");
+      console.log(`[EXULU] Database '${dbName}' does not exist. Creating it...`);
+      await defaultKnex.raw(`CREATE DATABASE ${dbName}`);
+      console.log(`[EXULU] Database '${dbName}' created successfully.`);
     } else {
-      console.log("[EXULU] Database 'exulu' already exists.");
+      console.log(`[EXULU] Database '${dbName}' already exists.`);
     }
+  } catch (error) {
+    console.error("[EXULU] Error while checking to ensure the database exists, this could be if the user running the server does not have database admin rights, it is fine to ignore this if you are sure the database exists.", error);
+    return;
   } finally {
     await defaultKnex.destroy();
   }
@@ -126,7 +127,7 @@ async function ensureDatabaseExists() {
 async function postgresClient() {
   if (!db["exulu"]) {
     try {
-      console.log("[EXULU] Connecting to exulu database.");
+      console.log(`[EXULU] Connecting to ${dbName} database.`);
       console.log("[EXULU] POSTGRES_DB_HOST:", process.env.POSTGRES_DB_HOST);
       console.log("[EXULU] POSTGRES_DB_PORT:", process.env.POSTGRES_DB_PORT);
       console.log("[EXULU] POSTGRES_DB_USER:", process.env.POSTGRES_DB_USER);
@@ -134,7 +135,7 @@ async function postgresClient() {
       console.log("[EXULU] POSTGRES_DB_SSL:", process.env.POSTGRES_DB_SSL);
       console.log("[EXULU] Database exists checked:", databaseExistsChecked);
       if (!databaseExistsChecked) {
-        console.log("[EXULU] Ensuring exulu database exists...");
+        console.log(`[EXULU] Ensuring ${dbName} database exists...`);
         await ensureDatabaseExists();
         databaseExistsChecked = true;
       }
@@ -144,12 +145,16 @@ async function postgresClient() {
           host: process.env.POSTGRES_DB_HOST,
           port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
           user: process.env.POSTGRES_DB_USER,
-          database: "exulu",
+          database: dbName,
           password: process.env.POSTGRES_DB_PASSWORD,
           ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
         }
       });
-      await knex.schema.createExtensionIfNotExists("vector");
+      try {
+        await knex.schema.createExtensionIfNotExists("vector");
+      } catch (error) {
+        console.error("[EXULU] Error creating vector extension, this might be fine if you already activated the extension and the 'user' running this script does not have higher level database permissions.", error);
+      }
       db["exulu"] = knex;
     } catch (error) {
       console.error("[EXULU] Error initializing exulu database.", error);
@@ -160,6 +165,16 @@ async function postgresClient() {
     db: db["exulu"]
   };
 }
+var refreshPostgresClient = async () => {
+  if (db["exulu"]) {
+    await db["exulu"].destroy();
+    db["exulu"] = void 0;
+  }
+  const { db: refreshed } = await postgresClient();
+  return {
+    db: refreshed
+  };
+};
 
 // src/registry/classes.ts
 import pgvector2 from "pgvector/knex";
@@ -234,6 +249,15 @@ var mapType = (t, type, name, defaultValue, unique) => {
     if (unique) t.unique(name);
     return;
   }
+  if (type === "markdown") {
+    if (defaultValue) {
+      t.text(name).defaultTo(defaultValue);
+    } else {
+      t.text(name);
+    }
+    if (unique) t.unique(name);
+    return;
+  }
   if (type === "shortText") {
     if (defaultValue) {
       t.string(name, 100).defaultTo(defaultValue);
@@ -381,7 +405,6 @@ var ExuluEvalUtils = {
 // src/registry/classes.ts
 import CryptoJS2 from "crypto-js";
 import "express";
-import "@opentelemetry/api";
 
 // src/registry/utils/graphql.ts
 import { makeExecutableSchema } from "@graphql-tools/schema";
@@ -394,7 +417,7 @@ import { jwtVerify, importJWK } from "jose";
 var getToken = async (authHeader) => {
   const token = authHeader.split(" ")[1];
   if (!token) {
-    throw new Error("No token provided");
+    throw new Error("No token provided for user authentication in headers.");
   }
   if (!process.env.NEXTAUTH_SECRET) {
     throw new Error("No NEXTAUTH_SECRET provided");
@@ -437,7 +460,7 @@ var authentication = async ({
       code: 200,
       user: {
         type: "api",
-        id: "XXXX-XXXX-XXXX-XXXX",
+        id: 192837465,
         email: "internal@exulu.com",
         role: {
           id: "internal",
@@ -758,7 +781,8 @@ var agentSessionsSchema = {
     },
     {
       name: "project",
-      type: "uuid"
+      type: "uuid",
+      required: false
     }
   ]
 };
@@ -858,12 +882,13 @@ var projectsSchema = {
       type: "text"
     },
     {
-      name: "custom_instructions",
-      type: "longText"
+      name: "project_items",
+      // array of items as global ids ('<context_id>/<item_id>')
+      type: "json"
     },
     {
-      name: "context_files",
-      type: "json"
+      name: "custom_instructions",
+      type: "longText"
     }
   ]
 };
@@ -883,20 +908,24 @@ var agentsSchema = {
       name: "image",
       type: "text"
     },
+    {
+      name: "category",
+      type: "text"
+    },
     {
       name: "description",
       type: "text"
     },
     {
-      name: "providerApiKey",
+      name: "instructions",
       type: "text"
     },
     {
-      name: "backend",
+      name: "providerapikey",
       type: "text"
     },
     {
-      name: "type",
+      name: "backend",
       type: "text"
     },
     {
@@ -1230,18 +1259,22 @@ var rbacSchema = {
 };
 var addRBACfields = (schema) => {
   if (schema.RBAC) {
-    schema.fields.push({
-      name: "rights_mode",
-      type: "text",
-      required: false,
-      default: "private"
-    });
-    schema.fields.push({
-      name: "created_by",
-      type: "number",
-      required: true,
-      default: 0
-    });
+    if (!schema.fields.some((field) => field.name === "rights_mode")) {
+      schema.fields.push({
+        name: "rights_mode",
+        type: "text",
+        required: false,
+        default: "private"
+      });
+    }
+    if (!schema.fields.some((field) => field.name === "created_by")) {
+      schema.fields.push({
+        name: "created_by",
+        type: "number",
+        required: true,
+        default: 0
+      });
+    }
   }
   return schema;
 };
@@ -1273,6 +1306,210 @@ var VectorMethodEnum = {
 
 // src/registry/utils/graphql.ts
 import "knex";
+
+// src/registry/rate-limiter.ts
+var rateLimiter = async (key, windowSeconds, limit, points) => {
+  try {
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      console.warn("[EXULU] Rate limiting disabled - Redis not available");
+      return {
+        status: true,
+        retryAfter: null
+      };
+    }
+    const redisKey = `exulu/${key}`;
+    const current = await client2.incrBy(redisKey, points);
+    if (current === points) {
+      await client2.expire(redisKey, windowSeconds);
+    }
+    if (current > limit) {
+      const ttl = await client2.ttl(redisKey);
+      return {
+        status: false,
+        retryAfter: ttl
+      };
+    }
+    return {
+      status: true,
+      retryAfter: null
+    };
+  } catch (error) {
+    console.error("[EXULU] Rate limiting error:", error);
+    return {
+      status: true,
+      retryAfter: null
+    };
+  }
+};
+
+// src/registry/utils.ts
+var bullmq = {
+  validate: (id, data) => {
+    if (!data) {
+      throw new Error(`Missing job data for job ${id}.`);
+    }
+    if (!data.type) {
+      throw new Error(`Missing property "type" in data for job ${id}.`);
+    }
+    if (!data.inputs) {
+      throw new Error(`Missing property "inputs" in data for job ${id}.`);
+    }
+    if (data.type !== "embedder" && data.type !== "workflow") {
+      throw new Error(`Property "type" in data for job ${id} must be of value "embedder" or "workflow".`);
+    }
+    if (!data.workflow && !data.embedder) {
+      throw new Error(`Either a workflow or embedder must be set for job ${id}.`);
+    }
+  }
+};
+var getEnabledTools = async (agentInstance, allExuluTools, disabledTools = [], agents, user) => {
+  let enabledTools = [];
+  if (agentInstance.tools) {
+    const results = await Promise.all(agentInstance.tools.map(
+      async ({ config, id, type }) => {
+        let hydrated;
+        if (type === "agent") {
+          if (id === agentInstance.id) {
+            return null;
+          }
+          const instance = await loadAgent(id);
+          if (!instance) {
+            throw new Error("Trying to load a tool of type 'agent', but the associated agent with id " + id + " was not found in the database.");
+          }
+          const backend = agents.find((a) => a.id === instance.backend);
+          if (!backend) {
+            throw new Error("Trying to load a tool of type 'agent', but the associated agent with id " + id + " does not have a backend set for it.");
+          }
+          const hasAccessToAgent = await checkRecordAccess(instance, "read", user);
+          if (!hasAccessToAgent) {
+            return null;
+          }
+          hydrated = await backend.tool(instance.id, agents);
+        } else {
+          hydrated = allExuluTools.find((t) => t.id === id);
+        }
+        return hydrated;
+      }
+    ));
+    enabledTools = results.filter(Boolean);
+  }
+  console.log("[EXULU] available tools", enabledTools?.length);
+  console.log("[EXULU] disabled tools", disabledTools?.length);
+  enabledTools = enabledTools.filter((tool2) => !disabledTools.includes(tool2.id));
+  return enabledTools;
+};
+var loadAgentCache = /* @__PURE__ */ new Map();
+var loadAgents = async () => {
+  const { db: db3 } = await postgresClient();
+  const agents = await db3.from("agents");
+  for (const agent of agents) {
+    const agentRbac = await RBACResolver(db3, "agent", agent.id, agent.rights_mode || "private");
+    agent.RBAC = agentRbac;
+    loadAgentCache.set(agent.id, {
+      agent,
+      expiresAt: new Date(Date.now() + 1e3 * 60 * 1)
+      // 1 minute
+    });
+  }
+  return agents;
+};
+var loadAgent = async (id) => {
+  const cachedAgent = loadAgentCache.get(id);
+  if (cachedAgent && cachedAgent.expiresAt > /* @__PURE__ */ new Date()) {
+    return cachedAgent.agent;
+  }
+  const { db: db3 } = await postgresClient();
+  const agentInstance = await db3.from("agents").where({
+    id
+  }).first();
+  const agentRbac = await RBACResolver(db3, "agent", agentInstance.id, agentInstance.rights_mode || "private");
+  agentInstance.RBAC = agentRbac;
+  if (!agentInstance) {
+    throw new Error("Agent instance not found.");
+  }
+  loadAgentCache.set(id, {
+    agent: agentInstance,
+    expiresAt: new Date(Date.now() + 1e3 * 60 * 1)
+    // 1 minute
+  });
+  return agentInstance;
+};
+var checkAgentRateLimit = async (agent) => {
+  if (agent.rateLimit) {
+    console.log("[EXULU] rate limiting agent.", agent.rateLimit);
+    const limit = await rateLimiter(
+      agent.rateLimit.name || agent.id,
+      agent.rateLimit.rate_limit.time,
+      agent.rateLimit.rate_limit.limit,
+      1
+    );
+    if (!limit.status) {
+      throw new Error("Rate limit exceeded.");
+    }
+  }
+};
+var checkRecordAccessCache = /* @__PURE__ */ new Map();
+var checkRecordAccess = async (record, request, user) => {
+  const setRecordAccessCache = (hasAccess2) => {
+    checkRecordAccessCache.set(`${record.id}-${request}-${user?.id}`, {
+      hasAccess: hasAccess2,
+      expiresAt: new Date(Date.now() + 1e3 * 60 * 1)
+      // 1 minute
+    });
+  };
+  const cachedAccess = checkRecordAccessCache.get(`${record.id}-${request}-${user?.id}`);
+  if (cachedAccess && cachedAccess.expiresAt > /* @__PURE__ */ new Date()) {
+    return cachedAccess.hasAccess;
+  }
+  const isPublic = record.rights_mode === "public";
+  const byUsers = record.rights_mode === "users";
+  const byRoles = record.rights_mode === "roles";
+  const isCreator = user ? record.created_by === user.id.toString() : false;
+  const isAdmin = user ? user.super_admin : false;
+  const isApi = user ? user.type === "api" : false;
+  let hasAccess = "none";
+  if (isPublic || isCreator || isAdmin || isApi) {
+    setRecordAccessCache(true);
+    return true;
+  }
+  if (byUsers) {
+    if (!user) {
+      setRecordAccessCache(false);
+      return false;
+    }
+    console.log("record.RBAC?.users", record.RBAC?.users);
+    console.log("user.id", user.id.toString());
+    hasAccess = record.RBAC?.users?.find((x) => x.id === user.id)?.rights || "none";
+    if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
+      console.error(`Your current user ${user.id} does not have access to this record, current access type is: ${hasAccess}.`);
+      setRecordAccessCache(false);
+      return false;
+    } else {
+      setRecordAccessCache(true);
+      return true;
+    }
+  }
+  if (byRoles) {
+    if (!user) {
+      setRecordAccessCache(false);
+      return false;
+    }
+    hasAccess = record.RBAC?.roles?.find((x) => x.id === user.role?.id)?.rights || "none";
+    if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
+      console.error(`Your current role ${user.role?.name} does not have access to this record, current access type is: ${hasAccess}.`);
+      setRecordAccessCache(false);
+      return false;
+    } else {
+      setRecordAccessCache(true);
+      return true;
+    }
+  }
+  setRecordAccessCache(false);
+  return false;
+};
+
+// src/registry/utils/graphql.ts
 var GraphQLDate = new GraphQLScalarType({
   name: "Date",
   description: "Date custom scalar type",
@@ -1313,6 +1550,7 @@ var map = (field) => {
     case "text":
     case "shortText":
     case "longText":
+    case "markdown":
     case "code":
       type = "String";
       break;
@@ -1364,6 +1602,7 @@ ${enumValues}
     fields.push("  rateLimit: RateLimiterRule");
     fields.push("  streaming: Boolean");
     fields.push("  capabilities: AgentCapabilities");
+    fields.push("  maxContextLength: Int");
     fields.push("  slug: String");
   }
   const rbacField = table.RBAC ? "  RBAC: RBACData" : "";
@@ -1698,7 +1937,6 @@ function createMutations(table, agents, contexts, tools) {
         input.id = db3.fn.uuid();
       }
       const columns = await db3(tableNamePlural).columnInfo();
-      console.log("[EXULU] Columns", columns);
       const insert = db3(tableNamePlural).insert({
         ...input,
         ...table.RBAC ? { rights_mode: "private" } : {}
@@ -1713,7 +1951,7 @@ function createMutations(table, agents, contexts, tools) {
       const { job } = await postprocessUpdate({ table, requestedFields, agents, contexts, tools, result: results[0], user: context.user.id, role: context.user.role?.id });
       return {
         // Filter result to only include requested fields
-        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: results[0] }),
+        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: results[0], user: context.user }),
         job
       };
     },
@@ -1756,7 +1994,7 @@ function createMutations(table, agents, contexts, tools) {
       }
       const { job } = await postprocessUpdate({ table, requestedFields, agents, contexts, tools, result, user: context.user.id, role: context.user.role?.id });
       return {
-        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result }),
+        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user.id }),
         job
       };
     },
@@ -1792,7 +2030,7 @@ function createMutations(table, agents, contexts, tools) {
       const result = await db3.from(tableNamePlural).select(Object.keys(columns)).where({ id }).first();
       const { job } = await postprocessUpdate({ table, requestedFields, agents, contexts, tools, result, user: context.user.id, role: context.user.role?.id });
       return {
-        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result }),
+        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user.id }),
         job
       };
     },
@@ -1824,7 +2062,7 @@ function createMutations(table, agents, contexts, tools) {
         }).del();
       }
       await postprocessDeletion({ table, requestedFields, agents, contexts, tools, result });
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user.id });
     },
     [`${tableNamePlural}RemoveOne`]: async (_, args, context, info) => {
       const { where } = args;
@@ -1848,7 +2086,7 @@ function createMutations(table, agents, contexts, tools) {
       }
       await db3(tableNamePlural).where(where).del();
       await postprocessDeletion({ table, requestedFields, agents, contexts, tools, result });
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user.id });
     }
   };
   if (table.type === "items") {
@@ -1928,7 +2166,6 @@ function createMutations(table, agents, contexts, tools) {
   return mutations;
 }
 var applyAccessControl = (table, user, query) => {
-  console.log("table", table);
   const tableNamePlural = table.name.plural.toLowerCase();
   if (!user.super_admin && table.name.plural === "jobs") {
     query = query.where("created_by", user.id);
@@ -1955,7 +2192,6 @@ var applyAccessControl = (table, user, query) => {
         });
       });
       if (user.role) {
-        console.log("user.role", user.role);
         this.orWhere(function() {
           this.where("rights_mode", "roles").whereExists(function() {
             this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "Role").where("rbac.role_id", user.role.id);
@@ -1996,14 +2232,15 @@ var backendAgentFields = [
   "slug",
   "rateLimit",
   "streaming",
-  "capabilities"
+  "capabilities",
+  "maxContextLength"
 ];
 var removeAgentFields = (requestedFields) => {
   const filtered = requestedFields.filter((field) => !backendAgentFields.includes(field));
   filtered.push("backend");
   return filtered;
 };
-var addAgentFields = (requestedFields, agents, result, tools) => {
+var addAgentFields = async (requestedFields, agents, result, tools, user) => {
   let backend = agents.find((a) => a.id === result?.backend);
   if (requestedFields.includes("providerName")) {
     result.providerName = backend?.providerName || "";
@@ -2018,13 +2255,39 @@ var addAgentFields = (requestedFields, agents, result, tools) => {
     result.rateLimit = backend?.rateLimit || "";
   }
   if (requestedFields.includes("tools")) {
-    result.tools = result.tools ? result.tools.map((tool2) => {
-      return {
-        ...tool2,
-        name: tools.find((t) => t.id === tool2.toolId)?.name || "",
-        description: tools.find((t) => t.id === tool2.toolId)?.description || ""
-      };
-    }) : [];
+    if (result.tools) {
+      result.tools = await Promise.all(result.tools.map(async (tool2) => {
+        let hydrated;
+        if (tool2.type === "agent") {
+          if (tool2.id === result.id) {
+            return null;
+          }
+          const instance = await loadAgent(tool2.id);
+          if (!instance) {
+            throw new Error("Trying to load a tool of type 'agent', but the associated agent with id " + tool2.id + " was not found in the database.");
+          }
+          const backend2 = agents.find((a) => a.id === instance.backend);
+          if (!backend2) {
+            throw new Error("Trying to load a tool of type 'agent', but the associated agent with id " + tool2.id + " does not have a backend set for it.");
+          }
+          const hasAccessToAgent = await checkRecordAccess(instance, "read", user);
+          if (!hasAccessToAgent) {
+            return null;
+          }
+          hydrated = await backend2.tool(instance.id, agents);
+        } else {
+          hydrated = tools.find((t) => t.id === tool2.id);
+        }
+        return {
+          ...tool2,
+          name: hydrated?.name || "",
+          description: hydrated?.description || ""
+        };
+      }));
+      result.tools = result.tools.filter((tool2) => tool2 !== null);
+    } else {
+      result.tools = [];
+    }
   }
   if (requestedFields.includes("streaming")) {
     result.streaming = backend?.streaming || false;
@@ -2032,6 +2295,9 @@ var addAgentFields = (requestedFields, agents, result, tools) => {
   if (requestedFields.includes("capabilities")) {
     result.capabilities = backend?.capabilities || [];
   }
+  if (requestedFields.includes("maxContextLength")) {
+    result.maxContextLength = backend?.maxContextLength || 0;
+  }
   if (!requestedFields.includes("backend")) {
     delete result.backend;
   }
@@ -2081,7 +2347,11 @@ var postprocessUpdate = async ({
       const { db: db3 } = await postgresClient();
       console.log("[EXULU] Deleting chunks for item", result.id);
       await db3.from(getChunksTableName(context.id)).where({ source: result.id }).delete();
+      console.log("[EXULU] Deleted chunks for item", result.id);
+      console.log("[EXULU] Embedder", context.embedder);
+      console.log("[EXULU] Configuration", context.configuration);
       if (context.embedder && (context.configuration.calculateVectors === "onUpdate" || context.configuration.calculateVectors === "always")) {
+        console.log("[EXULU] Generating embeddings for item", result.id);
         const { job } = await context.embeddings.generate.one({
           item: result,
           user,
@@ -2142,7 +2412,8 @@ var finalizeRequestedFields = async ({
   agents,
   contexts,
   tools,
-  result
+  result,
+  user
 }) => {
   if (!result) {
     return result;
@@ -2152,11 +2423,11 @@ var finalizeRequestedFields = async ({
   }
   if (Array.isArray(result)) {
     result = result.map((item) => {
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: item });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: item, user });
     });
   } else {
     if (table.name.singular === "agent") {
-      result = addAgentFields(requestedFields, agents, result, tools);
+      result = await addAgentFields(requestedFields, agents, result, tools, user);
       if (!requestedFields.includes("backend")) {
         delete result.backend;
       }
@@ -2203,7 +2474,6 @@ var applyFilters = (query, filters) => {
     Object.entries(filter).forEach(([fieldName, operators]) => {
       if (operators) {
         if (operators.and !== void 0) {
-          console.log("operators.and", operators.and);
           operators.and.forEach((operator) => {
             query = converOperatorToQuery(query, fieldName, operator);
           });
@@ -2214,7 +2484,6 @@ var applyFilters = (query, filters) => {
           });
         }
         query = converOperatorToQuery(query, fieldName, operators);
-        console.log("query", query);
       }
     });
   });
@@ -2237,7 +2506,7 @@ function createQueries(table, agents, tools, contexts) {
       let query = db3.from(tableNamePlural).select(sanitizedFields).where({ id: args.id });
       query = applyAccessControl(table, context.user, query);
       let result = await query.first();
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user });
     },
     [`${tableNameSingular}ByIds`]: async (_, args, context, info) => {
       const { db: db3 } = context;
@@ -2246,7 +2515,7 @@ function createQueries(table, agents, tools, contexts) {
       let query = db3.from(tableNamePlural).select(sanitizedFields).whereIn("id", args.ids);
       query = applyAccessControl(table, context.user, query);
       let result = await query;
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user });
     },
     [`${tableNameSingular}One`]: async (_, args, context, info) => {
       const { filters = [], sort } = args;
@@ -2258,7 +2527,7 @@ function createQueries(table, agents, tools, contexts) {
       query = applyAccessControl(table, context.user, query);
       query = applySorting(query, sort);
       let result = await query.first();
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user });
     },
     [`${tableNamePlural}Pagination`]: async (_, args, context, info) => {
       const { limit = 10, page = 0, filters = [], sort } = args;
@@ -2269,7 +2538,6 @@ function createQueries(table, agents, tools, contexts) {
       let countQuery = db3(tableNamePlural);
       countQuery = applyFilters(countQuery, filters);
       countQuery = applyAccessControl(table, context.user, countQuery);
-      console.log("countQuery", countQuery);
       const countResult = await countQuery.count("* as count");
       const itemCount = Number(countResult[0]?.count || 0);
       const pageCount = Math.ceil(itemCount / limit);
@@ -2294,7 +2562,7 @@ function createQueries(table, agents, tools, contexts) {
           hasPreviousPage,
           hasNextPage
         },
-        items: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: items })
+        items: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: items, user: context.user })
       };
     },
     // Add generic statistics query for all tables
@@ -2312,7 +2580,6 @@ function createQueries(table, agents, tools, contexts) {
           query = query.count("* as count");
         }
         const results = await query;
-        console.log("!!! results !!!", results);
         return results.map((r) => ({
           group: r[groupBy],
           count: r.count ? Number(r.count) : 0
@@ -2321,7 +2588,6 @@ function createQueries(table, agents, tools, contexts) {
         if (tableNamePlural === "tracking") {
           query = query.sum("total as count");
           const [{ count }] = await query.sum("total as count");
-          console.log("!!! count !!!", count);
           return [{
             group: "total",
             count: count ? Number(count) : 0
@@ -2406,7 +2672,6 @@ var vectorSearch = async ({
   let countQuery = db3(mainTable);
   countQuery = applyFilters(countQuery, filters);
   countQuery = applyAccessControl(table, user, countQuery);
-  console.log("countQuery", countQuery);
   const columns = await db3(mainTable).columnInfo();
   let itemsQuery = db3(mainTable).select(Object.keys(columns).map((column) => mainTable + "." + column));
   itemsQuery = applyFilters(itemsQuery, filters);
@@ -2545,7 +2810,6 @@ var vectorSearch = async ({
       ];
       items = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
   }
-  console.log("items", items);
   const seenSources = /* @__PURE__ */ new Map();
   items = items.reduce((acc, item) => {
     if (!seenSources.has(item.source)) {
@@ -2588,7 +2852,6 @@ var vectorSearch = async ({
     }
     return acc;
   }, []);
-  console.log("items", items);
   items.forEach((item) => {
     if (!item.chunks?.length) {
       return;
@@ -2610,7 +2873,6 @@ var vectorSearch = async ({
       item.averageRelevance = average;
       item.totalRelevance = total;
     } else if (method === "hybridSearch") {
-      console.log("item.chunks", item.chunks);
       const scores = item.chunks.map((c) => typeof c.hybrid_score === "number" ? c.hybrid_score * 10 + 1 : 0);
       const total = scores.reduce((a, b) => a + b, 0);
       const average = scores.length ? total / scores.length : 0;
@@ -2699,6 +2961,10 @@ var contextToTableDefinition = (context) => {
     name: "textlength",
     type: "number"
   });
+  definition.fields.push({
+    name: "ttl",
+    type: "text"
+  });
   definition.fields.push({
     name: "embeddings_updated_at",
     type: "date"
@@ -2983,8 +3249,19 @@ type PageInfo {
   };
   resolvers.Query["tools"] = async (_, args, context, info) => {
     const requestedFields = getRequestedFields(info);
+    const instances = await loadAgents();
+    let agentTools = await Promise.all(
+      instances.map(async (instance) => {
+        const backend = agents.find((a) => a.id === instance.backend);
+        if (!backend) {
+          return null;
+        }
+        return await backend.tool(instance.id, agents);
+      })
+    );
+    const filtered = agentTools.filter((tool2) => tool2 !== null);
     return {
-      items: tools.map((tool2) => {
+      items: [...filtered, ...tools].map((tool2) => {
         const object = {};
         requestedFields.forEach((field) => {
           object[field] = tool2[field];
@@ -3080,7 +3357,6 @@ type Tool {
 
 enum EnumProviderType {
   agent
-  custom
 }
 
 type StatisticsResult {
@@ -3114,6 +3390,13 @@ var validateCreateOrRemoveSuperAdminPermission = async (tableNamePlural, input,
 };
 
 // src/registry/classes.ts
+import {
+  PutObjectCommand,
+  S3Client,
+  S3ServiceException
+} from "@aws-sdk/client-s3";
+import { randomUUID } from "crypto";
+var s3Client;
 function sanitizeToolName(name) {
   if (typeof name !== "string") return "";
   let sanitized = name.replace(/[^a-zA-Z0-9_-]+/g, "_");
@@ -3123,13 +3406,14 @@ function sanitizeToolName(name) {
   }
   return sanitized;
 }
-var convertToolsArrayToObject = (tools, configs, providerApiKey, user, role) => {
-  if (!tools) return {};
-  const sanitizedTools = tools ? tools.map((tool2) => ({
+var convertToolsArrayToObject = (currentTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, filesContext2) => {
+  if (!currentTools) return {};
+  if (!allExuluTools) return {};
+  const sanitizedTools = currentTools ? currentTools.map((tool2) => ({
     ...tool2,
     name: sanitizeToolName(tool2.name)
   })) : [];
-  console.log("[EXULU] Sanitized tools", sanitizedTools);
+  console.log("[EXULU] Sanitized tools", sanitizedTools.map((x) => x.name + " (" + x.id + ")"));
   const askForConfirmation = {
     description: "Ask the user for confirmation.",
     inputSchema: z.object({
@@ -3142,29 +3426,91 @@ var convertToolsArrayToObject = (tools, configs, providerApiKey, user, role) =>
         ...prev,
         [cur.name]: {
           ...cur.tool,
-          execute: async (inputs, options) => {
+          async *execute(inputs, options) {
             if (!cur.tool?.execute) {
               console.error("[EXULU] Tool execute function is undefined.", cur.tool);
               throw new Error("Tool execute function is undefined.");
             }
-            let config = configs?.find((config2) => config2.toolId === cur.id);
+            let config = configs?.find((config2) => config2.id === cur.id);
             if (config) {
               config = await hydrateVariables(config || []);
             }
+            let upload = void 0;
+            if (exuluConfig?.fileUploads?.s3endpoint && exuluConfig?.fileUploads?.s3key && exuluConfig?.fileUploads?.s3secret && exuluConfig?.fileUploads?.s3Bucket && filesContext2) {
+              s3Client ??= new S3Client({
+                region: exuluConfig?.fileUploads?.s3region,
+                ...exuluConfig?.fileUploads?.s3endpoint && {
+                  forcePathStyle: true,
+                  endpoint: exuluConfig?.fileUploads?.s3endpoint
+                },
+                credentials: {
+                  accessKeyId: exuluConfig?.fileUploads?.s3key ?? "",
+                  secretAccessKey: exuluConfig?.fileUploads?.s3secret ?? ""
+                }
+              });
+              upload = async ({
+                name,
+                data,
+                type,
+                tags
+              }) => {
+                const mime = getMimeType(type);
+                const key = `${user}/${generateS3Key(name)}${type}`;
+                const command = new PutObjectCommand({
+                  Bucket: exuluConfig?.fileUploads?.s3Bucket,
+                  Key: key,
+                  Body: data,
+                  ContentType: mime
+                });
+                try {
+                  const response2 = await s3Client.send(command);
+                  console.log(response2);
+                  const { item } = await filesContext2.createItem({
+                    name: `${name}${type}`,
+                    type: mime,
+                    rights_mode: "private",
+                    s3key: key,
+                    tags
+                  }, user?.id, user?.role?.id, false);
+                  return item;
+                } catch (caught) {
+                  if (caught instanceof S3ServiceException && caught.name === "EntityTooLarge") {
+                    console.error(
+                      `Error from S3 while uploading object to ${exuluConfig?.fileUploads?.s3Bucket}.                                 The object was too large. To upload objects larger than 5GB, use the S3 console (160GB max)                                 or the multipart upload API (5TB max).`
+                    );
+                  } else if (caught instanceof S3ServiceException) {
+                    console.error(
+                      `Error from S3 while uploading object to ${exuluConfig?.fileUploads?.s3Bucket}.  ${caught.name}: ${caught.message}`
+                    );
+                  } else {
+                    throw caught;
+                  }
+                }
+              };
+            }
+            const contextsMap = contexts?.reduce((acc, curr) => {
+              acc[curr.id] = curr;
+              return acc;
+            }, {});
             console.log("[EXULU] Config", config);
-            return await cur.tool.execute({
+            const response = await cur.tool.execute({
               ...inputs,
               // Convert config to object format if a config object 
               // is available, after we added the .value property
               // by hydrating it from the variables table.
-              providerApiKey,
+              providerapikey,
+              allExuluTools,
+              currentTools,
               user,
-              role,
+              contexts: contextsMap,
+              upload,
               config: config ? config.config.reduce((acc, curr) => {
                 acc[curr.name] = curr.value;
                 return acc;
               }, {}) : {}
             }, options);
+            yield response;
+            return response;
           }
         }
       }),
@@ -3199,6 +3545,18 @@ function generateSlug(name) {
   const slug = lowercase.replace(/[\W_]+/g, "-").replace(/^-+|-+$/g, "");
   return slug;
 }
+function errorHandler(error) {
+  if (error == null) {
+    return "unknown error";
+  }
+  if (typeof error === "string") {
+    return error;
+  }
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return JSON.stringify(error);
+}
 var ExuluAgent2 = class {
   // Must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or 
   // underscores and be a max length of 80 characters and at least 5 characters long.
@@ -3209,13 +3567,14 @@ var ExuluAgent2 = class {
   slug = "";
   type;
   streaming = false;
+  maxContextLength;
   rateLimit;
   config;
   // private memory: Memory | undefined; // TODO do own implementation
   evals;
   model;
   capabilities;
-  constructor({ id, name, description, config, rateLimit, capabilities, type, evals }) {
+  constructor({ id, name, description, config, rateLimit, capabilities, type, evals, maxContextLength }) {
     this.id = id;
     this.name = name;
     this.evals = evals;
@@ -3223,6 +3582,7 @@ var ExuluAgent2 = class {
     this.rateLimit = rateLimit;
     this.config = config;
     this.type = type;
+    this.maxContextLength = maxContextLength;
     this.capabilities = capabilities || {
       text: false,
       images: [],
@@ -3249,31 +3609,82 @@ var ExuluAgent2 = class {
   }
   // Exports the agent as a tool that can be used by another agent
   // todo test this
-  tool = () => {
+  tool = async (instance, agents) => {
+    const agentInstance = await loadAgent(instance);
+    if (!agentInstance) {
+      return null;
+    }
     return new ExuluTool2({
-      id: this.id,
-      name: `${this.name}`,
+      id: agentInstance.id,
+      name: `${agentInstance.name}`,
       type: "agent",
       inputSchema: z.object({
-        prompt: z.string()
+        prompt: z.string().describe("The prompt (usually a question for the agent) to send to the agent."),
+        information: z.string().describe("A summary of relevant context / information from the current session")
       }),
-      description: `A function that calls an AI agent named: ${this.name}. The agent does the following: ${this.description}.`,
+      description: `This tool calls an AI agent named: ${agentInstance.name}. The agent does the following: ${agentInstance.description}.`,
       config: [],
-      execute: async ({ prompt, config, providerApiKey, user, role }) => {
-        return await this.generateSync({
-          prompt,
-          providerApiKey,
+      execute: async ({ prompt, information, user, allExuluTools }) => {
+        const hasAccessToAgent = await checkRecordAccess(agentInstance, "read", user);
+        if (!hasAccessToAgent) {
+          throw new Error("You don't have access to this agent.");
+        }
+        let enabledTools = await getEnabledTools(agentInstance, allExuluTools, [], agents, user);
+        const variableName = agentInstance.providerapikey;
+        if (!variableName) {
+          throw new Error("Provider API key variable not set for agent: " + agentInstance.name + " (" + agentInstance.id + ") being called as a tool.");
+        }
+        const { db: db3 } = await postgresClient();
+        const variable = await db3.from("variables").where({ name: variableName }).first();
+        if (!variable) {
+          throw new Error("Provider API key variable not found for agent: " + agentInstance.name + " (" + agentInstance.id + ") being called as a tool.");
+        }
+        let providerapikey = variable.value;
+        if (!variable.encrypted) {
+          throw new Error("Provider API key variable not encrypted for agent: " + agentInstance.name + " (" + agentInstance.id + ") being called as a tool, for security reasons you are only allowed to use encrypted variables for provider API keys.");
+        }
+        if (variable.encrypted) {
+          const bytes = CryptoJS2.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+          providerapikey = bytes.toString(CryptoJS2.enc.Utf8);
+        }
+        console.log("[EXULU] Enabled tools for agent '" + agentInstance.name + " (" + agentInstance.id + ") that is being called as a tool", enabledTools.map((x) => x.name + " (" + x.id + ")"));
+        console.log("[EXULU] Prompt for agent '" + agentInstance.name + "' that is being called as a tool", prompt.slice(0, 100) + "...");
+        console.log("[EXULU] Instructions for agent '" + agentInstance.name + "' that is being called as a tool", agentInstance.instructions?.slice(0, 100) + "...");
+        const response = await this.generateSync({
+          instructions: agentInstance.instructions,
+          prompt: "The user has asked the following question: " + prompt + " and the following information is available: " + information,
+          providerapikey,
           user,
-          role,
+          currentTools: enabledTools,
+          allExuluTools,
           statistics: {
-            label: "",
+            label: agentInstance.name,
             trigger: "tool"
           }
         });
+        return {
+          result: response
+        };
       }
     });
   };
-  generateSync = async ({ prompt, user, role, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
+  generateSync = async ({
+    prompt,
+    user,
+    session,
+    message,
+    currentTools,
+    allExuluTools,
+    statistics,
+    toolConfigs,
+    providerapikey,
+    contexts,
+    exuluConfig,
+    filesContext: filesContext2,
+    outputSchema,
+    instructions
+  }) => {
+    console.log("[EXULU] Called generate sync for agent: " + this.name, "with prompt: " + prompt?.slice(0, 100) + "...");
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
@@ -3286,14 +3697,18 @@ var ExuluAgent2 = class {
     if (!prompt && !message) {
       throw new Error("Prompt or message is required for generating.");
     }
+    if (outputSchema && !prompt) {
+      throw new Error("Prompt is required for generating with an output schema.");
+    }
     const model = this.model.create({
-      apiKey: providerApiKey
+      apiKey: providerapikey
     });
+    console.log("[EXULU] Model for agent: " + this.name, " created for generating sync.");
     let messages = [];
     if (message && session && user) {
       const previousMessages = await getAgentMessages({
         session,
-        user,
+        user: user.id,
         limit: 50,
         page: 1
       });
@@ -3303,56 +3718,134 @@ var ExuluAgent2 = class {
         messages: [...previousMessagesContent, message]
       });
     }
-    console.log("[EXULU] Model provider key", providerApiKey);
-    console.log("[EXULU] Tool configs", toolConfigs);
+    console.log("[EXULU] Message count for agent: " + this.name, "loaded for generating sync.", messages.length);
+    const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
+    let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
+    system += "\n\n" + genericContext;
     if (prompt) {
-      const { text } = await generateText({
-        model,
-        // Should be a LanguageModelV1
-        system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
-        prompt,
-        maxRetries: 2,
-        tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
-        stopWhen: [stepCountIs(5)]
-      });
-      if (statistics) {
-        await updateStatistic({
-          name: "count",
-          label: statistics.label,
-          type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-          trigger: statistics.trigger,
-          count: 1,
-          user,
-          role
+      let result = { object: null, text: "" };
+      let tokens = 0;
+      if (outputSchema) {
+        const { object, usage } = await generateObject({
+          model,
+          system,
+          prompt,
+          maxRetries: 3,
+          schema: outputSchema
+        });
+        result.object = object;
+        tokens = usage.totalTokens || 0;
+      } else {
+        console.log("[EXULU] Generating text for agent: " + this.name, "with prompt: " + prompt?.slice(0, 100) + "...");
+        const { text, totalUsage } = await generateText({
+          model,
+          system,
+          prompt,
+          maxRetries: 2,
+          tools: convertToolsArrayToObject(
+            currentTools,
+            allExuluTools,
+            toolConfigs,
+            providerapikey,
+            contexts,
+            user,
+            exuluConfig,
+            filesContext2
+          ),
+          stopWhen: [stepCountIs(2)]
         });
+        result.text = text;
+        tokens = totalUsage?.totalTokens || 0;
       }
-      return text;
+      if (statistics) {
+        await Promise.all([
+          updateStatistic({
+            name: "count",
+            label: statistics.label,
+            type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+            trigger: statistics.trigger,
+            count: 1,
+            user: user?.id,
+            role: user?.role?.id
+          }),
+          ...tokens ? [
+            updateStatistic({
+              name: "tokens",
+              label: statistics.label,
+              type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+              trigger: statistics.trigger,
+              count: tokens
+            })
+          ] : []
+        ]);
+      }
+      return result.text || result.object;
     }
     if (messages) {
-      const { text } = await generateText({
+      console.log("[EXULU] Generating text for agent: " + this.name, "with messages: " + messages.length);
+      const { text, totalUsage } = await generateText({
         model,
         // Should be a LanguageModelV1
-        system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
-        messages: convertToModelMessages(messages),
+        system,
+        messages: convertToModelMessages(messages, {
+          ignoreIncompleteToolCalls: true
+        }),
         maxRetries: 2,
-        tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
-        stopWhen: [stepCountIs(5)]
+        tools: convertToolsArrayToObject(
+          currentTools,
+          allExuluTools,
+          toolConfigs,
+          providerapikey,
+          contexts,
+          user,
+          exuluConfig,
+          filesContext2
+        ),
+        stopWhen: [stepCountIs(2)]
       });
       if (statistics) {
-        await updateStatistic({
-          name: "count",
-          label: statistics.label,
-          type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-          trigger: statistics.trigger,
-          count: 1,
-          user,
-          role
-        });
+        await Promise.all([
+          updateStatistic({
+            name: "count",
+            label: statistics.label,
+            type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+            trigger: statistics.trigger,
+            count: 1,
+            user: user?.id,
+            role: user?.role?.id
+          }),
+          ...totalUsage?.totalTokens ? [
+            updateStatistic({
+              name: "tokens",
+              label: statistics.label,
+              type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+              trigger: statistics.trigger,
+              count: totalUsage?.totalTokens,
+              user: user?.id,
+              role: user?.role?.id
+            })
+          ] : []
+        ]);
       }
       return text;
     }
+    return "";
   };
-  generateStream = async ({ express: express3, user, role, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
+  generateStream = async ({
+    express: express3,
+    user,
+    session,
+    message,
+    currentTools,
+    allExuluTools,
+    statistics,
+    toolConfigs,
+    providerapikey,
+    contexts,
+    exuluConfig,
+    filesContext: filesContext2,
+    instructions
+  }) => {
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
@@ -3363,60 +3856,110 @@ var ExuluAgent2 = class {
       throw new Error("Message is required for streaming.");
     }
     const model = this.model.create({
-      apiKey: providerApiKey
+      apiKey: providerapikey
     });
     let messages = [];
     const previousMessages = await getAgentMessages({
       session,
-      user,
+      user: user.id,
       limit: 50,
       page: 1
     });
-    const previousMessagesContent = previousMessages.map((message2) => JSON.parse(message2.content));
+    const previousMessagesContent = previousMessages.map(
+      (message2) => JSON.parse(message2.content)
+    );
     messages = await validateUIMessages({
       // append the new message to the previous messages:
       messages: [...previousMessagesContent, message]
     });
+    const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
+    let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
+    system += "\n\n" + genericContext;
+    console.log("[EXULU] tools for agent: " + this.name, currentTools?.map((x) => x.name + " (" + x.id + ")"));
+    console.log("[EXULU] system", system.slice(0, 100) + "...");
     const result = streamText({
       model,
       // Should be a LanguageModelV1
-      messages: convertToModelMessages(messages),
-      system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
+      messages: convertToModelMessages(messages, {
+        ignoreIncompleteToolCalls: true
+      }),
+      // prepareStep could be used here to set the model for the first step or change other params
+      system,
       maxRetries: 2,
-      tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
-      onError: (error) => console.error("[EXULU] chat stream error.", error),
-      stopWhen: [stepCountIs(5)]
+      providerOptions: {
+        openai: {
+          reasoningSummary: "auto"
+        }
+      },
+      tools: convertToolsArrayToObject(
+        currentTools,
+        allExuluTools,
+        toolConfigs,
+        providerapikey,
+        contexts,
+        user,
+        exuluConfig,
+        filesContext2
+      ),
+      onError: (error) => console.error("[EXULU] chat stream error.", error)
+      // stopWhen: [stepCountIs(1)],
     });
     result.consumeStream();
     result.pipeUIMessageStreamToResponse(express3.res, {
+      messageMetadata: ({ part }) => {
+        if (part.type === "finish") {
+          return {
+            totalTokens: part.totalUsage.totalTokens,
+            reasoningTokens: part.totalUsage.reasoningTokens,
+            inputTokens: part.totalUsage.inputTokens,
+            outputTokens: part.totalUsage.outputTokens,
+            cachedInputTokens: part.totalUsage.cachedInputTokens
+          };
+        }
+      },
       originalMessages: messages,
       sendReasoning: true,
+      sendSources: true,
+      onError: (error) => {
+        console.error("[EXULU] chat response error.", error);
+        return errorHandler(error);
+      },
       generateMessageId: createIdGenerator({
         prefix: "msg_",
         size: 16
       }),
-      onFinish: async ({ messages: messages2 }) => {
-        console.info(
-          "[EXULU] chat stream finished.",
-          messages2
-        );
+      onFinish: async ({ messages: messages2, isContinuation, isAborted, responseMessage }) => {
         if (session) {
           await saveChat({
             session,
-            user,
-            messages: messages2
+            user: user.id,
+            messages: messages2.filter((x) => !previousMessagesContent.find((y) => y.id === x.id))
           });
         }
+        const metadata = messages2[messages2.length - 1]?.metadata;
         if (statistics) {
-          await updateStatistic({
-            name: "count",
-            label: statistics.label,
-            type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-            trigger: statistics.trigger,
-            count: 1,
-            user,
-            role
-          });
+          await Promise.all([
+            updateStatistic({
+              name: "count",
+              label: statistics.label,
+              type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+              trigger: statistics.trigger,
+              count: 1,
+              user: user.id,
+              role: user?.role?.id
+            }),
+            ...metadata?.totalTokens ? [
+              updateStatistic({
+                name: "tokens",
+                label: statistics.label,
+                type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+                trigger: statistics.trigger,
+                count: metadata?.totalTokens,
+                user: user.id,
+                role: user?.role?.id
+              })
+            ] : []
+          ]);
         }
       }
     });
@@ -3425,7 +3968,12 @@ var ExuluAgent2 = class {
 };
 var getAgentMessages = async ({ session, user, limit, page }) => {
   const { db: db3 } = await postgresClient();
-  const messages = await db3.from("agent_messages").where({ session, user }).limit(limit).offset(page * limit);
+  console.log("[EXULU] getting agent messages for session: " + session + " and user: " + user + " and page: " + page);
+  const query = db3.from("agent_messages").where({ session, user }).limit(limit);
+  if (page > 0) {
+    query.offset((page - 1) * limit);
+  }
+  const messages = await query;
   return messages;
 };
 var saveChat = async ({ session, user, messages }) => {
@@ -3535,22 +4083,22 @@ var ExuluEval = class {
                 throw new Error("Prompt is required for running an agent.");
               }
               const { db: db4 } = await postgresClient();
-              const variableName = runner.agent.providerApiKey;
+              const variableName = runner.agent.providerapikey;
               const variable = await db4.from("variables").where({ name: variableName }).first();
               if (!variable) {
-                throw new Error(`Provider API key for variable "${runner.agent.providerApiKey}" not found.`);
+                throw new Error(`Provider API key for variable "${runner.agent.providerapikey}" not found.`);
               }
-              let providerApiKey = variable.value;
+              let providerapikey = variable.value;
               if (!variable.encrypted) {
-                throw new Error(`Provider API key for variable "${runner.agent.providerApiKey}" is not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys.`);
+                throw new Error(`Provider API key for variable "${runner.agent.providerapikey}" is not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys.`);
               }
               if (variable.encrypted) {
                 const bytes = CryptoJS2.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-                providerApiKey = bytes.toString(CryptoJS2.enc.Utf8);
+                providerapikey = bytes.toString(CryptoJS2.enc.Utf8);
               }
               const result = await runner.agent.generateSync({
                 prompt: data.prompt,
-                providerApiKey
+                providerapikey
               });
               data.result = result;
             }
@@ -3709,10 +4257,6 @@ var ExuluContext = class {
       label: statistics?.label || this.name,
       trigger: statistics?.trigger || "agent"
     }, user, role);
-    const exists = await db3.schema.hasTable(getChunksTableName(this.id));
-    if (!exists) {
-      await this.createChunksTable();
-    }
     await db3.from(getChunksTableName(this.id)).where({ source }).delete();
     await db3.from(getChunksTableName(this.id)).insert(chunks.map((chunk) => ({
       source,
@@ -3729,21 +4273,115 @@ var ExuluContext = class {
       job
     };
   };
-  embeddings = {
-    generate: {
-      one: async ({
-        item,
+  createItem = async (item, user, role, upsert) => {
+    const { db: db3 } = await postgresClient();
+    const mutation = db3.from(getTableName(
+      this.id
+    )).insert(
+      {
+        ...item,
+        tags: item.tags ? Array.isArray(item.tags) ? item.tags.join(",") : item.tags : void 0
+      }
+    ).returning("id");
+    if (upsert) {
+      mutation.onConflict().merge();
+    }
+    const results = await mutation;
+    if (!results[0]) {
+      throw new Error("Failed to create item.");
+    }
+    if (this.embedder && (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always")) {
+      const { job } = await this.embeddings.generate.one({
+        item: results[0],
         user,
         role,
-        trigger
-      }) => {
-        console.log("[EXULU] Generating embeddings for item", item.id);
-        if (!this.embedder) {
-          throw new Error("Embedder is not set for this context.");
-        }
-        if (!item.id) {
-          throw new Error("Item id is required for generating embeddings.");
-        }
+        trigger: "api"
+      });
+      return {
+        item: results[0],
+        job
+      };
+    }
+    return {
+      item: results[0],
+      job: void 0
+    };
+  };
+  updateItem = async (item, user, role) => {
+    const { db: db3 } = await postgresClient();
+    const record = await db3.from(
+      getTableName(this.id)
+    ).where(
+      { id: item.id }
+    ).first();
+    if (!record) {
+      throw new Error("Item not found.");
+    }
+    const mutation = db3.from(
+      getTableName(this.id)
+    ).where(
+      { id: record.id }
+    ).update(
+      {
+        ...item,
+        tags: item.tags ? Array.isArray(item.tags) ? item.tags.join(",") : item.tags : void 0
+      }
+    ).returning("id");
+    await mutation;
+    if (this.embedder && (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always")) {
+      const { job } = await this.embeddings.generate.one({
+        item: record,
+        // important we need to full record here with all fields
+        user,
+        role,
+        trigger: "api"
+      });
+      return {
+        item: record,
+        job
+      };
+    }
+    return {
+      item: record,
+      job: void 0
+    };
+  };
+  deleteItem = async (item, user, role) => {
+    if (!item.id) {
+      throw new Error("Item id is required for deleting item.");
+    }
+    const { db: db3 } = await postgresClient();
+    await db3.from(getTableName(this.id)).where({ id: item.id }).delete();
+    if (!this.embedder) {
+      return {
+        id: item.id,
+        job: void 0
+      };
+    }
+    const chunks = await db3.from(getChunksTableName(this.id)).where({ source: item.id }).select("id");
+    if (chunks.length > 0) {
+      await db3.from(getChunksTableName(this.id)).where({ source: item.id }).delete();
+    }
+    return {
+      id: item.id,
+      job: void 0
+    };
+  };
+  embeddings = {
+    generate: {
+      one: async ({
+        item,
+        user,
+        role,
+        trigger
+      }) => {
+        console.log("[EXULU] Generating embeddings for item", item.id);
+        if (!this.embedder) {
+          throw new Error("Embedder is not set for this context.");
+        }
+        if (!item.id) {
+          throw new Error("Item id is required for generating embeddings.");
+        }
         if (this.embedder.queue?.name) {
           console.log("[EXULU] embedder is in queue mode, scheduling job.");
           const job = await bullmqDecorator({
@@ -3796,301 +4434,6 @@ var ExuluContext = class {
       }
     }
   };
-  getItems = async ({
-    statistics,
-    limit,
-    sort,
-    order,
-    page,
-    name,
-    user,
-    role,
-    archived,
-    query,
-    method
-  }) => {
-    if (!query && limit > 500) {
-      throw new Error("Limit cannot be greater than 500.");
-    }
-    if (query && limit > 50) {
-      throw new Error("Limit cannot be greater than 50 when using a vector search query.");
-    }
-    if (page < 1) page = 1;
-    if (limit < 1) limit = 10;
-    let offset = (page - 1) * limit;
-    const mainTable = getTableName(this.id);
-    const { db: db3 } = await postgresClient();
-    const columns = await db3(mainTable).columnInfo();
-    const totalQuery = db3.count("* as count").from(mainTable).first();
-    const itemsQuery = db3.select(Object.keys(columns).map((column) => mainTable + "." + column)).from(mainTable).offset(offset).limit(limit);
-    if (sort) {
-      itemsQuery.orderBy(sort, order === "desc" ? "desc" : "asc");
-    }
-    if (typeof name === "string") {
-      itemsQuery.whereILike("name", `%${name}%`);
-      totalQuery.whereILike("name", `%${name}%`);
-    }
-    if (typeof archived === "boolean") {
-      itemsQuery.where("archived", archived);
-      totalQuery.where("archived", archived);
-    }
-    if (!query) {
-      const total = await totalQuery;
-      let items = await itemsQuery;
-      const last = Math.ceil(total.count / limit);
-      return {
-        pagination: {
-          totalCount: parseInt(total.count),
-          currentPage: page,
-          limit,
-          from: offset,
-          pageCount: last || 1,
-          to: offset + items.length,
-          lastPage: last || 1,
-          nextPage: page + 1 > last ? null : page + 1,
-          previousPage: page - 1 || null
-        },
-        filters: {
-          archived,
-          name,
-          query
-        },
-        context: {
-          name: this.name,
-          id: this.id,
-          embedder: this.embedder?.name || void 0
-        },
-        items
-      };
-    }
-    if (typeof query === "string" && this.embedder) {
-      if (!method) {
-        method = "cosineDistance";
-      }
-      itemsQuery.limit(limit * 5);
-      if (statistics) {
-        await updateStatistic({
-          name: "count",
-          label: statistics.label,
-          type: STATISTICS_TYPE_ENUM.CONTEXT_RETRIEVE,
-          trigger: statistics.trigger,
-          user,
-          role
-        });
-      }
-      if (this.queryRewriter) {
-        query = await this.queryRewriter(query);
-      }
-      const chunksTable = getChunksTableName(this.id);
-      itemsQuery.leftJoin(chunksTable, function() {
-        this.on(chunksTable + ".source", "=", mainTable + ".id");
-      });
-      itemsQuery.select(chunksTable + ".id as chunk_id");
-      itemsQuery.select(chunksTable + ".source");
-      itemsQuery.select(chunksTable + ".content");
-      itemsQuery.select(chunksTable + ".chunk_index");
-      itemsQuery.select(chunksTable + ".created_at as chunk_created_at");
-      itemsQuery.select(chunksTable + ".updated_at as chunk_updated_at");
-      const { chunks } = await this.embedder.generateFromQuery(query, {
-        label: this.name,
-        trigger: "agent"
-      }, user, role);
-      if (!chunks?.[0]?.vector) {
-        throw new Error("No vector generated for query.");
-      }
-      const vector = chunks[0].vector;
-      const vectorStr = `ARRAY[${vector.join(",")}]`;
-      const vectorExpr = `${vectorStr}::vector`;
-      const language = this.configuration.language || "english";
-      let items = [];
-      switch (method) {
-        case "tsvector":
-          itemsQuery.select(db3.raw(
-            `ts_rank(${chunksTable}.fts, websearch_to_tsquery(?, ?)) as fts_rank`,
-            [language, query]
-          )).whereRaw(
-            `${chunksTable}.fts @@ websearch_to_tsquery(?, ?)`,
-            [language, query]
-          ).orderByRaw(`fts_rank DESC`);
-          items = await itemsQuery;
-          break;
-        case "cosineDistance":
-        default:
-          itemsQuery.whereNotNull(`${chunksTable}.embedding`);
-          itemsQuery.select(
-            db3.raw(`1 - (${chunksTable}.embedding <=> ${vectorExpr}) AS cosine_distance`)
-          );
-          itemsQuery.orderByRaw(
-            `${chunksTable}.embedding <=> ${vectorExpr} ASC NULLS LAST`
-          );
-          items = await itemsQuery;
-          break;
-        case "hybridSearch":
-          const matchCount = Math.min(limit * 5, 30);
-          const fullTextWeight = 1;
-          const semanticWeight = 1;
-          const rrfK = 50;
-          const hybridSQL = `
-                    WITH full_text AS (
-                      SELECT
-                        c.id,
-                        c.source,
-                        row_number() OVER (
-                          ORDER BY ts_rank_cd(c.fts, websearch_to_tsquery(?, ?)) DESC
-                        ) AS rank_ix
-                      FROM ${chunksTable} c
-                      WHERE c.fts @@ websearch_to_tsquery(?, ?)
-                      ORDER BY rank_ix
-                      LIMIT LEAST(?, 30) * 2
-                    ),
-                    semantic AS (
-                      SELECT
-                        c.id,
-                        c.source,
-                        row_number() OVER (
-                          ORDER BY c.embedding <=> ${vectorExpr} ASC
-                        ) AS rank_ix
-                      FROM ${chunksTable} c
-                      WHERE c.embedding IS NOT NULL
-                      ORDER BY rank_ix
-                      LIMIT LEAST(?, 30) * 2
-                    )
-                    SELECT
-                      m.*,
-                      c.id AS chunk_id,
-                      c.source,
-                      c.content,
-                      c.chunk_index,
-                      c.created_at AS chunk_created_at,
-                      c.updated_at AS chunk_updated_at,
-                
-                      /* Per-signal scores for introspection */
-                      ts_rank(c.fts, websearch_to_tsquery(?, ?)) AS fts_rank,
-                      (1 - (c.embedding <=> ${vectorExpr})) AS cosine_distance,
-                
-                      /* Hybrid RRF score */
-                      (
-                        COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
-                        +
-                        COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
-                      )::float AS hybrid_score
-                
-                    FROM full_text ft
-                    FULL OUTER JOIN semantic se
-                      ON ft.id = se.id
-                    JOIN ${chunksTable} c
-                      ON COALESCE(ft.id, se.id) = c.id
-                    JOIN ${mainTable} m
-                      ON m.id = c.source
-                    ORDER BY hybrid_score DESC
-                    LIMIT LEAST(?, 30)
-                    OFFSET 0
-                  `;
-          const bindings = [
-            // full_text: websearch_to_tsquery(lang, query) in rank and where
-            language,
-            query,
-            language,
-            query,
-            matchCount,
-            // full_text limit
-            matchCount,
-            // semantic limit
-            // fts_rank (ts_rank) call
-            language,
-            query,
-            // RRF fusion parameters
-            rrfK,
-            fullTextWeight,
-            rrfK,
-            semanticWeight,
-            matchCount
-            // final limit
-          ];
-          items = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
-      }
-      console.log("items", items);
-      const seenSources = /* @__PURE__ */ new Map();
-      items = items.reduce((acc, item) => {
-        if (!seenSources.has(item.source)) {
-          seenSources.set(item.source, {
-            ...Object.fromEntries(
-              Object.keys(item).filter(
-                (key) => key !== "cosine_distance" && // kept per chunk below
-                key !== "fts_rank" && // kept per chunk below
-                key !== "hybrid_score" && // we will compute per item below
-                key !== "content" && key !== "source" && key !== "chunk_index" && key !== "chunk_id" && key !== "chunk_created_at" && key !== "chunk_updated_at"
-              ).map((key) => [key, item[key]])
-            ),
-            chunks: [{
-              content: item.content,
-              chunk_index: item.chunk_index,
-              ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
-              ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
-              ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
-            }]
-          });
-          acc.push(seenSources.get(item.source));
-        } else {
-          seenSources.get(item.source).chunks.push({
-            content: item.content,
-            chunk_index: item.chunk_index,
-            ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
-            ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
-            ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
-          });
-        }
-        return acc;
-      }, []);
-      console.log("items", items);
-      items.forEach((item) => {
-        if (!item.chunks?.length) {
-          return;
-        }
-        if (method === "tsvector") {
-          const ranks = item.chunks.map((c) => typeof c.fts_rank === "number" ? c.fts_rank : 0);
-          const total = ranks.reduce((a, b) => a + b, 0);
-          const average = ranks.length ? total / ranks.length : 0;
-          item.averageRelevance = average;
-          item.totalRelevance = total;
-        } else if (method === "cosineDistance") {
-          let methodProperty = "cosine_distance";
-          const average = item.chunks.reduce((acc, item2) => {
-            return acc + item2[methodProperty];
-          }, 0) / item.chunks.length;
-          const total = item.chunks.reduce((acc, item2) => {
-            return acc + item2[methodProperty];
-          }, 0);
-          item.averageRelevance = average;
-          item.totalRelevance = total;
-        } else if (method === "hybridSearch") {
-          console.log("item.chunks", item.chunks);
-          const scores = item.chunks.map((c) => typeof c.hybrid_score === "number" ? c.hybrid_score * 10 + 1 : 0);
-          const total = scores.reduce((a, b) => a + b, 0);
-          const average = scores.length ? total / scores.length : 0;
-          item.averageRelevance = average;
-          item.totalRelevance = total;
-        }
-      });
-      if (this.resultReranker && query) {
-        items = await this.resultReranker(items);
-      }
-      items = items.slice(0, limit);
-      return {
-        filters: {
-          archived,
-          name,
-          query
-        },
-        context: {
-          name: this.name,
-          id: this.id,
-          embedder: this.embedder.name
-        },
-        items
-      };
-    }
-  };
   createItemsTable = async () => {
     const { db: db3 } = await postgresClient();
     const tableName = getTableName(this.id);
@@ -4104,6 +4447,7 @@ var ExuluContext = class {
       table.boolean("archived").defaultTo(false);
       table.text("external_id");
       table.text("created_by");
+      table.text("ttl");
       table.text("rights_mode").defaultTo(this.configuration?.defaultRightsMode ?? "private");
       table.integer("textlength");
       table.text("source");
@@ -4121,7 +4465,7 @@ var ExuluContext = class {
     });
   };
   createChunksTable = async () => {
-    const { db: db3 } = await postgresClient();
+    const { db: db3 } = await refreshPostgresClient();
     const tableName = getChunksTableName(this.id);
     console.log("[EXULU] Creating table: " + tableName);
     await db3.schema.createTable(tableName, (table) => {
@@ -4146,8 +4490,8 @@ var ExuluContext = class {
             CREATE INDEX IF NOT EXISTS ${tableName}_embedding_hnsw_cosine
             ON ${tableName}
             USING hnsw (embedding vector_cosine_ops)
-            WHERE embedding IS NOT NULL
             WITH (m = 16, ef_construction = 64)
+            WHERE embedding IS NOT NULL
         `);
     return;
   };
@@ -4164,7 +4508,7 @@ var ExuluContext = class {
       description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
       execute: async ({ query, user, role }) => {
         const { db: db3 } = await postgresClient();
-        await vectorSearch({
+        const result = await vectorSearch({
           page: 1,
           limit: 10,
           query,
@@ -4177,6 +4521,9 @@ var ExuluContext = class {
           sort: void 0,
           trigger: "agent"
         });
+        return {
+          items: result.items
+        };
       }
     });
   };
@@ -4192,7 +4539,6 @@ var updateStatistic = async (statistic) => {
     type: statistic.type,
     createdAt: currentDate
   }).first();
-  console.log("!!! existing !!!", existing);
   if (!existing) {
     await db3.from("tracking").insert({
       name: statistic.name,
@@ -4214,6 +4560,53 @@ var updateStatistic = async (statistic) => {
     });
   }
 };
+var generateS3Key = (filename) => `${randomUUID()}-${filename}`;
+var getMimeType = (type) => {
+  switch (type) {
+    case ".png":
+      return "image/png";
+    case ".jpg":
+      return "image/jpg";
+    case ".jpeg":
+      return "image/jpeg";
+    case ".gif":
+      return "image/gif";
+    case ".webp":
+      return "image/webp";
+    case ".pdf":
+      return "application/pdf";
+    case ".docx":
+      return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+    case ".xlsx":
+      return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+    case ".xls":
+      return "application/vnd.ms-excel";
+    case ".csv":
+      return "text/csv";
+    case ".pptx":
+      return "application/vnd.openxmlformats-officedocument.presentationml.presentation";
+    case ".ppt":
+      return "application/vnd.ms-powerpoint";
+    case ".m4a":
+      return "audio/mp4";
+    case ".mp4":
+      return "audio/mp4";
+    case ".mpeg":
+      return "audio/mpeg";
+    case ".mp3":
+      return "audio/mp3";
+    case ".wav":
+      return "audio/wav";
+    case ".txt":
+      return "text/plain";
+    case ".md":
+      return "text/markdown";
+    case ".json":
+      return "application/json";
+    default:
+      return "";
+  }
+};
 
 // src/registry/index.ts
 import "express";
@@ -4221,42 +4614,6 @@ import "express";
 // src/registry/routes.ts
 import "express";
 
-// src/registry/rate-limiter.ts
-var rateLimiter = async (key, windowSeconds, limit, points) => {
-  try {
-    const { client: client2 } = await redisClient();
-    if (!client2) {
-      console.warn("[EXULU] Rate limiting disabled - Redis not available");
-      return {
-        status: true,
-        retryAfter: null
-      };
-    }
-    const redisKey = `exulu/${key}`;
-    const current = await client2.incrBy(redisKey, points);
-    if (current === points) {
-      await client2.expire(redisKey, windowSeconds);
-    }
-    if (current > limit) {
-      const ttl = await client2.ttl(redisKey);
-      return {
-        status: false,
-        retryAfter: ttl
-      };
-    }
-    return {
-      status: true,
-      retryAfter: null
-    };
-  } catch (error) {
-    console.error("[EXULU] Rate limiting error:", error);
-    return {
-      status: true,
-      retryAfter: null
-    };
-  }
-};
-
 // src/bullmq/queues.ts
 import { Queue as Queue3 } from "bullmq";
 import { BullMQOtel } from "bullmq-otel";
@@ -4300,22 +4657,22 @@ import { expressMiddleware } from "@as-integrations/express5";
 // src/registry/uppy.ts
 import "express";
 import {
-  S3Client,
+  S3Client as S3Client2,
   AbortMultipartUploadCommand,
   CompleteMultipartUploadCommand,
   CreateMultipartUploadCommand,
   GetObjectCommand,
   ListPartsCommand,
-  PutObjectCommand,
+  PutObjectCommand as PutObjectCommand2,
   UploadPartCommand,
-  ListObjectsV2Command
+  DeleteObjectCommand
 } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import {
   STSClient,
   GetFederationTokenCommand
 } from "@aws-sdk/client-sts";
-import { randomUUID } from "crypto";
+import { randomUUID as randomUUID2 } from "crypto";
 var createUppyRoutes = async (app, config) => {
   const policy = {
     Version: "2012-10-17",
@@ -4332,11 +4689,11 @@ var createUppyRoutes = async (app, config) => {
       }
     ]
   };
-  let s3Client;
+  let s3Client2;
   let stsClient;
   const expiresIn = 60 * 60 * 24 * 1;
   function getS3Client() {
-    s3Client ??= new S3Client({
+    s3Client2 ??= new S3Client2({
       region: config.fileUploads.s3region,
       ...config.fileUploads.s3endpoint && {
         forcePathStyle: true,
@@ -4347,7 +4704,7 @@ var createUppyRoutes = async (app, config) => {
         secretAccessKey: config.fileUploads.s3secret
       }
     });
-    return s3Client;
+    return s3Client2;
   }
   function getSTSClient() {
     stsClient ??= new STSClient({
@@ -4360,54 +4717,51 @@ var createUppyRoutes = async (app, config) => {
     });
     return stsClient;
   }
-  app.get("/s3/list", async (req, res, next) => {
-    req.accepts;
+  app.delete("/s3/delete", async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
+    const internalkey = req.headers["internal-key"] || null;
+    const { db: db3 } = await postgresClient();
     let authtoken = null;
-    if (typeof apikey !== "string") {
+    if (typeof apikey !== "string" && typeof internalkey !== "string") {
       authtoken = await getToken(req.headers.authorization ?? "");
     }
-    const { db: db3 } = await postgresClient();
     const authenticationResult = await authentication({
       authtoken,
       apikey,
+      internalkey,
       db: db3
     });
     if (!authenticationResult.user?.id) {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
-    const { prefix = "" } = req.query;
-    if (typeof prefix !== "string") {
-      res.status(400).json({ error: "Invalid prefix parameter. Must be a string." });
+    const { key } = req.query;
+    if (typeof key !== "string" || key.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `key` query parameter." });
+      return;
+    }
+    const userPrefix = key.split("/")[0];
+    console.log("userPrefix", userPrefix);
+    console.log("authenticationResult.user.id", authenticationResult.user.id);
+    if (!userPrefix) {
+      res.status(405).json({ error: 'Invalid key, does not contain a user prefix like "<user_id>/<key>.' });
       return;
     }
-    if (authenticationResult.user.type !== "api" && !prefix.includes(authenticationResult.user.id)) {
-      res.status(405).json({ error: "Not allowed to list files in this folder based on authenticated user." });
+    if (userPrefix !== authenticationResult.user.id.toString()) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
       return;
     }
-    try {
-      const command = new ListObjectsV2Command({
-        Bucket: config.fileUploads.s3Bucket,
-        Prefix: prefix,
-        MaxKeys: 1e3
-        // Adjust this value based on your needs
-      });
-      const data = await getS3Client().send(command);
-      const files = data.Contents?.map((item) => ({
-        key: item.Key,
-        size: item.Size,
-        lastModified: item.LastModified
-      })) || [];
-      res.setHeader("Access-Control-Allow-Origin", "*");
-      res.status(200).json({
-        files,
-        isTruncated: data.IsTruncated,
-        nextContinuationToken: data.NextContinuationToken
-      });
-    } catch (err) {
-      next(err);
+    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id.toString())) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
+      return;
     }
+    const client2 = getS3Client();
+    const command = new DeleteObjectCommand({
+      Bucket: config.fileUploads.s3Bucket,
+      Key: key
+    });
+    await client2.send(command);
+    res.json({ key });
   });
   app.get("/s3/download", async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
@@ -4432,7 +4786,16 @@ var createUppyRoutes = async (app, config) => {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
-    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id)) {
+    const userPrefix = key.split("/")[0];
+    if (!userPrefix) {
+      res.status(405).json({ error: 'Invalid key, does not contain a user prefix like "<user_id>/<key>.' });
+      return;
+    }
+    if (userPrefix !== authenticationResult.user.id.toString()) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
+      return;
+    }
+    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id.toString())) {
       res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
       return;
     }
@@ -4482,7 +4845,7 @@ var createUppyRoutes = async (app, config) => {
       contentType: params.type
     };
   };
-  const generateS3Key = (filename) => `${randomUUID()}-${filename}`;
+  const generateS3Key2 = (filename) => `${randomUUID2()}-${filename}`;
   const signOnServer = async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
     const { db: db3 } = await postgresClient();
@@ -4501,16 +4864,11 @@ var createUppyRoutes = async (app, config) => {
     }
     const { filename, contentType } = extractFileParameters(req);
     validateFileParameters(filename, contentType);
-    const key = generateS3Key(filename);
-    let folder = "";
-    if (authenticationResult.user.type === "api") {
-      folder = `api/`;
-    } else {
-      folder = `${authenticationResult.user.id}/`;
-    }
+    const key = generateS3Key2(filename);
+    let folder = `${authenticationResult.user.id}/`;
     getSignedUrl(
       getS3Client(),
-      new PutObjectCommand({
+      new PutObjectCommand2({
         Bucket: config.fileUploads.s3Bucket,
         Key: folder + key,
         ContentType: contentType
@@ -4556,7 +4914,7 @@ var createUppyRoutes = async (app, config) => {
     if (typeof type !== "string") {
       return res.status(400).json({ error: "s3: content type must be a string" });
     }
-    const key = `${randomUUID()}-${filename}`;
+    const key = `${randomUUID2()}-${filename}`;
     let folder = "";
     if (authenticationResult.user.type === "api") {
       folder = `api/`;
@@ -4697,6 +5055,13 @@ var createUppyRoutes = async (app, config) => {
 import { InMemoryLRUCache } from "@apollo/utils.keyvaluecache";
 import bodyParser from "body-parser";
 import CryptoJS3 from "crypto-js";
+import OpenAI from "openai";
+import fs from "fs";
+import { randomUUID as randomUUID3 } from "crypto";
+import "@opentelemetry/api";
+import { jsonSchema } from "ai";
+import proxy from "express-http-proxy";
+import Anthropic from "@anthropic-ai/sdk";
 
 // src/registry/utils/claude-messages.ts
 var CLAUDE_MESSAGES = {
@@ -4721,10 +5086,6 @@ var CLAUDE_MESSAGES = {
 };
 
 // src/registry/routes.ts
-import OpenAI from "openai";
-import fs2 from "fs";
-import { randomUUID as randomUUID2 } from "crypto";
-import "@opentelemetry/api";
 var REQUEST_SIZE_LIMIT = "50mb";
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -4774,7 +5135,7 @@ var createRecurringJobs = async () => {
   );
   return queue;
 };
-var createExpressRoutes = async (app, logger, agents, tools, contexts, config, tracer) => {
+var createExpressRoutes = async (app, logger, agents, tools, contexts, config, tracer, filesContext2) => {
   var corsOptions = {
     origin: "*",
     exposedHeaders: "*",
@@ -4799,7 +5160,7 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
   if (redisServer.host?.length && redisServer.port?.length) {
     await createRecurringJobs();
   } else {
-    console.log("[o_o]", "[EXULU] no redis server configured, not setting up recurring jobs.", "[o_o]");
+    console.log("[EXULU] no redis server configured, not setting up recurring jobs.");
   }
   const schema = createSDL([
     usersSchema2(),
@@ -4814,7 +5175,7 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
     workflowTemplatesSchema2(),
     statisticsSchema2(),
     rbacSchema2()
-  ], contexts, agents, tools);
+  ], contexts ?? [], agents, tools);
   const server = new ApolloServer({
     cache: new InMemoryLRUCache(),
     schema,
@@ -4872,7 +5233,7 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
       });
       return;
     }
-    let providerApiKey = variable.value;
+    let providerapikey = variable.value;
     if (!variable.encrypted) {
       res.status(400).json({
         message: "Provider API key variable not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys."
@@ -4881,10 +5242,10 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
     }
     if (variable.encrypted) {
       const bytes = CryptoJS3.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-      providerApiKey = bytes.toString(CryptoJS3.enc.Utf8);
+      providerapikey = bytes.toString(CryptoJS3.enc.Utf8);
     }
     const openai = new OpenAI({
-      apiKey: providerApiKey
+      apiKey: providerapikey
     });
     let style_reference = "";
     if (style === "origami") {
@@ -4931,11 +5292,11 @@ Mood: friendly and intelligent.
       return;
     }
     const image_bytes = Buffer.from(image_base64, "base64");
-    const uuid = randomUUID2();
-    if (!fs2.existsSync("public")) {
-      fs2.mkdirSync("public");
+    const uuid = randomUUID3();
+    if (!fs.existsSync("public")) {
+      fs.mkdirSync("public");
     }
-    fs2.writeFileSync(`public/${uuid}.png`, image_bytes);
+    fs.writeFileSync(`public/${uuid}.png`, image_bytes);
     res.status(200).json({
       message: "Image generated successfully.",
       image: `${process.env.BACKEND}/${uuid}.png`
@@ -4957,6 +5318,12 @@ Mood: friendly and intelligent.
     const slug = agent.slug;
     if (!slug) return;
     app.post(slug + "/:instance", async (req, res) => {
+      const headers = {
+        stream: req.headers["stream"] === "true" || false,
+        user: req.headers["user"] || null,
+        session: req.headers["session"] || null
+      };
+      await checkAgentRateLimit(agent);
       const instance = req.params.instance;
       if (!instance) {
         res.status(400).json({
@@ -4965,38 +5332,7 @@ Mood: friendly and intelligent.
         return;
       }
       const { db: db3 } = await postgresClient();
-      const agentInstance = await db3.from("agents").where({
-        id: instance
-      }).first();
-      const agentRbac = await RBACResolver(db3, "agent", agentInstance.id, agentInstance.rights_mode || "private");
-      agentInstance.RBAC = agentRbac;
-      if (!agentInstance) {
-        res.status(400).json({
-          message: "Agent instance not found."
-        });
-        return;
-      }
-      if (agent.rateLimit) {
-        console.log("[EXULU] rate limiting agent.", agent.rateLimit);
-        const limit = await rateLimiter(
-          agent.rateLimit.name || agent.id,
-          agent.rateLimit.rate_limit.time,
-          agent.rateLimit.rate_limit.limit,
-          1
-        );
-        if (!limit.status) {
-          res.status(429).json({
-            message: "Rate limit exceeded.",
-            retryAfter: limit.retryAfter
-          });
-          return;
-        }
-      }
-      const headers = {
-        stream: req.headers["stream"] === "true" || false,
-        user: req.headers["user"] || null,
-        session: req.headers["session"] || null
-      };
+      const agentInstance = await loadAgent(instance);
       const requestValidationResult = requestValidators.agents(req);
       if (requestValidationResult.error) {
         res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
@@ -5008,85 +5344,24 @@ Mood: friendly and intelligent.
         return;
       }
       const user = authenticationResult.user;
-      const agentIsPublic = agentInstance.rights_mode === "public";
-      const agentByUsers = agentInstance.rights_mode === "users";
-      const agentByRoles = agentInstance.rights_mode === "roles";
-      const isAgentCreator = agentInstance.created_by === user.id;
-      const isAdmin = user.super_admin;
-      const isApi = user.type === "api";
-      let hasAccessToAgent = "none";
-      if (agentIsPublic || isAgentCreator || isAdmin || isApi) {
-        hasAccessToAgent = "write";
-      }
-      if (agentByUsers) {
-        hasAccessToAgent = agentInstance.RBAC?.users?.find((x) => x.id === user.id)?.rights || "none";
-        if (!hasAccessToAgent || hasAccessToAgent === "none" || hasAccessToAgent === "read") {
-          res.status(410).json({
-            message: `Your current user ${user.id} does not have access to this agent.`
-          });
-          return;
-        }
-      }
-      if (agentByRoles) {
-        hasAccessToAgent = agentInstance.RBAC?.roles?.find((x) => x.id === user.role?.id)?.rights || "none";
-        if (!hasAccessToAgent || hasAccessToAgent === "none" || hasAccessToAgent === "read") {
-          res.status(410).json({
-            message: `Your current role ${user.role?.name} does not have access to this agent.`
-          });
-          return;
-        }
-      }
-      let hasAccessToSession = "none";
-      ;
-      if (headers.session) {
-        const session = await db3.from("agents").where({
-          id: instance
-        }).first();
-        const sessionIsPublic = agentInstance.rights_mode === "public";
-        const sessionByUsers = agentInstance.rights_mode === "users";
-        const sessionByRoles = agentInstance.rights_mode === "roles";
-        const isSessionCreator = agentInstance.created_by === user.id;
-        const isAdmin2 = user.super_admin;
-        const isApi2 = user.type === "api";
-        if (sessionIsPublic || isSessionCreator || isAdmin2 || isApi2) {
-          hasAccessToSession = "write";
-        }
-        if (sessionByUsers) {
-          hasAccessToSession = session.RBAC?.users?.find((x) => x.id === user.id)?.rights || "none";
-          if (!hasAccessToSession || hasAccessToSession === "none" || hasAccessToSession === "read") {
-            res.status(410).json({
-              message: `Your current user ${user.id} does not have access to this session.`
-            });
-            return;
-          }
-        }
-        if (sessionByRoles) {
-          hasAccessToSession = session.RBAC?.roles?.find((x) => x.id === user.role?.id)?.rights || "none";
-          if (!hasAccessToSession || hasAccessToSession === "none" || hasAccessToSession === "read") {
-            res.status(410).json({
-              message: `Your current role ${user.role?.name} does not have access to this session.`
-            });
-            return;
-          }
-        }
-      }
-      if (!hasAccessToAgent || hasAccessToAgent === "none") {
-        res.status(410).json({
+      const hasAccessToAgent = await checkRecordAccess(agentInstance, "read", user);
+      if (!hasAccessToAgent) {
+        res.status(401).json({
           message: "You don't have access to this agent."
         });
         return;
       }
-      if (!hasAccessToSession || hasAccessToSession === "none") {
-        res.status(410).json({
-          message: "You don't have access to this session."
-        });
-        return;
-      }
-      if (headers.session && !hasAccessToSession) {
-        res.status(410).json({
-          message: "You don't have access to this session."
-        });
-        return;
+      if (headers.session) {
+        const session = await db3.from("agent_sessions").where({
+          id: headers.session
+        }).first();
+        let hasAccessToSession = await checkRecordAccess(session, "write", user);
+        if (!hasAccessToSession) {
+          res.status(401).json({
+            message: "You don't have access to this session."
+          });
+          return;
+        }
       }
       if (user.type !== "api" && !user.super_admin && req.body.resourceId !== user.id) {
         res.status(400).json({
@@ -5094,16 +5369,11 @@ Mood: friendly and intelligent.
         });
         return;
       }
-      console.log("[EXULU] agent tools", agentInstance.tools);
-      let enabledTools = agentInstance.tools ? agentInstance.tools.map(
-        ({ config: config2, toolId }) => tools.find(({ id }) => id === toolId)
-      ).filter(Boolean) : [];
-      console.log("[EXULU] available tools", enabledTools?.length);
+      console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
       const disabledTools = req.body.disabledTools ? req.body.disabledTools : [];
-      console.log("[EXULU] disabled tools", disabledTools?.length);
-      enabledTools = enabledTools.filter((tool2) => !disabledTools.includes(tool2.id));
-      console.log("[EXULU] enabled tools", enabledTools?.length);
-      const variableName = agentInstance.providerApiKey;
+      let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
+      console.log("[EXULU] enabled tools", enabledTools?.map((x) => x.name + " (" + x.id + ")"));
+      const variableName = agentInstance.providerapikey;
       const variable = await db3.from("variables").where({ name: variableName }).first();
       if (!variable) {
         res.status(400).json({
@@ -5111,7 +5381,7 @@ Mood: friendly and intelligent.
         });
         return;
       }
-      let providerApiKey = variable.value;
+      let providerapikey = variable.value;
       if (!variable.encrypted) {
         res.status(400).json({
           message: "Provider API key variable not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys."
@@ -5120,7 +5390,7 @@ Mood: friendly and intelligent.
       }
       if (variable.encrypted) {
         const bytes = CryptoJS3.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-        providerApiKey = bytes.toString(CryptoJS3.enc.Utf8);
+        providerapikey = bytes.toString(CryptoJS3.enc.Utf8);
       }
       if (!!headers.stream) {
         await agent.generateStream({
@@ -5128,13 +5398,17 @@ Mood: friendly and intelligent.
             res,
             req
           },
-          user: user?.id,
-          role: user?.role?.id,
+          contexts,
+          user,
+          instructions: agentInstance.instructions,
           session: headers.session,
           message: req.body.message,
-          tools: enabledTools,
-          providerApiKey,
+          currentTools: enabledTools,
+          allExuluTools: tools,
+          providerapikey,
           toolConfigs: agentInstance.tools,
+          exuluConfig: config,
+          filesContext: filesContext2,
           statistics: {
             label: agent.name,
             trigger: "agent"
@@ -5143,13 +5417,17 @@ Mood: friendly and intelligent.
         return;
       } else {
         const response = await agent.generateSync({
-          user: user?.id,
+          user,
+          instructions: agentInstance.instructions,
           session: headers.session,
-          role: user?.role?.id,
           message: req.body.message,
-          tools: enabledTools,
-          providerApiKey,
+          contexts,
+          currentTools: enabledTools,
+          allExuluTools: tools,
+          providerapikey,
+          exuluConfig: config,
           toolConfigs: agentInstance.tools,
+          filesContext: filesContext2,
           statistics: {
             label: agent.name,
             trigger: "agent"
@@ -5165,10 +5443,92 @@ Mood: friendly and intelligent.
   } else {
     console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in ExuluApp instance.");
   }
-  const TARGET_API = "https://api.anthropic.com";
+  app.use("/xxx/anthropic/:id", express.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }), proxy(
+    (req, res, next) => {
+      return "https://api.anthropic.com";
+    },
+    {
+      limit: "50mb",
+      memoizeHost: false,
+      preserveHostHdr: true,
+      secure: false,
+      reqAsBuffer: true,
+      proxyReqBodyDecorator: function(bodyContent, srcReq) {
+        return bodyContent;
+      },
+      userResDecorator: function(proxyRes, proxyResData, userReq, userRes) {
+        console.log("[EXULU] Proxy response!", proxyResData);
+        proxyResData = proxyResData.toString();
+        console.log("[EXULU] Proxy response string!", proxyResData);
+        return proxyResData;
+      },
+      proxyReqPathResolver: (req) => {
+        const prefix = `/gateway/anthropic/${req.params.id}`;
+        let path2 = req.url.startsWith(prefix) ? req.url.slice(prefix.length) : req.url;
+        if (!path2.startsWith("/")) path2 = "/" + path2;
+        console.log("[EXULU] Provider path:", path2);
+        return path2;
+      },
+      proxyReqOptDecorator: function(proxyReqOpts, srcReq) {
+        return new Promise(async (resolve, reject) => {
+          try {
+            const authenticationResult = await requestValidators.authenticate(srcReq);
+            if (!authenticationResult.user?.id) {
+              console.log("[EXULU] failed authentication result", authenticationResult);
+              reject(authenticationResult.message);
+              return;
+            }
+            console.log("[EXULU] Authenticated call", authenticationResult.user?.email);
+            const { db: db3 } = await postgresClient();
+            let query = db3("agents");
+            query.select("*");
+            query = applyAccessControl(agentsSchema2(), authenticationResult.user, query);
+            query.where({ id: srcReq.params.id });
+            const agent = await query.first();
+            if (!agent) {
+              reject(new Error("Agent with id " + srcReq.params.id + " not found."));
+              return;
+            }
+            console.log("[EXULU] Agent loaded", agent.name);
+            const backend = agents.find((x) => x.id === agent.backend);
+            if (!process.env.NEXTAUTH_SECRET) {
+              reject(new Error("Missing NEXTAUTH_SECRET"));
+              return;
+            }
+            if (!agent.providerapikey) {
+              reject(new Error("API Key not set for agent"));
+              return;
+            }
+            const variableName = agent.providerapikey;
+            const variable = await db3.from("variables").where({ name: variableName }).first();
+            console.log("[EXULU] Variable loaded", variable);
+            let providerapikey = variable.value;
+            if (!variable.encrypted) {
+              reject(new Error("API Key not encrypted for agent"));
+              return;
+            }
+            if (variable.encrypted) {
+              const bytes = CryptoJS3.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+              providerapikey = bytes.toString(CryptoJS3.enc.Utf8);
+            }
+            console.log("[EXULU] Provider API key", providerapikey);
+            proxyReqOpts.headers["x-api-key"] = providerapikey;
+            proxyReqOpts.rejectUnauthorized = false;
+            delete proxyReqOpts.headers["provider"];
+            const url = new URL("https://api.anthropic.com");
+            proxyReqOpts.headers["host"] = url.host;
+            proxyReqOpts.headers["anthropic-version"] = "2023-06-01";
+            console.log("[EXULU] Proxy request headers", proxyReqOpts.headers);
+            resolve(proxyReqOpts);
+          } catch (error) {
+            console.error("[EXULU] Proxy error", error);
+            reject(error);
+          }
+        });
+      }
+    }
+  ));
   app.use("/gateway/anthropic/:id", express.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }), async (req, res) => {
-    const path3 = req.url;
-    const url = `${TARGET_API}${path3}`;
     try {
       if (!req.body.tools) {
         req.body.tools = [];
@@ -5200,13 +5560,13 @@ Mood: friendly and intelligent.
         res.end(Buffer.from(arrayBuffer));
         return;
       }
-      if (!agent.providerApiKey) {
+      if (!agent.providerapikey) {
         const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.not_enabled);
         res.setHeader("Content-Type", "application/json");
         res.end(Buffer.from(arrayBuffer));
         return;
       }
-      const variableName = agent.providerApiKey;
+      const variableName = agent.providerapikey;
       const variable = await db3.from("variables").where({ name: variableName }).first();
       if (!variable) {
         const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.anthropic_token_variable_not_found);
@@ -5232,45 +5592,21 @@ Mood: friendly and intelligent.
       };
       if (req.headers["accept"]) headers["accept"] = req.headers["accept"];
       if (req.headers["user-agent"]) headers["user-agent"] = req.headers["user-agent"];
-      const response = await fetch(url, {
-        method: req.method,
-        headers,
-        body: req.method !== "GET" ? JSON.stringify(req.body) : void 0
-      });
-      await updateStatistic({
-        name: "count",
-        label: "Claude Code",
-        type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-        trigger: "claude-code",
-        count: 1,
-        user: authenticationResult.user?.id,
-        role: authenticationResult.user.role?.id
+      const client2 = new Anthropic({
+        apiKey: anthropicApiKey
       });
-      response.headers.forEach((value, key) => {
-        res.setHeader(key, value);
-      });
-      res.status(response.status);
-      const isStreaming = response.headers.get("content-type")?.includes("text/event-stream");
-      if (isStreaming && !response?.body) {
-        const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.missing_body);
-        res.setHeader("Content-Type", "application/json");
-        res.end(Buffer.from(arrayBuffer));
-        return;
-      }
-      if (isStreaming) {
-        const reader = response.body.getReader();
-        const decoder = new TextDecoder();
-        while (true) {
-          const { done, value } = await reader.read();
-          if (done) break;
-          const chunk = decoder.decode(value, { stream: true });
-          res.write(chunk);
+      for await (const event of client2.messages.stream(req.body)) {
+        console.log("[EXULU] Event", event);
+        if (event.message?.usage) {
         }
-        res.end();
-        return;
+        const msg = `event: ${event.type}
+data: ${JSON.stringify(event)}
+
+`;
+        res.write(msg);
       }
-      const data = await response.arrayBuffer();
-      res.end(Buffer.from(data));
+      res.write("event: done\ndata: [DONE]\n\n");
+      res.end();
     } catch (error) {
       console.error("[PROXY] Manual proxy error:", error);
       if (!res.headersSent) {
@@ -5303,33 +5639,10 @@ var createCustomAnthropicStreamingMessage = (message) => {
 // src/registry/workers.ts
 import IORedis from "ioredis";
 import { Worker } from "bullmq";
-
-// src/registry/utils.ts
-var bullmq = {
-  validate: (id, data) => {
-    if (!data) {
-      throw new Error(`Missing job data for job ${id}.`);
-    }
-    if (!data.type) {
-      throw new Error(`Missing property "type" in data for job ${id}.`);
-    }
-    if (!data.inputs) {
-      throw new Error(`Missing property "inputs" in data for job ${id}.`);
-    }
-    if (data.type !== "embedder" && data.type !== "workflow") {
-      throw new Error(`Property "type" in data for job ${id} must be of value "embedder" or "workflow".`);
-    }
-    if (!data.workflow && !data.embedder) {
-      throw new Error(`Either a workflow or embedder must be set for job ${id}.`);
-    }
-  }
-};
-
-// src/registry/workers.ts
-import * as fs3 from "fs";
-import path2 from "path";
+import * as fs2 from "fs";
+import path from "path";
 import "@opentelemetry/api";
-var defaultLogsDir = path2.join(process.cwd(), "logs");
+var defaultLogsDir = path.join(process.cwd(), "logs");
 var redisConnection;
 var createWorkers = async (queues2, logger, contexts, _logsDir, tracer) => {
   if (!redisServer.host || !redisServer.port) {
@@ -5370,8 +5683,6 @@ var createWorkers = async (queues2, logger, contexts, _logsDir, tracer) => {
             }, data.role, bullmqJob.id);
             return result;
           }
-          if (bullmqJob.data.type === "workflow") {
-          }
         } catch (error) {
           await db3.from("jobs").where({ redis: bullmqJob.id }).update({
             status: "failed",
@@ -5406,16 +5717,16 @@ var createLogsCleanerWorker = (logsDir) => {
     global_queues.logs_cleaner,
     async (job) => {
       console.log(`[EXULU] recurring job ${job.id}.`);
-      const folder = fs3.readdirSync(logsDir);
+      const folder = fs2.readdirSync(logsDir);
       const files = folder.filter((file) => file.endsWith(".log"));
       const now = /* @__PURE__ */ new Date();
       const daysToKeep = job.data.ttld;
       const dateToKeep = new Date(now.getTime() - daysToKeep * 24 * 60 * 60 * 1e3);
       files.forEach((file) => {
-        const filePath = path2.join(logsDir, file);
-        const fileStats = fs3.statSync(filePath);
+        const filePath = path.join(logsDir, file);
+        const fileStats = fs2.statSync(filePath);
         if (fileStats.mtime < dateToKeep) {
-          fs3.unlinkSync(filePath);
+          fs2.unlinkSync(filePath);
         }
       });
     },
@@ -5435,7 +5746,7 @@ var createLogsCleanerWorker = (logsDir) => {
 
 // src/mcp/index.ts
 import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { randomUUID as randomUUID3 } from "crypto";
+import { randomUUID as randomUUID4 } from "crypto";
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
 import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
 import { z as z2 } from "zod";
@@ -5517,7 +5828,6 @@ ${code}`
     if (!this.server) {
       throw new Error("MCP server not initialized.");
     }
-    console.log("[EXULU] Wiring up MCP server routes to express app.");
     this.express.post("/mcp", async (req, res) => {
       if (!this.server) {
         throw new Error("MCP server not initialized.");
@@ -5528,7 +5838,7 @@ ${code}`
         transport = this.transports[sessionId];
       } else if (!sessionId && isInitializeRequest(req.body)) {
         transport = new StreamableHTTPServerTransport({
-          sessionIdGenerator: () => randomUUID3(),
+          sessionIdGenerator: () => randomUUID4(),
           onsessioninitialized: (sessionId2) => {
             this.transports[sessionId2] = transport;
           }
@@ -5570,35 +5880,49 @@ ${code}`
 // src/registry/index.ts
 import express2 from "express";
 
-// src/templates/agents/claude-code.ts
-var claudeCodeAgent = new ExuluAgent2({
+// src/templates/agents/claude-sonnet-4.ts
+import { createAnthropic } from "@ai-sdk/anthropic";
+var claudeSonnet4Agent = new ExuluAgent2({
   id: `claude_code_agent`,
   name: `Claude Code Agent`,
   description: `Claude Code agent, enabling the creation of multiple Claude Code Agent instances with different configurations (rate limits, functions, etc).`,
-  type: "custom"
+  type: "agent",
+  config: {
+    name: `Default Claude Code agent`,
+    instructions: "You are a coding assistant.",
+    model: {
+      create: ({ apiKey }) => {
+        const anthropic = createAnthropic({
+          apiKey
+        });
+        return anthropic.languageModel("claude-sonnet-4-20250514");
+      }
+    }
+  }
 });
 
 // src/templates/agents/claude-opus-4.ts
-import { createAnthropic } from "@ai-sdk/anthropic";
-var defaultAgent = new ExuluAgent2({
+import { createAnthropic as createAnthropic2 } from "@ai-sdk/anthropic";
+var claudeOpus4Agent = new ExuluAgent2({
   id: `default_claude_4_opus_agent`,
-  name: `Default Claude 4 Opus Agent`,
-  description: `Basic agent without any defined tools, that can support MCP's.`,
+  name: `Default Claude 4 Opus anthropic provider`,
+  description: `Basic agent claude 4 opus agent you can use to chat with.`,
   type: "agent",
   capabilities: {
     text: true,
     images: [".png", ".jpg", ".jpeg", ".webp"],
-    files: [".pdf", ".docx", ".xlsx", ".xls", ".csv", ".pptx", ".ppt"],
+    files: [".pdf", ".docx", ".xlsx", ".xls", ".csv", ".pptx", ".ppt", ".json"],
     audio: [],
     video: []
   },
   evals: [],
+  maxContextLength: 2e5,
   config: {
     name: `Default agent`,
     instructions: "You are a helpful assistant.",
     model: {
       create: ({ apiKey }) => {
-        const anthropic = createAnthropic({
+        const anthropic = createAnthropic2({
           apiKey
         });
         return anthropic.languageModel("claude-4-opus-20250514");
@@ -5614,8 +5938,79 @@ var defaultAgent = new ExuluAgent2({
   }
 });
 
+// src/templates/agents/gpt-5.ts
+import { createOpenAI } from "@ai-sdk/openai";
+var gpt5MiniAgent = new ExuluAgent2({
+  id: `default_gpt_5_mini_agent`,
+  name: `Default GPT 5 Mini OpenAI provider`,
+  description: `Basic agent gpt 5 mini agent you can use to chat with.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [".png", ".jpg", ".jpeg", ".webp"],
+    files: [".pdf", ".docx", ".xlsx", ".xls", ".csv", ".pptx", ".ppt", ".json"],
+    audio: [],
+    video: []
+  },
+  evals: [],
+  maxContextLength: 128e3,
+  config: {
+    name: `Default agent`,
+    instructions: "You are a helpful assistant.",
+    model: {
+      create: ({ apiKey }) => {
+        const openai = createOpenAI({
+          apiKey
+        });
+        return openai.languageModel("gpt-5-mini");
+      }
+      // todo add a field of type string that adds a dropdown list from which the user can select the model
+      // todo for each model, check which provider is used, and require the admin to add one or multiple
+      //   API keys for the provider (which we can then auto-rotate).
+      // todo also add custom fields for rate limiting, so the admin can set custom rate limits for the agent
+      //   and allow him/her to decide if the rate limit is per user or per agent.
+      // todo finally allow switching on or off immutable audit logs on the agent. Which then enables OTEL
+      //   and stores the logs into the pre-defined storage.
+    }
+  }
+});
+var gpt5agent = new ExuluAgent2({
+  id: `default_gpt_5_agent`,
+  name: `Default GPT 5 OpenAI provider`,
+  description: `Basic agent gpt 5 agent you can use to chat with.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [".png", ".jpg", ".jpeg", ".webp"],
+    files: [".pdf", ".docx", ".xlsx", ".xls", ".csv", ".pptx", ".ppt", ".json"],
+    audio: [],
+    video: []
+  },
+  evals: [],
+  maxContextLength: 128e3,
+  config: {
+    name: `Default agent`,
+    instructions: "You are a helpful assistant.",
+    model: {
+      create: ({ apiKey }) => {
+        const openai = createOpenAI({
+          apiKey
+        });
+        return openai.languageModel("gpt-5");
+      }
+      // todo add a field of type string that adds a dropdown list from which the user can select the model
+      // todo for each model, check which provider is used, and require the admin to add one or multiple
+      //   API keys for the provider (which we can then auto-rotate).
+      // todo also add custom fields for rate limiting, so the admin can set custom rate limits for the agent
+      //   and allow him/her to decide if the rate limit is per user or per agent.
+      // todo finally allow switching on or off immutable audit logs on the agent. Which then enables OTEL
+      //   and stores the logs into the pre-defined storage.
+    }
+  }
+});
+
 // src/registry/index.ts
-import { trace as trace2 } from "@opentelemetry/api";
+import { trace } from "@opentelemetry/api";
 
 // src/registry/logger.ts
 import { OpenTelemetryTransportV3 } from "@opentelemetry/winston-transport";
@@ -5667,21 +6062,21 @@ var codeStandardsContext = new ExuluContext({
   active: true
 });
 
-// src/templates/contexts/projects.ts
-var projectsContext = new ExuluContext({
-  id: "projects",
-  name: "Projects",
-  description: "Default context that stores files and data related to projects in Exulu.",
+// src/templates/contexts/outputs.ts
+var outputsContext = new ExuluContext({
+  id: "outputs_default_context",
+  name: "Outputs",
+  description: "Outputs from agent sessions that you have saved for re-used later.",
   configuration: {
-    defaultRightsMode: "projects"
+    defaultRightsMode: "private",
+    calculateVectors: "manual"
   },
-  fields: [{
-    name: "Type",
-    type: "text"
-  }, {
-    name: "Summary",
-    type: "longText"
-  }],
+  fields: [
+    {
+      name: "content",
+      type: "longText"
+    }
+  ],
   active: true
 });
 
@@ -5699,14 +6094,6 @@ var filesContext = new ExuluContext({
       name: "type",
       type: "text"
     },
-    {
-      name: "s3bucket",
-      type: "text"
-    },
-    {
-      name: "s3region",
-      type: "text"
-    },
     {
       name: "url",
       type: "text"
@@ -5716,10 +6103,6 @@ var filesContext = new ExuluContext({
       // ID of the file in S3 storage
       type: "text"
     },
-    {
-      name: "s3endpoint",
-      type: "text"
-    },
     {
       name: "content",
       type: "longText"
@@ -5749,22 +6132,25 @@ var ExuluApp = class {
   create = async ({ contexts, agents, config, tools }) => {
     this._contexts = {
       ...contexts,
-      projectsContext,
       codeStandardsContext,
-      filesContext
+      filesContext,
+      outputsContext
     };
     this._agents = [
-      claudeCodeAgent,
-      defaultAgent,
+      claudeSonnet4Agent,
+      claudeOpus4Agent,
+      gpt5MiniAgent,
+      gpt5agent,
       ...agents ?? []
     ];
     this._config = config;
     this._tools = [
       ...tools ?? [],
       // Add contexts as tools
-      ...Object.values(contexts || {}).map((context) => context.tool()),
-      // Add agents as tools
-      ...(agents || []).map((agent) => agent.tool())
+      ...Object.values(contexts || {}).map((context) => context.tool())
+      // Because agents are stored in the database,  we add those as tools
+      // at request time, not during ExuluApp initialization. We add them
+      // in the grahql tools resolver.
     ];
     const checks = [
       ...Object.keys(this._contexts || {}).map((x) => ({
@@ -5857,7 +6243,7 @@ var ExuluApp = class {
       create: async () => {
         let tracer;
         if (this._config?.telemetry?.enabled) {
-          tracer = trace2.getTracer("exulu", "1.0.0");
+          tracer = trace.getTracer("exulu", "1.0.0");
         }
         const logger = logger_default({
           enableOtel: this._config?.workers?.telemetry?.enabled ?? false
@@ -5881,7 +6267,7 @@ var ExuluApp = class {
         const app = this._expressApp;
         let tracer;
         if (this._config?.telemetry?.enabled) {
-          tracer = trace2.getTracer("exulu", "1.0.0");
+          tracer = trace.getTracer("exulu", "1.0.0");
         }
         const logger = logger_default({
           enableOtel: this._config?.telemetry?.enabled ?? false
@@ -5893,7 +6279,8 @@ var ExuluApp = class {
           this._tools,
           Object.values(this._contexts ?? {}),
           this._config,
-          tracer
+          tracer,
+          filesContext
         );
         if (this._config?.MCP.enabled) {
           const mcp = new ExuluMCP();
@@ -6173,7 +6560,7 @@ var RecursiveRules = class _RecursiveRules {
    * @param {string} path - The path to the recipe.
    * @returns {Promise<RecursiveRules>} The RecursiveRules object.
    */
-  static async fromRecipe(name = "default", lang = "en", path3) {
+  static async fromRecipe(name = "default", lang = "en", path2) {
     throw new Error("Not implemented");
   }
 };
@@ -7162,7 +7549,20 @@ var generateApiKey = async (name, email) => {
 };
 
 // src/postgres/init-db.ts
-var { agentsSchema: agentsSchema3, evalResultsSchema: evalResultsSchema3, jobsSchema: jobsSchema3, agentSessionsSchema: agentSessionsSchema3, agentMessagesSchema: agentMessagesSchema3, rolesSchema: rolesSchema3, usersSchema: usersSchema3, statisticsSchema: statisticsSchema3, variablesSchema: variablesSchema3, workflowTemplatesSchema: workflowTemplatesSchema3, rbacSchema: rbacSchema3, projectsSchema: projectsSchema3 } = coreSchemas.get();
+var {
+  agentsSchema: agentsSchema3,
+  evalResultsSchema: evalResultsSchema3,
+  jobsSchema: jobsSchema3,
+  agentSessionsSchema: agentSessionsSchema3,
+  agentMessagesSchema: agentMessagesSchema3,
+  rolesSchema: rolesSchema3,
+  usersSchema: usersSchema3,
+  statisticsSchema: statisticsSchema3,
+  variablesSchema: variablesSchema3,
+  workflowTemplatesSchema: workflowTemplatesSchema3,
+  rbacSchema: rbacSchema3,
+  projectsSchema: projectsSchema3
+} = coreSchemas.get();
 var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
   for (const field of fields) {
     const { type, name, default: defaultValue, unique } = field;
@@ -7219,6 +7619,7 @@ var up = async function(knex) {
     }
   };
   for (const schema of schemas) {
+    console.log(`[EXULU] Creating ${schema.name.plural} table.`, schema.fields);
     await createTable(schema);
   }
   if (!await knex.schema.hasTable("verification_token")) {
@@ -7375,6 +7776,9 @@ var create = ({
   return sdk;
 };
 
+// src/index.ts
+import CryptoJS4 from "crypto-js";
+
 // types/enums/jobs.ts
 var JOB_STATUS_ENUM = {
   completed: "completed",
@@ -7393,6 +7797,113 @@ var ExuluJobs = {
     validate: validateJob
   }
 };
+var ExuluDefaultContexts = {
+  files: filesContext,
+  codeStandards: codeStandardsContext,
+  outputs: outputsContext
+};
+var ExuluDefaultAgents = {
+  anthropic: {
+    opus4: claudeOpus4Agent,
+    sonnet4: claudeSonnet4Agent
+  },
+  openai: {
+    gpt5Mini: gpt5MiniAgent,
+    gpt5: gpt5agent
+  }
+};
+var ExuluVariables = {
+  get: async (name) => {
+    const { db: db3 } = await postgresClient();
+    let variable = await db3.from("variables").where({ name }).first();
+    if (!variable) {
+      throw new Error(`Variable ${name} not found.`);
+    }
+    if (variable.encrypted) {
+      const bytes = CryptoJS4.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+      variable.value = bytes.toString(CryptoJS4.enc.Utf8);
+    }
+    return variable.value;
+  }
+};
+var ExuluUtils = {
+  batch: async ({
+    fn,
+    size,
+    inputs,
+    delay,
+    retries
+  }) => {
+    if (!size) {
+      size = 10;
+    }
+    if (!inputs) {
+      throw new Error("Inputs are required.");
+    }
+    if (!delay) {
+      delay = 0;
+    }
+    let results = [];
+    let lastBatchTime = 0;
+    for (let start = 0; start < inputs.length; start += size) {
+      const currentTime = Date.now();
+      const timeSinceLastBatch = currentTime - lastBatchTime;
+      if (timeSinceLastBatch < delay * 1e3) {
+        console.log("[EXULU] Utils function, waiting for", delay - timeSinceLastBatch, "seconds");
+        await new Promise((resolve) => setTimeout(resolve, delay * 1e3 - timeSinceLastBatch));
+      }
+      lastBatchTime = Date.now();
+      console.log(`[EXULU] Utils function, processing batch ${start / size + 1} of ${Math.ceil(inputs.length / size)} (${Math.min(start + 1, inputs.length)}-${Math.min(start + size, inputs.length)} of ${inputs.length})`);
+      const end = start + size > inputs.length ? inputs.length : start + size;
+      const slicedResults = await Promise.all(inputs.slice(start, end).map((data, i) => {
+        if (retries?.max) {
+          return ExuluUtils.retry({
+            fn: async () => {
+              return await fn(data);
+            },
+            retries: retries.max,
+            delays: retries.delays
+          });
+        } else {
+          return fn(data);
+        }
+      }));
+      results = [
+        ...results,
+        ...slicedResults
+      ];
+    }
+    return results;
+  },
+  retry: async ({
+    fn,
+    retries,
+    delays
+  }) => {
+    if (!retries) {
+      retries = 3;
+    }
+    if (!delays) {
+      delays = [1e3, 5e3, 1e4];
+    }
+    for (let i = 0; i < retries; i++) {
+      try {
+        return await fn();
+      } catch (error) {
+        console.error(`[EXULU] Util function, retry attempt ${i + 1} failed:`, error);
+        if (i >= retries - 1) {
+          throw error;
+        }
+        if (!delays[i]) {
+          delays[i] = delays[delays.length - 1] || 1e4;
+        }
+        const delay = delays && delays[i] ? delays[i] : 1e4;
+        console.log(`[EXULU] Util function, retrying in ${delay / 1e3} seconds...`);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+    }
+  }
+};
 var ExuluOtel = {
   create: ({
     SIGNOZ_ACCESS_TOKEN,
@@ -7435,11 +7946,15 @@ export {
   authentication as ExuluAuthentication,
   ExuluChunkers,
   ExuluContext,
+  ExuluDefaultAgents,
+  ExuluDefaultContexts,
   ExuluEmbedder,
   ExuluEval,
   ExuluJobs,
   ExuluOtel,
   queues as ExuluQueues,
   ExuluTool2 as ExuluTool,
+  ExuluUtils,
+  ExuluVariables,
   db2 as db
 };