@exulu/backend 1.23.2 → 1.24.0

package/dist/index.cjs

@@ -37,12 +37,16 @@ __export(index_exports, {
   ExuluAuthentication: () => authentication,
   ExuluChunkers: () => ExuluChunkers,
   ExuluContext: () => ExuluContext,
+  ExuluDefaultAgents: () => ExuluDefaultAgents,
+  ExuluDefaultContexts: () => ExuluDefaultContexts,
   ExuluEmbedder: () => ExuluEmbedder,
   ExuluEval: () => ExuluEval,
   ExuluJobs: () => ExuluJobs,
   ExuluOtel: () => ExuluOtel,
   ExuluQueues: () => queues,
   ExuluTool: () => ExuluTool2,
+  ExuluUtils: () => ExuluUtils,
+  ExuluVariables: () => ExuluVariables,
   db: () => db2
 });
 module.exports = __toCommonJS(index_exports);
@@ -110,11 +114,8 @@ var validateJob = (job) => {
 };
 
 // src/registry/classes.ts
-var import_zod = require("zod");
 var import_bullmq2 = require("bullmq");
-var import_zod2 = require("zod");
-var fs = require("fs");
-var path = require("path");
+var import_zod = require("zod");
 var import_ai = require("ai");
 
 // types/enums/statistics.ts
@@ -141,8 +142,9 @@ var import_knex2 = require("knex");
 var import_knex3 = require("pgvector/knex");
 var db = {};
 var databaseExistsChecked = false;
+var dbName = process.env.POSTGRES_DB_NAME || "exulu";
 async function ensureDatabaseExists() {
-  console.log("[EXULU] Ensuring exulu database exists...");
+  console.log(`[EXULU] Ensuring ${dbName} database exists...`);
   const defaultKnex = (0, import_knex.default)({
     client: "pg",
     connection: {
@@ -157,14 +159,14 @@ async function ensureDatabaseExists() {
   });
   try {
     const result = await defaultKnex.raw(`
-            SELECT 1 FROM pg_database WHERE datname = 'exulu'
+            SELECT 1 FROM pg_database WHERE datname = '${dbName}'
         `);
     if (result.rows.length === 0) {
-      console.log("[EXULU] Database 'exulu' does not exist. Creating it...");
-      await defaultKnex.raw(`CREATE DATABASE exulu`);
-      console.log("[EXULU] Database 'exulu' created successfully.");
+      console.log(`[EXULU] Database '${dbName}' does not exist. Creating it...`);
+      await defaultKnex.raw(`CREATE DATABASE ${dbName}`);
+      console.log(`[EXULU] Database '${dbName}' created successfully.`);
     } else {
-      console.log("[EXULU] Database 'exulu' already exists.");
+      console.log(`[EXULU] Database '${dbName}' already exists.`);
     }
   } finally {
     await defaultKnex.destroy();
@@ -173,7 +175,7 @@ async function ensureDatabaseExists() {
 async function postgresClient() {
   if (!db["exulu"]) {
     try {
-      console.log("[EXULU] Connecting to exulu database.");
+      console.log(`[EXULU] Connecting to ${dbName} database.`);
       console.log("[EXULU] POSTGRES_DB_HOST:", process.env.POSTGRES_DB_HOST);
       console.log("[EXULU] POSTGRES_DB_PORT:", process.env.POSTGRES_DB_PORT);
       console.log("[EXULU] POSTGRES_DB_USER:", process.env.POSTGRES_DB_USER);
@@ -181,7 +183,7 @@ async function postgresClient() {
       console.log("[EXULU] POSTGRES_DB_SSL:", process.env.POSTGRES_DB_SSL);
       console.log("[EXULU] Database exists checked:", databaseExistsChecked);
       if (!databaseExistsChecked) {
-        console.log("[EXULU] Ensuring exulu database exists...");
+        console.log(`[EXULU] Ensuring ${dbName} database exists...`);
         await ensureDatabaseExists();
         databaseExistsChecked = true;
       }
@@ -191,7 +193,7 @@ async function postgresClient() {
           host: process.env.POSTGRES_DB_HOST,
           port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
           user: process.env.POSTGRES_DB_USER,
-          database: "exulu",
+          database: dbName,
           password: process.env.POSTGRES_DB_PASSWORD,
           ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
         }
@@ -207,6 +209,16 @@ async function postgresClient() {
     db: db["exulu"]
   };
 }
+var refreshPostgresClient = async () => {
+  if (db["exulu"]) {
+    await db["exulu"].destroy();
+    db["exulu"] = void 0;
+  }
+  const { db: refreshed } = await postgresClient();
+  return {
+    db: refreshed
+  };
+};
 
 // src/registry/classes.ts
 var import_knex5 = __toESM(require("pgvector/knex"), 1);
@@ -281,6 +293,15 @@ var mapType = (t, type, name, defaultValue, unique) => {
     if (unique) t.unique(name);
     return;
   }
+  if (type === "markdown") {
+    if (defaultValue) {
+      t.text(name).defaultTo(defaultValue);
+    } else {
+      t.text(name);
+    }
+    if (unique) t.unique(name);
+    return;
+  }
   if (type === "shortText") {
     if (defaultValue) {
       t.string(name, 100).defaultTo(defaultValue);
@@ -428,12 +449,11 @@ var ExuluEvalUtils = {
 // src/registry/classes.ts
 var import_crypto_js2 = __toESM(require("crypto-js"), 1);
 var import_express = require("express");
-var import_api = require("@opentelemetry/api");
 
 // src/registry/utils/graphql.ts
 var import_schema = require("@graphql-tools/schema");
 var import_graphql_type_json = __toESM(require("graphql-type-json"), 1);
-var import_graphql = require("graphql");
+var import_graphql2 = require("graphql");
 var import_crypto_js = __toESM(require("crypto-js"), 1);
 
 // src/auth/get-token.ts
@@ -484,7 +504,7 @@ var authentication = async ({
       code: 200,
       user: {
         type: "api",
-        id: "XXXX-XXXX-XXXX-XXXX",
+        id: 192837465,
         email: "internal@exulu.com",
         role: {
           id: "internal",
@@ -805,7 +825,8 @@ var agentSessionsSchema = {
     },
     {
       name: "project",
-      type: "uuid"
+      type: "uuid",
+      required: false
     }
   ]
 };
@@ -905,12 +926,13 @@ var projectsSchema = {
       type: "text"
     },
     {
-      name: "custom_instructions",
-      type: "longText"
+      name: "project_items",
+      // array of items as global ids ('<context_id>/<item_id>')
+      type: "json"
     },
     {
-      name: "context_files",
-      type: "json"
+      name: "custom_instructions",
+      type: "longText"
     }
   ]
 };
@@ -930,20 +952,24 @@ var agentsSchema = {
       name: "image",
       type: "text"
     },
+    {
+      name: "category",
+      type: "text"
+    },
     {
       name: "description",
       type: "text"
     },
     {
-      name: "providerApiKey",
+      name: "instructions",
       type: "text"
     },
     {
-      name: "backend",
+      name: "providerapikey",
       type: "text"
     },
     {
-      name: "type",
+      name: "backend",
       type: "text"
     },
     {
@@ -1277,18 +1303,22 @@ var rbacSchema = {
 };
 var addRBACfields = (schema) => {
   if (schema.RBAC) {
-    schema.fields.push({
-      name: "rights_mode",
-      type: "text",
-      required: false,
-      default: "private"
-    });
-    schema.fields.push({
-      name: "created_by",
-      type: "number",
-      required: true,
-      default: 0
-    });
+    if (!schema.fields.some((field) => field.name === "rights_mode")) {
+      schema.fields.push({
+        name: "rights_mode",
+        type: "text",
+        required: false,
+        default: "private"
+      });
+    }
+    if (!schema.fields.some((field) => field.name === "created_by")) {
+      schema.fields.push({
+        name: "created_by",
+        type: "number",
+        required: true,
+        default: 0
+      });
+    }
   }
   return schema;
 };
@@ -1320,7 +1350,211 @@ var VectorMethodEnum = {
 
 // src/registry/utils/graphql.ts
 var import_knex4 = require("knex");
-var GraphQLDate = new import_graphql.GraphQLScalarType({
+
+// src/registry/rate-limiter.ts
+var rateLimiter = async (key, windowSeconds, limit, points) => {
+  try {
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      console.warn("[EXULU] Rate limiting disabled - Redis not available");
+      return {
+        status: true,
+        retryAfter: null
+      };
+    }
+    const redisKey = `exulu/${key}`;
+    const current = await client2.incrBy(redisKey, points);
+    if (current === points) {
+      await client2.expire(redisKey, windowSeconds);
+    }
+    if (current > limit) {
+      const ttl = await client2.ttl(redisKey);
+      return {
+        status: false,
+        retryAfter: ttl
+      };
+    }
+    return {
+      status: true,
+      retryAfter: null
+    };
+  } catch (error) {
+    console.error("[EXULU] Rate limiting error:", error);
+    return {
+      status: true,
+      retryAfter: null
+    };
+  }
+};
+
+// src/registry/utils.ts
+var bullmq = {
+  validate: (id, data) => {
+    if (!data) {
+      throw new Error(`Missing job data for job ${id}.`);
+    }
+    if (!data.type) {
+      throw new Error(`Missing property "type" in data for job ${id}.`);
+    }
+    if (!data.inputs) {
+      throw new Error(`Missing property "inputs" in data for job ${id}.`);
+    }
+    if (data.type !== "embedder" && data.type !== "workflow") {
+      throw new Error(`Property "type" in data for job ${id} must be of value "embedder" or "workflow".`);
+    }
+    if (!data.workflow && !data.embedder) {
+      throw new Error(`Either a workflow or embedder must be set for job ${id}.`);
+    }
+  }
+};
+var getEnabledTools = async (agentInstance, allExuluTools, disabledTools = [], agents, user) => {
+  let enabledTools = [];
+  if (agentInstance.tools) {
+    const results = await Promise.all(agentInstance.tools.map(
+      async ({ config, id, type }) => {
+        let hydrated;
+        if (type === "agent") {
+          if (id === agentInstance.id) {
+            return null;
+          }
+          const instance = await loadAgent(id);
+          if (!instance) {
+            throw new Error("Trying to load a tool of type 'agent', but the associated agent with id " + id + " was not found in the database.");
+          }
+          const backend = agents.find((a) => a.id === instance.backend);
+          if (!backend) {
+            throw new Error("Trying to load a tool of type 'agent', but the associated agent with id " + id + " does not have a backend set for it.");
+          }
+          const hasAccessToAgent = await checkRecordAccess(instance, "read", user);
+          if (!hasAccessToAgent) {
+            return null;
+          }
+          hydrated = await backend.tool(instance.id, agents);
+        } else {
+          hydrated = allExuluTools.find((t) => t.id === id);
+        }
+        return hydrated;
+      }
+    ));
+    enabledTools = results.filter(Boolean);
+  }
+  console.log("[EXULU] available tools", enabledTools?.length);
+  console.log("[EXULU] disabled tools", disabledTools?.length);
+  enabledTools = enabledTools.filter((tool2) => !disabledTools.includes(tool2.id));
+  return enabledTools;
+};
+var loadAgentCache = /* @__PURE__ */ new Map();
+var loadAgents = async () => {
+  const { db: db3 } = await postgresClient();
+  const agents = await db3.from("agents");
+  for (const agent of agents) {
+    const agentRbac = await RBACResolver(db3, "agent", agent.id, agent.rights_mode || "private");
+    agent.RBAC = agentRbac;
+    loadAgentCache.set(agent.id, {
+      agent,
+      expiresAt: new Date(Date.now() + 1e3 * 60 * 1)
+      // 1 minute
+    });
+  }
+  return agents;
+};
+var loadAgent = async (id) => {
+  const cachedAgent = loadAgentCache.get(id);
+  if (cachedAgent && cachedAgent.expiresAt > /* @__PURE__ */ new Date()) {
+    return cachedAgent.agent;
+  }
+  const { db: db3 } = await postgresClient();
+  const agentInstance = await db3.from("agents").where({
+    id
+  }).first();
+  const agentRbac = await RBACResolver(db3, "agent", agentInstance.id, agentInstance.rights_mode || "private");
+  agentInstance.RBAC = agentRbac;
+  if (!agentInstance) {
+    throw new Error("Agent instance not found.");
+  }
+  loadAgentCache.set(id, {
+    agent: agentInstance,
+    expiresAt: new Date(Date.now() + 1e3 * 60 * 1)
+    // 1 minute
+  });
+  return agentInstance;
+};
+var checkAgentRateLimit = async (agent) => {
+  if (agent.rateLimit) {
+    console.log("[EXULU] rate limiting agent.", agent.rateLimit);
+    const limit = await rateLimiter(
+      agent.rateLimit.name || agent.id,
+      agent.rateLimit.rate_limit.time,
+      agent.rateLimit.rate_limit.limit,
+      1
+    );
+    if (!limit.status) {
+      throw new Error("Rate limit exceeded.");
+    }
+  }
+};
+var checkRecordAccessCache = /* @__PURE__ */ new Map();
+var checkRecordAccess = async (record, request, user) => {
+  const setRecordAccessCache = (hasAccess2) => {
+    checkRecordAccessCache.set(`${record.id}-${request}-${user?.id}`, {
+      hasAccess: hasAccess2,
+      expiresAt: new Date(Date.now() + 1e3 * 60 * 1)
+      // 1 minute
+    });
+  };
+  const cachedAccess = checkRecordAccessCache.get(`${record.id}-${request}-${user?.id}`);
+  if (cachedAccess && cachedAccess.expiresAt > /* @__PURE__ */ new Date()) {
+    return cachedAccess.hasAccess;
+  }
+  const isPublic = record.rights_mode === "public";
+  const byUsers = record.rights_mode === "users";
+  const byRoles = record.rights_mode === "roles";
+  const isCreator = user ? record.created_by === user.id.toString() : false;
+  const isAdmin = user ? user.super_admin : false;
+  const isApi = user ? user.type === "api" : false;
+  let hasAccess = "none";
+  if (isPublic || isCreator || isAdmin || isApi) {
+    setRecordAccessCache(true);
+    return true;
+  }
+  if (byUsers) {
+    if (!user) {
+      setRecordAccessCache(false);
+      return false;
+    }
+    console.log("record.RBAC?.users", record.RBAC?.users);
+    console.log("user.id", user.id.toString());
+    hasAccess = record.RBAC?.users?.find((x) => x.id === user.id)?.rights || "none";
+    if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
+      console.error(`Your current user ${user.id} does not have access to this record, current access type is: ${hasAccess}.`);
+      setRecordAccessCache(false);
+      return false;
+    } else {
+      setRecordAccessCache(true);
+      return true;
+    }
+  }
+  if (byRoles) {
+    if (!user) {
+      setRecordAccessCache(false);
+      return false;
+    }
+    hasAccess = record.RBAC?.roles?.find((x) => x.id === user.role?.id)?.rights || "none";
+    if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
+      console.error(`Your current role ${user.role?.name} does not have access to this record, current access type is: ${hasAccess}.`);
+      setRecordAccessCache(false);
+      return false;
+    } else {
+      setRecordAccessCache(true);
+      return true;
+    }
+  }
+  setRecordAccessCache(false);
+  return false;
+};
+
+// src/registry/utils/graphql.ts
+var GraphQLDate = new import_graphql2.GraphQLScalarType({
   name: "Date",
   description: "Date custom scalar type",
   serialize(value) {
@@ -1345,10 +1579,10 @@ var GraphQLDate = new import_graphql.GraphQLScalarType({
     return value;
   },
   parseLiteral(ast) {
-    if (ast.kind === import_graphql.Kind.STRING) {
+    if (ast.kind === import_graphql2.Kind.STRING) {
       return new Date(ast.value);
     }
-    if (ast.kind === import_graphql.Kind.INT) {
+    if (ast.kind === import_graphql2.Kind.INT) {
       return new Date(parseInt(ast.value, 10));
     }
     return null;
@@ -1360,6 +1594,7 @@ var map = (field) => {
     case "text":
     case "shortText":
     case "longText":
+    case "markdown":
     case "code":
       type = "String";
       break;
@@ -1411,6 +1646,7 @@ ${enumValues}
     fields.push("  rateLimit: RateLimiterRule");
     fields.push("  streaming: Boolean");
     fields.push("  capabilities: AgentCapabilities");
+    fields.push("  maxContextLength: Int");
     fields.push("  slug: String");
   }
   const rbacField = table.RBAC ? "  RBAC: RBACData" : "";
@@ -1745,7 +1981,6 @@ function createMutations(table, agents, contexts, tools) {
         input.id = db3.fn.uuid();
       }
       const columns = await db3(tableNamePlural).columnInfo();
-      console.log("[EXULU] Columns", columns);
       const insert = db3(tableNamePlural).insert({
         ...input,
         ...table.RBAC ? { rights_mode: "private" } : {}
@@ -1760,7 +1995,7 @@ function createMutations(table, agents, contexts, tools) {
       const { job } = await postprocessUpdate({ table, requestedFields, agents, contexts, tools, result: results[0], user: context.user.id, role: context.user.role?.id });
       return {
         // Filter result to only include requested fields
-        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: results[0] }),
+        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: results[0], user: context.user }),
         job
       };
     },
@@ -1803,7 +2038,7 @@ function createMutations(table, agents, contexts, tools) {
       }
       const { job } = await postprocessUpdate({ table, requestedFields, agents, contexts, tools, result, user: context.user.id, role: context.user.role?.id });
       return {
-        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result }),
+        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user.id }),
         job
       };
     },
@@ -1839,7 +2074,7 @@ function createMutations(table, agents, contexts, tools) {
       const result = await db3.from(tableNamePlural).select(Object.keys(columns)).where({ id }).first();
       const { job } = await postprocessUpdate({ table, requestedFields, agents, contexts, tools, result, user: context.user.id, role: context.user.role?.id });
       return {
-        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result }),
+        item: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user.id }),
         job
       };
     },
@@ -1871,7 +2106,7 @@ function createMutations(table, agents, contexts, tools) {
         }).del();
       }
       await postprocessDeletion({ table, requestedFields, agents, contexts, tools, result });
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user.id });
     },
     [`${tableNamePlural}RemoveOne`]: async (_, args, context, info) => {
       const { where } = args;
@@ -1895,7 +2130,7 @@ function createMutations(table, agents, contexts, tools) {
       }
       await db3(tableNamePlural).where(where).del();
       await postprocessDeletion({ table, requestedFields, agents, contexts, tools, result });
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user.id });
     }
   };
   if (table.type === "items") {
@@ -1975,7 +2210,6 @@ function createMutations(table, agents, contexts, tools) {
   return mutations;
 }
 var applyAccessControl = (table, user, query) => {
-  console.log("table", table);
   const tableNamePlural = table.name.plural.toLowerCase();
   if (!user.super_admin && table.name.plural === "jobs") {
     query = query.where("created_by", user.id);
@@ -2002,7 +2236,6 @@ var applyAccessControl = (table, user, query) => {
         });
       });
       if (user.role) {
-        console.log("user.role", user.role);
         this.orWhere(function() {
           this.where("rights_mode", "roles").whereExists(function() {
             this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "Role").where("rbac.role_id", user.role.id);
@@ -2043,14 +2276,15 @@ var backendAgentFields = [
   "slug",
   "rateLimit",
   "streaming",
-  "capabilities"
+  "capabilities",
+  "maxContextLength"
 ];
 var removeAgentFields = (requestedFields) => {
   const filtered = requestedFields.filter((field) => !backendAgentFields.includes(field));
   filtered.push("backend");
   return filtered;
 };
-var addAgentFields = (requestedFields, agents, result, tools) => {
+var addAgentFields = async (requestedFields, agents, result, tools, user) => {
   let backend = agents.find((a) => a.id === result?.backend);
   if (requestedFields.includes("providerName")) {
     result.providerName = backend?.providerName || "";
@@ -2065,13 +2299,39 @@ var addAgentFields = (requestedFields, agents, result, tools) => {
     result.rateLimit = backend?.rateLimit || "";
   }
   if (requestedFields.includes("tools")) {
-    result.tools = result.tools ? result.tools.map((tool2) => {
-      return {
-        ...tool2,
-        name: tools.find((t) => t.id === tool2.toolId)?.name || "",
-        description: tools.find((t) => t.id === tool2.toolId)?.description || ""
-      };
-    }) : [];
+    if (result.tools) {
+      result.tools = await Promise.all(result.tools.map(async (tool2) => {
+        let hydrated;
+        if (tool2.type === "agent") {
+          if (tool2.id === result.id) {
+            return null;
+          }
+          const instance = await loadAgent(tool2.id);
+          if (!instance) {
+            throw new Error("Trying to load a tool of type 'agent', but the associated agent with id " + tool2.id + " was not found in the database.");
+          }
+          const backend2 = agents.find((a) => a.id === instance.backend);
+          if (!backend2) {
+            throw new Error("Trying to load a tool of type 'agent', but the associated agent with id " + tool2.id + " does not have a backend set for it.");
+          }
+          const hasAccessToAgent = await checkRecordAccess(instance, "read", user);
+          if (!hasAccessToAgent) {
+            return null;
+          }
+          hydrated = await backend2.tool(instance.id, agents);
+        } else {
+          hydrated = tools.find((t) => t.id === tool2.id);
+        }
+        return {
+          ...tool2,
+          name: hydrated?.name || "",
+          description: hydrated?.description || ""
+        };
+      }));
+      result.tools = result.tools.filter((tool2) => tool2 !== null);
+    } else {
+      result.tools = [];
+    }
   }
   if (requestedFields.includes("streaming")) {
     result.streaming = backend?.streaming || false;
@@ -2079,6 +2339,9 @@ var addAgentFields = (requestedFields, agents, result, tools) => {
   if (requestedFields.includes("capabilities")) {
     result.capabilities = backend?.capabilities || [];
   }
+  if (requestedFields.includes("maxContextLength")) {
+    result.maxContextLength = backend?.maxContextLength || 0;
+  }
   if (!requestedFields.includes("backend")) {
     delete result.backend;
   }
@@ -2189,7 +2452,8 @@ var finalizeRequestedFields = async ({
   agents,
   contexts,
   tools,
-  result
+  result,
+  user
 }) => {
   if (!result) {
     return result;
@@ -2199,11 +2463,11 @@ var finalizeRequestedFields = async ({
   }
   if (Array.isArray(result)) {
     result = result.map((item) => {
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: item });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: item, user });
     });
   } else {
     if (table.name.singular === "agent") {
-      result = addAgentFields(requestedFields, agents, result, tools);
+      result = await addAgentFields(requestedFields, agents, result, tools, user);
       if (!requestedFields.includes("backend")) {
         delete result.backend;
       }
@@ -2250,7 +2514,6 @@ var applyFilters = (query, filters) => {
     Object.entries(filter).forEach(([fieldName, operators]) => {
       if (operators) {
         if (operators.and !== void 0) {
-          console.log("operators.and", operators.and);
           operators.and.forEach((operator) => {
             query = converOperatorToQuery(query, fieldName, operator);
           });
@@ -2261,7 +2524,6 @@ var applyFilters = (query, filters) => {
           });
         }
         query = converOperatorToQuery(query, fieldName, operators);
-        console.log("query", query);
       }
     });
   });
@@ -2284,7 +2546,7 @@ function createQueries(table, agents, tools, contexts) {
       let query = db3.from(tableNamePlural).select(sanitizedFields).where({ id: args.id });
       query = applyAccessControl(table, context.user, query);
       let result = await query.first();
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user });
     },
     [`${tableNameSingular}ByIds`]: async (_, args, context, info) => {
       const { db: db3 } = context;
@@ -2293,7 +2555,7 @@ function createQueries(table, agents, tools, contexts) {
       let query = db3.from(tableNamePlural).select(sanitizedFields).whereIn("id", args.ids);
       query = applyAccessControl(table, context.user, query);
       let result = await query;
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user });
     },
     [`${tableNameSingular}One`]: async (_, args, context, info) => {
       const { filters = [], sort } = args;
@@ -2305,7 +2567,7 @@ function createQueries(table, agents, tools, contexts) {
       query = applyAccessControl(table, context.user, query);
       query = applySorting(query, sort);
       let result = await query.first();
-      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result });
+      return finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result, user: context.user });
     },
     [`${tableNamePlural}Pagination`]: async (_, args, context, info) => {
       const { limit = 10, page = 0, filters = [], sort } = args;
@@ -2316,7 +2578,6 @@ function createQueries(table, agents, tools, contexts) {
       let countQuery = db3(tableNamePlural);
       countQuery = applyFilters(countQuery, filters);
       countQuery = applyAccessControl(table, context.user, countQuery);
-      console.log("countQuery", countQuery);
       const countResult = await countQuery.count("* as count");
       const itemCount = Number(countResult[0]?.count || 0);
       const pageCount = Math.ceil(itemCount / limit);
@@ -2341,7 +2602,7 @@ function createQueries(table, agents, tools, contexts) {
           hasPreviousPage,
           hasNextPage
         },
-        items: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: items })
+        items: finalizeRequestedFields({ table, requestedFields, agents, contexts, tools, result: items, user: context.user })
       };
     },
     // Add generic statistics query for all tables
@@ -2359,7 +2620,6 @@ function createQueries(table, agents, tools, contexts) {
           query = query.count("* as count");
         }
         const results = await query;
-        console.log("!!! results !!!", results);
         return results.map((r) => ({
           group: r[groupBy],
           count: r.count ? Number(r.count) : 0
@@ -2368,7 +2628,6 @@ function createQueries(table, agents, tools, contexts) {
         if (tableNamePlural === "tracking") {
           query = query.sum("total as count");
           const [{ count }] = await query.sum("total as count");
-          console.log("!!! count !!!", count);
           return [{
             group: "total",
             count: count ? Number(count) : 0
@@ -2453,7 +2712,6 @@ var vectorSearch = async ({
   let countQuery = db3(mainTable);
   countQuery = applyFilters(countQuery, filters);
   countQuery = applyAccessControl(table, user, countQuery);
-  console.log("countQuery", countQuery);
   const columns = await db3(mainTable).columnInfo();
   let itemsQuery = db3(mainTable).select(Object.keys(columns).map((column) => mainTable + "." + column));
   itemsQuery = applyFilters(itemsQuery, filters);
@@ -2592,7 +2850,6 @@ var vectorSearch = async ({
       ];
       items = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
   }
-  console.log("items", items);
   const seenSources = /* @__PURE__ */ new Map();
   items = items.reduce((acc, item) => {
     if (!seenSources.has(item.source)) {
@@ -2635,7 +2892,6 @@ var vectorSearch = async ({
     }
     return acc;
   }, []);
-  console.log("items", items);
   items.forEach((item) => {
     if (!item.chunks?.length) {
       return;
@@ -2657,7 +2913,6 @@ var vectorSearch = async ({
       item.averageRelevance = average;
       item.totalRelevance = total;
     } else if (method === "hybridSearch") {
-      console.log("item.chunks", item.chunks);
       const scores = item.chunks.map((c) => typeof c.hybrid_score === "number" ? c.hybrid_score * 10 + 1 : 0);
       const total = scores.reduce((a, b) => a + b, 0);
       const average = scores.length ? total / scores.length : 0;
@@ -2746,6 +3001,10 @@ var contextToTableDefinition = (context) => {
     name: "textlength",
     type: "number"
   });
+  definition.fields.push({
+    name: "ttl",
+    type: "text"
+  });
   definition.fields.push({
     name: "embeddings_updated_at",
     type: "date"
@@ -2789,7 +3048,7 @@ function createSDL(tables, contexts, agents, tools) {
     }
   });
   tables = [...tables, ...contextSchemas];
-  console.log("[EXULU] Creating SDL");
+  console.log("[EXULU] Creating SDL.");
   let typeDefs = `
     scalar JSON
     scalar Date
@@ -3030,8 +3289,19 @@ type PageInfo {
   };
   resolvers.Query["tools"] = async (_, args, context, info) => {
     const requestedFields = getRequestedFields(info);
+    const instances = await loadAgents();
+    let agentTools = await Promise.all(
+      instances.map(async (instance) => {
+        const backend = agents.find((a) => a.id === instance.backend);
+        if (!backend) {
+          return null;
+        }
+        return await backend.tool(instance.id, agents);
+      })
+    );
+    const filtered = agentTools.filter((tool2) => tool2 !== null);
     return {
-      items: tools.map((tool2) => {
+      items: [...filtered, ...tools].map((tool2) => {
         const object = {};
         requestedFields.forEach((field) => {
           object[field] = tool2[field];
@@ -3127,7 +3397,6 @@ type Tool {
 
 enum EnumProviderType {
   agent
-  custom
 }
 
 type StatisticsResult {
@@ -3161,6 +3430,9 @@ var validateCreateOrRemoveSuperAdminPermission = async (tableNamePlural, input,
 };
 
 // src/registry/classes.ts
+var import_client_s3 = require("@aws-sdk/client-s3");
+var import_node_crypto = require("crypto");
+var s3Client;
 function sanitizeToolName(name) {
   if (typeof name !== "string") return "";
   let sanitized = name.replace(/[^a-zA-Z0-9_-]+/g, "_");
@@ -3170,17 +3442,18 @@ function sanitizeToolName(name) {
   }
   return sanitized;
 }
-var convertToolsArrayToObject = (tools, configs, providerApiKey, user, role) => {
-  if (!tools) return {};
-  const sanitizedTools = tools ? tools.map((tool2) => ({
+var convertToolsArrayToObject = (currentTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, filesContext2) => {
+  if (!currentTools) return {};
+  if (!allExuluTools) return {};
+  const sanitizedTools = currentTools ? currentTools.map((tool2) => ({
     ...tool2,
     name: sanitizeToolName(tool2.name)
   })) : [];
-  console.log("[EXULU] Sanitized tools", sanitizedTools);
+  console.log("[EXULU] Sanitized tools", sanitizedTools.map((x) => x.name + " (" + x.id + ")"));
   const askForConfirmation = {
     description: "Ask the user for confirmation.",
-    inputSchema: import_zod2.z.object({
-      message: import_zod2.z.string().describe("The message to ask for confirmation.")
+    inputSchema: import_zod.z.object({
+      message: import_zod.z.string().describe("The message to ask for confirmation.")
     })
   };
   return {
@@ -3189,29 +3462,91 @@ var convertToolsArrayToObject = (tools, configs, providerApiKey, user, role) =>
         ...prev,
         [cur.name]: {
           ...cur.tool,
-          execute: async (inputs, options) => {
+          async *execute(inputs, options) {
             if (!cur.tool?.execute) {
               console.error("[EXULU] Tool execute function is undefined.", cur.tool);
               throw new Error("Tool execute function is undefined.");
             }
-            let config = configs?.find((config2) => config2.toolId === cur.id);
+            let config = configs?.find((config2) => config2.id === cur.id);
             if (config) {
               config = await hydrateVariables(config || []);
             }
+            let upload = void 0;
+            if (exuluConfig?.fileUploads?.s3endpoint && exuluConfig?.fileUploads?.s3key && exuluConfig?.fileUploads?.s3secret && exuluConfig?.fileUploads?.s3Bucket && filesContext2) {
+              s3Client ??= new import_client_s3.S3Client({
+                region: exuluConfig?.fileUploads?.s3region,
+                ...exuluConfig?.fileUploads?.s3endpoint && {
+                  forcePathStyle: true,
+                  endpoint: exuluConfig?.fileUploads?.s3endpoint
+                },
+                credentials: {
+                  accessKeyId: exuluConfig?.fileUploads?.s3key ?? "",
+                  secretAccessKey: exuluConfig?.fileUploads?.s3secret ?? ""
+                }
+              });
+              upload = async ({
+                name,
+                data,
+                type,
+                tags
+              }) => {
+                const mime = getMimeType(type);
+                const key = `${user}/${generateS3Key(name)}${type}`;
+                const command = new import_client_s3.PutObjectCommand({
+                  Bucket: exuluConfig?.fileUploads?.s3Bucket,
+                  Key: key,
+                  Body: data,
+                  ContentType: mime
+                });
+                try {
+                  const response2 = await s3Client.send(command);
+                  console.log(response2);
+                  const { item } = await filesContext2.createItem({
+                    name: `${name}${type}`,
+                    type: mime,
+                    rights_mode: "private",
+                    s3key: key,
+                    tags
+                  }, user?.id, user?.role?.id, false);
+                  return item;
+                } catch (caught) {
+                  if (caught instanceof import_client_s3.S3ServiceException && caught.name === "EntityTooLarge") {
+                    console.error(
+                      `Error from S3 while uploading object to ${exuluConfig?.fileUploads?.s3Bucket}.                                 The object was too large. To upload objects larger than 5GB, use the S3 console (160GB max)                                 or the multipart upload API (5TB max).`
+                    );
+                  } else if (caught instanceof import_client_s3.S3ServiceException) {
+                    console.error(
+                      `Error from S3 while uploading object to ${exuluConfig?.fileUploads?.s3Bucket}.  ${caught.name}: ${caught.message}`
+                    );
+                  } else {
+                    throw caught;
+                  }
+                }
+              };
+            }
+            const contextsMap = contexts?.reduce((acc, curr) => {
+              acc[curr.id] = curr;
+              return acc;
+            }, {});
             console.log("[EXULU] Config", config);
-            return await cur.tool.execute({
+            const response = await cur.tool.execute({
               ...inputs,
               // Convert config to object format if a config object 
               // is available, after we added the .value property
               // by hydrating it from the variables table.
-              providerApiKey,
+              providerapikey,
+              allExuluTools,
+              currentTools,
               user,
-              role,
+              contexts: contextsMap,
+              upload,
               config: config ? config.config.reduce((acc, curr) => {
                 acc[curr.name] = curr.value;
                 return acc;
               }, {}) : {}
             }, options);
+            yield response;
+            return response;
           }
         }
       }),
@@ -3246,6 +3581,18 @@ function generateSlug(name) {
   const slug = lowercase.replace(/[\W_]+/g, "-").replace(/^-+|-+$/g, "");
   return slug;
 }
+function errorHandler(error) {
+  if (error == null) {
+    return "unknown error";
+  }
+  if (typeof error === "string") {
+    return error;
+  }
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return JSON.stringify(error);
+}
 var ExuluAgent2 = class {
   // Must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or 
   // underscores and be a max length of 80 characters and at least 5 characters long.
@@ -3256,13 +3603,14 @@ var ExuluAgent2 = class {
   slug = "";
   type;
   streaming = false;
+  maxContextLength;
   rateLimit;
   config;
   // private memory: Memory | undefined; // TODO do own implementation
   evals;
   model;
   capabilities;
-  constructor({ id, name, description, config, rateLimit, capabilities, type, evals }) {
+  constructor({ id, name, description, config, rateLimit, capabilities, type, evals, maxContextLength }) {
     this.id = id;
     this.name = name;
     this.evals = evals;
@@ -3270,6 +3618,7 @@ var ExuluAgent2 = class {
     this.rateLimit = rateLimit;
     this.config = config;
     this.type = type;
+    this.maxContextLength = maxContextLength;
     this.capabilities = capabilities || {
       text: false,
       images: [],
@@ -3296,31 +3645,82 @@ var ExuluAgent2 = class {
   }
   // Exports the agent as a tool that can be used by another agent
   // todo test this
-  tool = () => {
+  tool = async (instance, agents) => {
+    const agentInstance = await loadAgent(instance);
+    if (!agentInstance) {
+      return null;
+    }
     return new ExuluTool2({
-      id: this.id,
-      name: `${this.name}`,
+      id: agentInstance.id,
+      name: `${agentInstance.name}`,
       type: "agent",
-      inputSchema: import_zod2.z.object({
-        prompt: import_zod2.z.string()
+      inputSchema: import_zod.z.object({
+        prompt: import_zod.z.string().describe("The prompt (usually a question for the agent) to send to the agent."),
+        information: import_zod.z.string().describe("A summary of relevant context / information from the current session")
       }),
-      description: `A function that calls an AI agent named: ${this.name}. The agent does the following: ${this.description}.`,
+      description: `This tool calls an AI agent named: ${agentInstance.name}. The agent does the following: ${agentInstance.description}.`,
       config: [],
-      execute: async ({ prompt, config, providerApiKey, user, role }) => {
-        return await this.generateSync({
-          prompt,
-          providerApiKey,
+      execute: async ({ prompt, information, user, allExuluTools }) => {
+        const hasAccessToAgent = await checkRecordAccess(agentInstance, "read", user);
+        if (!hasAccessToAgent) {
+          throw new Error("You don't have access to this agent.");
+        }
+        let enabledTools = await getEnabledTools(agentInstance, allExuluTools, [], agents, user);
+        const variableName = agentInstance.providerapikey;
+        if (!variableName) {
+          throw new Error("Provider API key variable not set for agent: " + agentInstance.name + " (" + agentInstance.id + ") being called as a tool.");
+        }
+        const { db: db3 } = await postgresClient();
+        const variable = await db3.from("variables").where({ name: variableName }).first();
+        if (!variable) {
+          throw new Error("Provider API key variable not found for agent: " + agentInstance.name + " (" + agentInstance.id + ") being called as a tool.");
+        }
+        let providerapikey = variable.value;
+        if (!variable.encrypted) {
+          throw new Error("Provider API key variable not encrypted for agent: " + agentInstance.name + " (" + agentInstance.id + ") being called as a tool, for security reasons you are only allowed to use encrypted variables for provider API keys.");
+        }
+        if (variable.encrypted) {
+          const bytes = import_crypto_js2.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+          providerapikey = bytes.toString(import_crypto_js2.default.enc.Utf8);
+        }
+        console.log("[EXULU] Enabled tools for agent '" + agentInstance.name + " (" + agentInstance.id + ") that is being called as a tool", enabledTools.map((x) => x.name + " (" + x.id + ")"));
+        console.log("[EXULU] Prompt for agent '" + agentInstance.name + "' that is being called as a tool", prompt.slice(0, 100) + "...");
+        console.log("[EXULU] Instructions for agent '" + agentInstance.name + "' that is being called as a tool", agentInstance.instructions?.slice(0, 100) + "...");
+        const response = await this.generateSync({
+          instructions: agentInstance.instructions,
+          prompt: "The user has asked the following question: " + prompt + " and the following information is available: " + information,
+          providerapikey,
           user,
-          role,
+          currentTools: enabledTools,
+          allExuluTools,
           statistics: {
-            label: "",
+            label: agentInstance.name,
             trigger: "tool"
           }
         });
+        return {
+          result: response
+        };
       }
     });
   };
-  generateSync = async ({ prompt, user, role, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
+  generateSync = async ({
+    prompt,
+    user,
+    session,
+    message,
+    currentTools,
+    allExuluTools,
+    statistics,
+    toolConfigs,
+    providerapikey,
+    contexts,
+    exuluConfig,
+    filesContext: filesContext2,
+    outputSchema,
+    instructions
+  }) => {
+    console.log("[EXULU] Called generate sync for agent: " + this.name, "with prompt: " + prompt?.slice(0, 100) + "...");
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
@@ -3333,14 +3733,18 @@ var ExuluAgent2 = class {
     if (!prompt && !message) {
       throw new Error("Prompt or message is required for generating.");
     }
+    if (outputSchema && !prompt) {
+      throw new Error("Prompt is required for generating with an output schema.");
+    }
     const model = this.model.create({
-      apiKey: providerApiKey
+      apiKey: providerapikey
     });
+    console.log("[EXULU] Model for agent: " + this.name, " created for generating sync.");
     let messages = [];
     if (message && session && user) {
       const previousMessages = await getAgentMessages({
         session,
-        user,
+        user: user.id,
         limit: 50,
         page: 1
       });
@@ -3350,56 +3754,134 @@ var ExuluAgent2 = class {
         messages: [...previousMessagesContent, message]
       });
     }
-    console.log("[EXULU] Model provider key", providerApiKey);
-    console.log("[EXULU] Tool configs", toolConfigs);
+    console.log("[EXULU] Message count for agent: " + this.name, "loaded for generating sync.", messages.length);
+    const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
+    let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
+    system += "\n\n" + genericContext;
     if (prompt) {
-      const { text } = await (0, import_ai.generateText)({
-        model,
-        // Should be a LanguageModelV1
-        system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
-        prompt,
-        maxRetries: 2,
-        tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
-        stopWhen: [(0, import_ai.stepCountIs)(5)]
-      });
-      if (statistics) {
-        await updateStatistic({
-          name: "count",
-          label: statistics.label,
-          type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-          trigger: statistics.trigger,
-          count: 1,
-          user,
-          role
+      let result = { object: null, text: "" };
+      let tokens = 0;
+      if (outputSchema) {
+        const { object, usage } = await (0, import_ai.generateObject)({
+          model,
+          system,
+          prompt,
+          maxRetries: 3,
+          schema: outputSchema
+        });
+        result.object = object;
+        tokens = usage.totalTokens || 0;
+      } else {
+        console.log("[EXULU] Generating text for agent: " + this.name, "with prompt: " + prompt?.slice(0, 100) + "...");
+        const { text, totalUsage } = await (0, import_ai.generateText)({
+          model,
+          system,
+          prompt,
+          maxRetries: 2,
+          tools: convertToolsArrayToObject(
+            currentTools,
+            allExuluTools,
+            toolConfigs,
+            providerapikey,
+            contexts,
+            user,
+            exuluConfig,
+            filesContext2
+          ),
+          stopWhen: [(0, import_ai.stepCountIs)(2)]
         });
+        result.text = text;
+        tokens = totalUsage?.totalTokens || 0;
       }
-      return text;
+      if (statistics) {
+        await Promise.all([
+          updateStatistic({
+            name: "count",
+            label: statistics.label,
+            type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+            trigger: statistics.trigger,
+            count: 1,
+            user: user?.id,
+            role: user?.role?.id
+          }),
+          ...tokens ? [
+            updateStatistic({
+              name: "tokens",
+              label: statistics.label,
+              type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+              trigger: statistics.trigger,
+              count: tokens
+            })
+          ] : []
+        ]);
+      }
+      return result.text || result.object;
     }
     if (messages) {
-      const { text } = await (0, import_ai.generateText)({
+      console.log("[EXULU] Generating text for agent: " + this.name, "with messages: " + messages.length);
+      const { text, totalUsage } = await (0, import_ai.generateText)({
         model,
         // Should be a LanguageModelV1
-        system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
-        messages: (0, import_ai.convertToModelMessages)(messages),
+        system,
+        messages: (0, import_ai.convertToModelMessages)(messages, {
+          ignoreIncompleteToolCalls: true
+        }),
         maxRetries: 2,
-        tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
-        stopWhen: [(0, import_ai.stepCountIs)(5)]
+        tools: convertToolsArrayToObject(
+          currentTools,
+          allExuluTools,
+          toolConfigs,
+          providerapikey,
+          contexts,
+          user,
+          exuluConfig,
+          filesContext2
+        ),
+        stopWhen: [(0, import_ai.stepCountIs)(2)]
       });
       if (statistics) {
-        await updateStatistic({
-          name: "count",
-          label: statistics.label,
-          type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-          trigger: statistics.trigger,
-          count: 1,
-          user,
-          role
-        });
+        await Promise.all([
+          updateStatistic({
+            name: "count",
+            label: statistics.label,
+            type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+            trigger: statistics.trigger,
+            count: 1,
+            user: user?.id,
+            role: user?.role?.id
+          }),
+          ...totalUsage?.totalTokens ? [
+            updateStatistic({
+              name: "tokens",
+              label: statistics.label,
+              type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+              trigger: statistics.trigger,
+              count: totalUsage?.totalTokens,
+              user: user?.id,
+              role: user?.role?.id
+            })
+          ] : []
+        ]);
       }
       return text;
     }
+    return "";
   };
-  generateStream = async ({ express: express3, user, role, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
+  generateStream = async ({
+    express: express3,
+    user,
+    session,
+    message,
+    currentTools,
+    allExuluTools,
+    statistics,
+    toolConfigs,
+    providerapikey,
+    contexts,
+    exuluConfig,
+    filesContext: filesContext2,
+    instructions
+  }) => {
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
@@ -3410,60 +3892,105 @@ var ExuluAgent2 = class {
       throw new Error("Message is required for streaming.");
     }
     const model = this.model.create({
-      apiKey: providerApiKey
+      apiKey: providerapikey
     });
     let messages = [];
     const previousMessages = await getAgentMessages({
       session,
-      user,
+      user: user.id,
       limit: 50,
       page: 1
     });
-    const previousMessagesContent = previousMessages.map((message2) => JSON.parse(message2.content));
+    const previousMessagesContent = previousMessages.map(
+      (message2) => JSON.parse(message2.content)
+    );
     messages = await (0, import_ai.validateUIMessages)({
       // append the new message to the previous messages:
       messages: [...previousMessagesContent, message]
     });
+    const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
+    let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
+    system += "\n\n" + genericContext;
+    console.log("[EXULU] tools for agent: " + this.name, currentTools?.map((x) => x.name + " (" + x.id + ")"));
+    console.log("[EXULU] system", system.slice(0, 100) + "...");
     const result = (0, import_ai.streamText)({
       model,
       // Should be a LanguageModelV1
-      messages: (0, import_ai.convertToModelMessages)(messages),
-      system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
+      messages: (0, import_ai.convertToModelMessages)(messages, {
+        ignoreIncompleteToolCalls: true
+      }),
+      // prepareStep could be used here to set the model for the first step or change other params
+      system,
       maxRetries: 2,
-      tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
-      onError: (error) => console.error("[EXULU] chat stream error.", error),
-      stopWhen: [(0, import_ai.stepCountIs)(5)]
+      tools: convertToolsArrayToObject(
+        currentTools,
+        allExuluTools,
+        toolConfigs,
+        providerapikey,
+        contexts,
+        user,
+        exuluConfig,
+        filesContext2
+      ),
+      onError: (error) => console.error("[EXULU] chat stream error.", error)
+      // stopWhen: [stepCountIs(1)],
     });
     result.consumeStream();
     result.pipeUIMessageStreamToResponse(express3.res, {
+      messageMetadata: ({ part }) => {
+        if (part.type === "finish") {
+          return {
+            totalTokens: part.totalUsage.totalTokens,
+            reasoningTokens: part.totalUsage.reasoningTokens,
+            inputTokens: part.totalUsage.inputTokens,
+            outputTokens: part.totalUsage.outputTokens,
+            cachedInputTokens: part.totalUsage.cachedInputTokens
+          };
+        }
+      },
       originalMessages: messages,
       sendReasoning: true,
+      sendSources: true,
+      onError: (error) => {
+        console.error("[EXULU] chat response error.", error);
+        return errorHandler(error);
+      },
       generateMessageId: (0, import_ai.createIdGenerator)({
         prefix: "msg_",
         size: 16
       }),
-      onFinish: async ({ messages: messages2 }) => {
-        console.info(
-          "[EXULU] chat stream finished.",
-          messages2
-        );
+      onFinish: async ({ messages: messages2, isContinuation, isAborted, responseMessage }) => {
         if (session) {
           await saveChat({
             session,
-            user,
-            messages: messages2
+            user: user.id,
+            messages: messages2.filter((x) => !previousMessagesContent.find((y) => y.id === x.id))
           });
         }
+        const metadata = messages2[messages2.length - 1]?.metadata;
         if (statistics) {
-          await updateStatistic({
-            name: "count",
-            label: statistics.label,
-            type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-            trigger: statistics.trigger,
-            count: 1,
-            user,
-            role
-          });
+          await Promise.all([
+            updateStatistic({
+              name: "count",
+              label: statistics.label,
+              type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+              trigger: statistics.trigger,
+              count: 1,
+              user: user.id,
+              role: user?.role?.id
+            }),
+            ...metadata?.totalTokens ? [
+              updateStatistic({
+                name: "tokens",
+                label: statistics.label,
+                type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+                trigger: statistics.trigger,
+                count: metadata?.totalTokens,
+                user: user.id,
+                role: user?.role?.id
+              })
+            ] : []
+          ]);
         }
       }
     });
@@ -3472,7 +3999,12 @@ var ExuluAgent2 = class {
 };
 var getAgentMessages = async ({ session, user, limit, page }) => {
   const { db: db3 } = await postgresClient();
-  const messages = await db3.from("agent_messages").where({ session, user }).limit(limit).offset(page * limit);
+  console.log("[EXULU] getting agent messages for session: " + session + " and user: " + user + " and page: " + page);
+  const query = db3.from("agent_messages").where({ session, user }).limit(limit);
+  if (page > 0) {
+    query.offset((page - 1) * limit);
+  }
+  const messages = await query;
   return messages;
 };
 var saveChat = async ({ session, user, messages }) => {
@@ -3582,31 +4114,31 @@ var ExuluEval = class {
                 throw new Error("Prompt is required for running an agent.");
               }
               const { db: db4 } = await postgresClient();
-              const variableName = runner.agent.providerApiKey;
+              const variableName = runner.agent.providerapikey;
               const variable = await db4.from("variables").where({ name: variableName }).first();
               if (!variable) {
-                throw new Error(`Provider API key for variable "${runner.agent.providerApiKey}" not found.`);
+                throw new Error(`Provider API key for variable "${runner.agent.providerapikey}" not found.`);
               }
-              let providerApiKey = variable.value;
+              let providerapikey = variable.value;
               if (!variable.encrypted) {
-                throw new Error(`Provider API key for variable "${runner.agent.providerApiKey}" is not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys.`);
+                throw new Error(`Provider API key for variable "${runner.agent.providerapikey}" is not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys.`);
               }
               if (variable.encrypted) {
                 const bytes = import_crypto_js2.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-                providerApiKey = bytes.toString(import_crypto_js2.default.enc.Utf8);
+                providerapikey = bytes.toString(import_crypto_js2.default.enc.Utf8);
               }
               const result = await runner.agent.generateSync({
                 prompt: data.prompt,
-                providerApiKey
+                providerapikey
               });
               data.result = result;
             }
             const { object } = await (0, import_ai.generateObject)({
               model,
               maxRetries: 3,
-              schema: import_zod2.z.object({
-                correctnessScore: import_zod2.z.number(),
-                comment: import_zod2.z.string()
+              schema: import_zod.z.object({
+                correctnessScore: import_zod.z.number(),
+                comment: import_zod.z.string()
               }),
               prompt: `You are checking if the below "actual_answers" contain the correct information as
                             presented in the "correct_answers" section to calculate the correctness score.
@@ -3677,7 +4209,7 @@ var ExuluTool2 = class {
     this.type = type;
     this.tool = (0, import_ai.tool)({
       description,
-      inputSchema: inputSchema || import_zod2.z.object({}),
+      inputSchema: inputSchema || import_zod.z.object({}),
       execute: execute2
     });
   }
@@ -3756,10 +4288,6 @@ var ExuluContext = class {
       label: statistics?.label || this.name,
       trigger: statistics?.trigger || "agent"
     }, user, role);
-    const exists = await db3.schema.hasTable(getChunksTableName(this.id));
-    if (!exists) {
-      await this.createChunksTable();
-    }
     await db3.from(getChunksTableName(this.id)).where({ source }).delete();
     await db3.from(getChunksTableName(this.id)).insert(chunks.map((chunk) => ({
       source,
@@ -3776,26 +4304,120 @@ var ExuluContext = class {
       job
     };
   };
-  embeddings = {
-    generate: {
-      one: async ({
-        item,
+  createItem = async (item, user, role, upsert) => {
+    const { db: db3 } = await postgresClient();
+    const mutation = db3.from(getTableName(
+      this.id
+    )).insert(
+      {
+        ...item,
+        tags: item.tags ? Array.isArray(item.tags) ? item.tags.join(",") : item.tags : void 0
+      }
+    ).returning("id");
+    if (upsert) {
+      mutation.onConflict().merge();
+    }
+    const results = await mutation;
+    if (!results[0]) {
+      throw new Error("Failed to create item.");
+    }
+    if (this.embedder && (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always")) {
+      const { job } = await this.embeddings.generate.one({
+        item: results[0],
         user,
         role,
-        trigger
-      }) => {
-        console.log("[EXULU] Generating embeddings for item", item.id);
-        if (!this.embedder) {
-          throw new Error("Embedder is not set for this context.");
-        }
-        if (!item.id) {
-          throw new Error("Item id is required for generating embeddings.");
-        }
-        if (this.embedder.queue?.name) {
-          console.log("[EXULU] embedder is in queue mode, scheduling job.");
-          const job = await bullmqDecorator({
-            label: `${this.embedder.name}`,
-            embedder: this.embedder.id,
+        trigger: "api"
+      });
+      return {
+        item: results[0],
+        job
+      };
+    }
+    return {
+      item: results[0],
+      job: void 0
+    };
+  };
+  updateItem = async (item, user, role) => {
+    const { db: db3 } = await postgresClient();
+    const record = await db3.from(
+      getTableName(this.id)
+    ).where(
+      { id: item.id }
+    ).first();
+    if (!record) {
+      throw new Error("Item not found.");
+    }
+    const mutation = db3.from(
+      getTableName(this.id)
+    ).where(
+      { id: record.id }
+    ).update(
+      {
+        ...item,
+        tags: item.tags ? Array.isArray(item.tags) ? item.tags.join(",") : item.tags : void 0
+      }
+    ).returning("id");
+    await mutation;
+    if (this.embedder && (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always")) {
+      const { job } = await this.embeddings.generate.one({
+        item: record,
+        // important we need to full record here with all fields
+        user,
+        role,
+        trigger: "api"
+      });
+      return {
+        item: record,
+        job
+      };
+    }
+    return {
+      item: record,
+      job: void 0
+    };
+  };
+  deleteItem = async (item, user, role) => {
+    if (!item.id) {
+      throw new Error("Item id is required for deleting item.");
+    }
+    const { db: db3 } = await postgresClient();
+    await db3.from(getTableName(this.id)).where({ id: item.id }).delete();
+    if (!this.embedder) {
+      return {
+        id: item.id,
+        job: void 0
+      };
+    }
+    const chunks = await db3.from(getChunksTableName(this.id)).where({ source: item.id }).select("id");
+    if (chunks.length > 0) {
+      await db3.from(getChunksTableName(this.id)).where({ source: item.id }).delete();
+    }
+    return {
+      id: item.id,
+      job: void 0
+    };
+  };
+  embeddings = {
+    generate: {
+      one: async ({
+        item,
+        user,
+        role,
+        trigger
+      }) => {
+        console.log("[EXULU] Generating embeddings for item", item.id);
+        if (!this.embedder) {
+          throw new Error("Embedder is not set for this context.");
+        }
+        if (!item.id) {
+          throw new Error("Item id is required for generating embeddings.");
+        }
+        if (this.embedder.queue?.name) {
+          console.log("[EXULU] embedder is in queue mode, scheduling job.");
+          const job = await bullmqDecorator({
+            label: `${this.embedder.name}`,
+            embedder: this.embedder.id,
             context: this.id,
             inputs: item,
             item: item.id,
@@ -3843,301 +4465,6 @@ var ExuluContext = class {
       }
     }
   };
-  getItems = async ({
-    statistics,
-    limit,
-    sort,
-    order,
-    page,
-    name,
-    user,
-    role,
-    archived,
-    query,
-    method
-  }) => {
-    if (!query && limit > 500) {
-      throw new Error("Limit cannot be greater than 500.");
-    }
-    if (query && limit > 50) {
-      throw new Error("Limit cannot be greater than 50 when using a vector search query.");
-    }
-    if (page < 1) page = 1;
-    if (limit < 1) limit = 10;
-    let offset = (page - 1) * limit;
-    const mainTable = getTableName(this.id);
-    const { db: db3 } = await postgresClient();
-    const columns = await db3(mainTable).columnInfo();
-    const totalQuery = db3.count("* as count").from(mainTable).first();
-    const itemsQuery = db3.select(Object.keys(columns).map((column) => mainTable + "." + column)).from(mainTable).offset(offset).limit(limit);
-    if (sort) {
-      itemsQuery.orderBy(sort, order === "desc" ? "desc" : "asc");
-    }
-    if (typeof name === "string") {
-      itemsQuery.whereILike("name", `%${name}%`);
-      totalQuery.whereILike("name", `%${name}%`);
-    }
-    if (typeof archived === "boolean") {
-      itemsQuery.where("archived", archived);
-      totalQuery.where("archived", archived);
-    }
-    if (!query) {
-      const total = await totalQuery;
-      let items = await itemsQuery;
-      const last = Math.ceil(total.count / limit);
-      return {
-        pagination: {
-          totalCount: parseInt(total.count),
-          currentPage: page,
-          limit,
-          from: offset,
-          pageCount: last || 1,
-          to: offset + items.length,
-          lastPage: last || 1,
-          nextPage: page + 1 > last ? null : page + 1,
-          previousPage: page - 1 || null
-        },
-        filters: {
-          archived,
-          name,
-          query
-        },
-        context: {
-          name: this.name,
-          id: this.id,
-          embedder: this.embedder?.name || void 0
-        },
-        items
-      };
-    }
-    if (typeof query === "string" && this.embedder) {
-      if (!method) {
-        method = "cosineDistance";
-      }
-      itemsQuery.limit(limit * 5);
-      if (statistics) {
-        await updateStatistic({
-          name: "count",
-          label: statistics.label,
-          type: STATISTICS_TYPE_ENUM.CONTEXT_RETRIEVE,
-          trigger: statistics.trigger,
-          user,
-          role
-        });
-      }
-      if (this.queryRewriter) {
-        query = await this.queryRewriter(query);
-      }
-      const chunksTable = getChunksTableName(this.id);
-      itemsQuery.leftJoin(chunksTable, function() {
-        this.on(chunksTable + ".source", "=", mainTable + ".id");
-      });
-      itemsQuery.select(chunksTable + ".id as chunk_id");
-      itemsQuery.select(chunksTable + ".source");
-      itemsQuery.select(chunksTable + ".content");
-      itemsQuery.select(chunksTable + ".chunk_index");
-      itemsQuery.select(chunksTable + ".created_at as chunk_created_at");
-      itemsQuery.select(chunksTable + ".updated_at as chunk_updated_at");
-      const { chunks } = await this.embedder.generateFromQuery(query, {
-        label: this.name,
-        trigger: "agent"
-      }, user, role);
-      if (!chunks?.[0]?.vector) {
-        throw new Error("No vector generated for query.");
-      }
-      const vector = chunks[0].vector;
-      const vectorStr = `ARRAY[${vector.join(",")}]`;
-      const vectorExpr = `${vectorStr}::vector`;
-      const language = this.configuration.language || "english";
-      let items = [];
-      switch (method) {
-        case "tsvector":
-          itemsQuery.select(db3.raw(
-            `ts_rank(${chunksTable}.fts, websearch_to_tsquery(?, ?)) as fts_rank`,
-            [language, query]
-          )).whereRaw(
-            `${chunksTable}.fts @@ websearch_to_tsquery(?, ?)`,
-            [language, query]
-          ).orderByRaw(`fts_rank DESC`);
-          items = await itemsQuery;
-          break;
-        case "cosineDistance":
-        default:
-          itemsQuery.whereNotNull(`${chunksTable}.embedding`);
-          itemsQuery.select(
-            db3.raw(`1 - (${chunksTable}.embedding <=> ${vectorExpr}) AS cosine_distance`)
-          );
-          itemsQuery.orderByRaw(
-            `${chunksTable}.embedding <=> ${vectorExpr} ASC NULLS LAST`
-          );
-          items = await itemsQuery;
-          break;
-        case "hybridSearch":
-          const matchCount = Math.min(limit * 5, 30);
-          const fullTextWeight = 1;
-          const semanticWeight = 1;
-          const rrfK = 50;
-          const hybridSQL = `
-                    WITH full_text AS (
-                      SELECT
-                        c.id,
-                        c.source,
-                        row_number() OVER (
-                          ORDER BY ts_rank_cd(c.fts, websearch_to_tsquery(?, ?)) DESC
-                        ) AS rank_ix
-                      FROM ${chunksTable} c
-                      WHERE c.fts @@ websearch_to_tsquery(?, ?)
-                      ORDER BY rank_ix
-                      LIMIT LEAST(?, 30) * 2
-                    ),
-                    semantic AS (
-                      SELECT
-                        c.id,
-                        c.source,
-                        row_number() OVER (
-                          ORDER BY c.embedding <=> ${vectorExpr} ASC
-                        ) AS rank_ix
-                      FROM ${chunksTable} c
-                      WHERE c.embedding IS NOT NULL
-                      ORDER BY rank_ix
-                      LIMIT LEAST(?, 30) * 2
-                    )
-                    SELECT
-                      m.*,
-                      c.id AS chunk_id,
-                      c.source,
-                      c.content,
-                      c.chunk_index,
-                      c.created_at AS chunk_created_at,
-                      c.updated_at AS chunk_updated_at,
-                
-                      /* Per-signal scores for introspection */
-                      ts_rank(c.fts, websearch_to_tsquery(?, ?)) AS fts_rank,
-                      (1 - (c.embedding <=> ${vectorExpr})) AS cosine_distance,
-                
-                      /* Hybrid RRF score */
-                      (
-                        COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
-                        +
-                        COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
-                      )::float AS hybrid_score
-                
-                    FROM full_text ft
-                    FULL OUTER JOIN semantic se
-                      ON ft.id = se.id
-                    JOIN ${chunksTable} c
-                      ON COALESCE(ft.id, se.id) = c.id
-                    JOIN ${mainTable} m
-                      ON m.id = c.source
-                    ORDER BY hybrid_score DESC
-                    LIMIT LEAST(?, 30)
-                    OFFSET 0
-                  `;
-          const bindings = [
-            // full_text: websearch_to_tsquery(lang, query) in rank and where
-            language,
-            query,
-            language,
-            query,
-            matchCount,
-            // full_text limit
-            matchCount,
-            // semantic limit
-            // fts_rank (ts_rank) call
-            language,
-            query,
-            // RRF fusion parameters
-            rrfK,
-            fullTextWeight,
-            rrfK,
-            semanticWeight,
-            matchCount
-            // final limit
-          ];
-          items = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
-      }
-      console.log("items", items);
-      const seenSources = /* @__PURE__ */ new Map();
-      items = items.reduce((acc, item) => {
-        if (!seenSources.has(item.source)) {
-          seenSources.set(item.source, {
-            ...Object.fromEntries(
-              Object.keys(item).filter(
-                (key) => key !== "cosine_distance" && // kept per chunk below
-                key !== "fts_rank" && // kept per chunk below
-                key !== "hybrid_score" && // we will compute per item below
-                key !== "content" && key !== "source" && key !== "chunk_index" && key !== "chunk_id" && key !== "chunk_created_at" && key !== "chunk_updated_at"
-              ).map((key) => [key, item[key]])
-            ),
-            chunks: [{
-              content: item.content,
-              chunk_index: item.chunk_index,
-              ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
-              ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
-              ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
-            }]
-          });
-          acc.push(seenSources.get(item.source));
-        } else {
-          seenSources.get(item.source).chunks.push({
-            content: item.content,
-            chunk_index: item.chunk_index,
-            ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
-            ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
-            ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
-          });
-        }
-        return acc;
-      }, []);
-      console.log("items", items);
-      items.forEach((item) => {
-        if (!item.chunks?.length) {
-          return;
-        }
-        if (method === "tsvector") {
-          const ranks = item.chunks.map((c) => typeof c.fts_rank === "number" ? c.fts_rank : 0);
-          const total = ranks.reduce((a, b) => a + b, 0);
-          const average = ranks.length ? total / ranks.length : 0;
-          item.averageRelevance = average;
-          item.totalRelevance = total;
-        } else if (method === "cosineDistance") {
-          let methodProperty = "cosine_distance";
-          const average = item.chunks.reduce((acc, item2) => {
-            return acc + item2[methodProperty];
-          }, 0) / item.chunks.length;
-          const total = item.chunks.reduce((acc, item2) => {
-            return acc + item2[methodProperty];
-          }, 0);
-          item.averageRelevance = average;
-          item.totalRelevance = total;
-        } else if (method === "hybridSearch") {
-          console.log("item.chunks", item.chunks);
-          const scores = item.chunks.map((c) => typeof c.hybrid_score === "number" ? c.hybrid_score * 10 + 1 : 0);
-          const total = scores.reduce((a, b) => a + b, 0);
-          const average = scores.length ? total / scores.length : 0;
-          item.averageRelevance = average;
-          item.totalRelevance = total;
-        }
-      });
-      if (this.resultReranker && query) {
-        items = await this.resultReranker(items);
-      }
-      items = items.slice(0, limit);
-      return {
-        filters: {
-          archived,
-          name,
-          query
-        },
-        context: {
-          name: this.name,
-          id: this.id,
-          embedder: this.embedder.name
-        },
-        items
-      };
-    }
-  };
   createItemsTable = async () => {
     const { db: db3 } = await postgresClient();
     const tableName = getTableName(this.id);
@@ -4151,6 +4478,7 @@ var ExuluContext = class {
       table.boolean("archived").defaultTo(false);
       table.text("external_id");
       table.text("created_by");
+      table.text("ttl");
       table.text("rights_mode").defaultTo(this.configuration?.defaultRightsMode ?? "private");
       table.integer("textlength");
       table.text("source");
@@ -4168,7 +4496,7 @@ var ExuluContext = class {
     });
   };
   createChunksTable = async () => {
-    const { db: db3 } = await postgresClient();
+    const { db: db3 } = await refreshPostgresClient();
     const tableName = getChunksTableName(this.id);
     console.log("[EXULU] Creating table: " + tableName);
     await db3.schema.createTable(tableName, (table) => {
@@ -4204,14 +4532,14 @@ var ExuluContext = class {
       id: this.id,
       name: `${this.name}`,
       type: "context",
-      inputSchema: import_zod2.z.object({
-        query: import_zod2.z.string()
+      inputSchema: import_zod.z.object({
+        query: import_zod.z.string()
       }),
       config: [],
       description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
       execute: async ({ query, user, role }) => {
         const { db: db3 } = await postgresClient();
-        await vectorSearch({
+        const result = await vectorSearch({
           page: 1,
           limit: 10,
           query,
@@ -4224,6 +4552,9 @@ var ExuluContext = class {
           sort: void 0,
           trigger: "agent"
         });
+        return {
+          items: result.items
+        };
       }
     });
   };
@@ -4239,7 +4570,6 @@ var updateStatistic = async (statistic) => {
     type: statistic.type,
     createdAt: currentDate
   }).first();
-  console.log("!!! existing !!!", existing);
   if (!existing) {
     await db3.from("tracking").insert({
       name: statistic.name,
@@ -4261,6 +4591,53 @@ var updateStatistic = async (statistic) => {
     });
   }
 };
+var generateS3Key = (filename) => `${(0, import_node_crypto.randomUUID)()}-${filename}`;
+var getMimeType = (type) => {
+  switch (type) {
+    case ".png":
+      return "image/png";
+    case ".jpg":
+      return "image/jpg";
+    case ".jpeg":
+      return "image/jpeg";
+    case ".gif":
+      return "image/gif";
+    case ".webp":
+      return "image/webp";
+    case ".pdf":
+      return "application/pdf";
+    case ".docx":
+      return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+    case ".xlsx":
+      return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+    case ".xls":
+      return "application/vnd.ms-excel";
+    case ".csv":
+      return "text/csv";
+    case ".pptx":
+      return "application/vnd.openxmlformats-officedocument.presentationml.presentation";
+    case ".ppt":
+      return "application/vnd.ms-powerpoint";
+    case ".m4a":
+      return "audio/mp4";
+    case ".mp4":
+      return "audio/mp4";
+    case ".mpeg":
+      return "audio/mpeg";
+    case ".mp3":
+      return "audio/mp3";
+    case ".wav":
+      return "audio/wav";
+    case ".txt":
+      return "text/plain";
+    case ".md":
+      return "text/markdown";
+    case ".json":
+      return "application/json";
+    default:
+      return "";
+  }
+};
 
 // src/registry/index.ts
 var import_express7 = require("express");
@@ -4268,42 +4645,6 @@ var import_express7 = require("express");
 // src/registry/routes.ts
 var import_express3 = require("express");
 
-// src/registry/rate-limiter.ts
-var rateLimiter = async (key, windowSeconds, limit, points) => {
-  try {
-    const { client: client2 } = await redisClient();
-    if (!client2) {
-      console.warn("[EXULU] Rate limiting disabled - Redis not available");
-      return {
-        status: true,
-        retryAfter: null
-      };
-    }
-    const redisKey = `exulu/${key}`;
-    const current = await client2.incrBy(redisKey, points);
-    if (current === points) {
-      await client2.expire(redisKey, windowSeconds);
-    }
-    if (current > limit) {
-      const ttl = await client2.ttl(redisKey);
-      return {
-        status: false,
-        retryAfter: ttl
-      };
-    }
-    return {
-      status: true,
-      retryAfter: null
-    };
-  } catch (error) {
-    console.error("[EXULU] Rate limiting error:", error);
-    return {
-      status: true,
-      retryAfter: null
-    };
-  }
-};
-
 // src/bullmq/queues.ts
 var import_bullmq4 = require("bullmq");
 var import_bullmq_otel = require("bullmq-otel");
@@ -4346,10 +4687,10 @@ var import_express5 = require("@as-integrations/express5");
 
 // src/registry/uppy.ts
 var import_express2 = require("express");
-var import_client_s3 = require("@aws-sdk/client-s3");
+var import_client_s32 = require("@aws-sdk/client-s3");
 var import_s3_request_presigner = require("@aws-sdk/s3-request-presigner");
 var import_client_sts = require("@aws-sdk/client-sts");
-var import_node_crypto = require("crypto");
+var import_node_crypto2 = require("crypto");
 var createUppyRoutes = async (app, config) => {
   const policy = {
     Version: "2012-10-17",
@@ -4366,11 +4707,11 @@ var createUppyRoutes = async (app, config) => {
       }
     ]
   };
-  let s3Client;
+  let s3Client2;
   let stsClient;
   const expiresIn = 60 * 60 * 24 * 1;
   function getS3Client() {
-    s3Client ??= new import_client_s3.S3Client({
+    s3Client2 ??= new import_client_s32.S3Client({
       region: config.fileUploads.s3region,
       ...config.fileUploads.s3endpoint && {
         forcePathStyle: true,
@@ -4381,7 +4722,7 @@ var createUppyRoutes = async (app, config) => {
         secretAccessKey: config.fileUploads.s3secret
       }
     });
-    return s3Client;
+    return s3Client2;
   }
   function getSTSClient() {
     stsClient ??= new import_client_sts.STSClient({
@@ -4394,54 +4735,51 @@ var createUppyRoutes = async (app, config) => {
     });
     return stsClient;
   }
-  app.get("/s3/list", async (req, res, next) => {
-    req.accepts;
+  app.delete("/s3/delete", async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
+    const internalkey = req.headers["internal-key"] || null;
+    const { db: db3 } = await postgresClient();
     let authtoken = null;
-    if (typeof apikey !== "string") {
+    if (typeof apikey !== "string" && typeof internalkey !== "string") {
       authtoken = await getToken(req.headers.authorization ?? "");
     }
-    const { db: db3 } = await postgresClient();
     const authenticationResult = await authentication({
       authtoken,
       apikey,
+      internalkey,
       db: db3
     });
     if (!authenticationResult.user?.id) {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
-    const { prefix = "" } = req.query;
-    if (typeof prefix !== "string") {
-      res.status(400).json({ error: "Invalid prefix parameter. Must be a string." });
+    const { key } = req.query;
+    if (typeof key !== "string" || key.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
-    if (authenticationResult.user.type !== "api" && !prefix.includes(authenticationResult.user.id)) {
-      res.status(405).json({ error: "Not allowed to list files in this folder based on authenticated user." });
+    const userPrefix = key.split("/")[0];
+    console.log("userPrefix", userPrefix);
+    console.log("authenticationResult.user.id", authenticationResult.user.id);
+    if (!userPrefix) {
+      res.status(405).json({ error: 'Invalid key, does not contain a user prefix like "<user_id>/<key>.' });
       return;
     }
-    try {
-      const command = new import_client_s3.ListObjectsV2Command({
-        Bucket: config.fileUploads.s3Bucket,
-        Prefix: prefix,
-        MaxKeys: 1e3
-        // Adjust this value based on your needs
-      });
-      const data = await getS3Client().send(command);
-      const files = data.Contents?.map((item) => ({
-        key: item.Key,
-        size: item.Size,
-        lastModified: item.LastModified
-      })) || [];
-      res.setHeader("Access-Control-Allow-Origin", "*");
-      res.status(200).json({
-        files,
-        isTruncated: data.IsTruncated,
-        nextContinuationToken: data.NextContinuationToken
-      });
-    } catch (err) {
-      next(err);
+    if (userPrefix !== authenticationResult.user.id.toString()) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
+      return;
     }
+    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id.toString())) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
+      return;
+    }
+    const client2 = getS3Client();
+    const command = new import_client_s32.DeleteObjectCommand({
+      Bucket: config.fileUploads.s3Bucket,
+      Key: key
+    });
+    await client2.send(command);
+    res.json({ key });
   });
   app.get("/s3/download", async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
@@ -4466,14 +4804,23 @@ var createUppyRoutes = async (app, config) => {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
-    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id)) {
+    const userPrefix = key.split("/")[0];
+    if (!userPrefix) {
+      res.status(405).json({ error: 'Invalid key, does not contain a user prefix like "<user_id>/<key>.' });
+      return;
+    }
+    if (userPrefix !== authenticationResult.user.id.toString()) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
+      return;
+    }
+    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id.toString())) {
       res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
       return;
     }
     try {
       const url = await (0, import_s3_request_presigner.getSignedUrl)(
         getS3Client(),
-        new import_client_s3.GetObjectCommand({
+        new import_client_s32.GetObjectCommand({
           Bucket: config.fileUploads.s3Bucket,
           Key: key
         }),
@@ -4516,7 +4863,7 @@ var createUppyRoutes = async (app, config) => {
       contentType: params.type
     };
   };
-  const generateS3Key = (filename) => `${(0, import_node_crypto.randomUUID)()}-${filename}`;
+  const generateS3Key2 = (filename) => `${(0, import_node_crypto2.randomUUID)()}-${filename}`;
   const signOnServer = async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
     const { db: db3 } = await postgresClient();
@@ -4535,16 +4882,11 @@ var createUppyRoutes = async (app, config) => {
     }
     const { filename, contentType } = extractFileParameters(req);
     validateFileParameters(filename, contentType);
-    const key = generateS3Key(filename);
-    let folder = "";
-    if (authenticationResult.user.type === "api") {
-      folder = `api/`;
-    } else {
-      folder = `${authenticationResult.user.id}/`;
-    }
+    const key = generateS3Key2(filename);
+    let folder = `${authenticationResult.user.id}/`;
     (0, import_s3_request_presigner.getSignedUrl)(
       getS3Client(),
-      new import_client_s3.PutObjectCommand({
+      new import_client_s32.PutObjectCommand({
         Bucket: config.fileUploads.s3Bucket,
         Key: folder + key,
         ContentType: contentType
@@ -4590,7 +4932,7 @@ var createUppyRoutes = async (app, config) => {
     if (typeof type !== "string") {
       return res.status(400).json({ error: "s3: content type must be a string" });
     }
-    const key = `${(0, import_node_crypto.randomUUID)()}-${filename}`;
+    const key = `${(0, import_node_crypto2.randomUUID)()}-${filename}`;
     let folder = "";
     if (authenticationResult.user.type === "api") {
       folder = `api/`;
@@ -4603,7 +4945,7 @@ var createUppyRoutes = async (app, config) => {
       ContentType: type,
       Metadata: metadata
     };
-    const command = new import_client_s3.CreateMultipartUploadCommand(params);
+    const command = new import_client_s32.CreateMultipartUploadCommand(params);
     return client2.send(command, (err, data) => {
       if (err) {
         next(err);
@@ -4629,7 +4971,7 @@ var createUppyRoutes = async (app, config) => {
     if (typeof key !== "string") {
       return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' });
     }
-    return (0, import_s3_request_presigner.getSignedUrl)(getS3Client(), new import_client_s3.UploadPartCommand({
+    return (0, import_s3_request_presigner.getSignedUrl)(getS3Client(), new import_client_s32.UploadPartCommand({
       Bucket: config.fileUploads.s3Bucket,
       Key: key,
       UploadId: uploadId,
@@ -4650,7 +4992,7 @@ var createUppyRoutes = async (app, config) => {
     }
     const parts = [];
     function listPartsPage(startAt) {
-      client2.send(new import_client_s3.ListPartsCommand({
+      client2.send(new import_client_s32.ListPartsCommand({
         Bucket: config.fileUploads.s3Bucket,
         Key: key,
         UploadId: uploadId,
@@ -4684,7 +5026,7 @@ var createUppyRoutes = async (app, config) => {
     if (!Array.isArray(parts) || !parts.every(isValidPart)) {
       return res.status(400).json({ error: "s3: `parts` must be an array of {ETag, PartNumber} objects." });
     }
-    return client2.send(new import_client_s3.CompleteMultipartUploadCommand({
+    return client2.send(new import_client_s32.CompleteMultipartUploadCommand({
       Bucket: config.fileUploads.s3Bucket,
       Key: key,
       UploadId: uploadId,
@@ -4710,7 +5052,7 @@ var createUppyRoutes = async (app, config) => {
     if (typeof key !== "string") {
       return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' });
     }
-    return client2.send(new import_client_s3.AbortMultipartUploadCommand({
+    return client2.send(new import_client_s32.AbortMultipartUploadCommand({
       Bucket: config.fileUploads.s3Bucket,
       Key: key,
       UploadId: uploadId
@@ -4728,7 +5070,7 @@ var createUppyRoutes = async (app, config) => {
 };
 
 // src/registry/routes.ts
-var import_utils2 = require("@apollo/utils.keyvaluecache");
+var import_utils4 = require("@apollo/utils.keyvaluecache");
 var import_body_parser = __toESM(require("body-parser"), 1);
 var import_crypto_js3 = __toESM(require("crypto-js"), 1);
 
@@ -4757,8 +5099,11 @@ var CLAUDE_MESSAGES = {
 // src/registry/routes.ts
 var import_openai = __toESM(require("openai"), 1);
 var import_fs = __toESM(require("fs"), 1);
-var import_node_crypto2 = require("crypto");
-var import_api2 = require("@opentelemetry/api");
+var import_node_crypto3 = require("crypto");
+var import_api = require("@opentelemetry/api");
+var import_ai2 = require("ai");
+var import_zod2 = require("zod");
+var import_client_s33 = require("@aws-sdk/client-s3");
 var REQUEST_SIZE_LIMIT = "50mb";
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -4808,8 +5153,7 @@ var createRecurringJobs = async () => {
   );
   return queue;
 };
-var createExpressRoutes = async (app, logger, agents, tools, contexts, config, tracer) => {
-  console.log("============= agents =============", agents?.length);
+var createExpressRoutes = async (app, logger, agents, tools, contexts, config, tracer, filesContext2) => {
   var corsOptions = {
     origin: "*",
     exposedHeaders: "*",
@@ -4834,7 +5178,7 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
   if (redisServer.host?.length && redisServer.port?.length) {
     await createRecurringJobs();
   } else {
-    console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
+    console.log("[EXULU] no redis server configured, not setting up recurring jobs.");
   }
   const schema = createSDL([
     usersSchema2(),
@@ -4849,9 +5193,9 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
     workflowTemplatesSchema2(),
     statisticsSchema2(),
     rbacSchema2()
-  ], contexts, agents, tools);
+  ], contexts ?? [], agents, tools);
   const server = new import_server3.ApolloServer({
-    cache: new import_utils2.InMemoryLRUCache(),
+    cache: new import_utils4.InMemoryLRUCache(),
     schema,
     introspection: true
   });
@@ -4907,7 +5251,7 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
       });
       return;
     }
-    let providerApiKey = variable.value;
+    let providerapikey = variable.value;
     if (!variable.encrypted) {
       res.status(400).json({
         message: "Provider API key variable not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys."
@@ -4916,10 +5260,10 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
     }
     if (variable.encrypted) {
       const bytes = import_crypto_js3.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-      providerApiKey = bytes.toString(import_crypto_js3.default.enc.Utf8);
+      providerapikey = bytes.toString(import_crypto_js3.default.enc.Utf8);
     }
     const openai = new import_openai.default({
-      apiKey: providerApiKey
+      apiKey: providerapikey
     });
     let style_reference = "";
     if (style === "origami") {
@@ -4966,7 +5310,7 @@ Mood: friendly and intelligent.
       return;
     }
     const image_bytes = Buffer.from(image_base64, "base64");
-    const uuid = (0, import_node_crypto2.randomUUID)();
+    const uuid = (0, import_node_crypto3.randomUUID)();
     if (!import_fs.default.existsSync("public")) {
       import_fs.default.mkdirSync("public");
     }
@@ -4992,6 +5336,12 @@ Mood: friendly and intelligent.
     const slug = agent.slug;
     if (!slug) return;
     app.post(slug + "/:instance", async (req, res) => {
+      const headers = {
+        stream: req.headers["stream"] === "true" || false,
+        user: req.headers["user"] || null,
+        session: req.headers["session"] || null
+      };
+      await checkAgentRateLimit(agent);
       const instance = req.params.instance;
       if (!instance) {
         res.status(400).json({
@@ -5000,38 +5350,7 @@ Mood: friendly and intelligent.
         return;
       }
       const { db: db3 } = await postgresClient();
-      const agentInstance = await db3.from("agents").where({
-        id: instance
-      }).first();
-      const agentRbac = await RBACResolver(db3, "agent", agentInstance.id, agentInstance.rights_mode || "private");
-      agentInstance.RBAC = agentRbac;
-      if (!agentInstance) {
-        res.status(400).json({
-          message: "Agent instance not found."
-        });
-        return;
-      }
-      if (agent.rateLimit) {
-        console.log("[EXULU] rate limiting agent.", agent.rateLimit);
-        const limit = await rateLimiter(
-          agent.rateLimit.name || agent.id,
-          agent.rateLimit.rate_limit.time,
-          agent.rateLimit.rate_limit.limit,
-          1
-        );
-        if (!limit.status) {
-          res.status(429).json({
-            message: "Rate limit exceeded.",
-            retryAfter: limit.retryAfter
-          });
-          return;
-        }
-      }
-      const headers = {
-        stream: req.headers["stream"] === "true" || false,
-        user: req.headers["user"] || null,
-        session: req.headers["session"] || null
-      };
+      const agentInstance = await loadAgent(instance);
       const requestValidationResult = requestValidators.agents(req);
       if (requestValidationResult.error) {
         res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
@@ -5043,85 +5362,24 @@ Mood: friendly and intelligent.
         return;
       }
       const user = authenticationResult.user;
-      const agentIsPublic = agentInstance.rights_mode === "public";
-      const agentByUsers = agentInstance.rights_mode === "users";
-      const agentByRoles = agentInstance.rights_mode === "roles";
-      const isAgentCreator = agentInstance.created_by === user.id;
-      const isAdmin = user.super_admin;
-      const isApi = user.type === "api";
-      let hasAccessToAgent = "none";
-      if (agentIsPublic || isAgentCreator || isAdmin || isApi) {
-        hasAccessToAgent = "write";
-      }
-      if (agentByUsers) {
-        hasAccessToAgent = agentInstance.RBAC?.users?.find((x) => x.id === user.id)?.rights || "none";
-        if (!hasAccessToAgent || hasAccessToAgent === "none" || hasAccessToAgent === "read") {
-          res.status(410).json({
-            message: `Your current user ${user.id} does not have access to this agent.`
-          });
-          return;
-        }
-      }
-      if (agentByRoles) {
-        hasAccessToAgent = agentInstance.RBAC?.roles?.find((x) => x.id === user.role?.id)?.rights || "none";
-        if (!hasAccessToAgent || hasAccessToAgent === "none" || hasAccessToAgent === "read") {
-          res.status(410).json({
-            message: `Your current role ${user.role?.name} does not have access to this agent.`
-          });
-          return;
-        }
-      }
-      let hasAccessToSession = "none";
-      ;
-      if (headers.session) {
-        const session = await db3.from("agents").where({
-          id: instance
-        }).first();
-        const sessionIsPublic = agentInstance.rights_mode === "public";
-        const sessionByUsers = agentInstance.rights_mode === "users";
-        const sessionByRoles = agentInstance.rights_mode === "roles";
-        const isSessionCreator = agentInstance.created_by === user.id;
-        const isAdmin2 = user.super_admin;
-        const isApi2 = user.type === "api";
-        if (sessionIsPublic || isSessionCreator || isAdmin2 || isApi2) {
-          hasAccessToSession = "write";
-        }
-        if (sessionByUsers) {
-          hasAccessToSession = session.RBAC?.users?.find((x) => x.id === user.id)?.rights || "none";
-          if (!hasAccessToSession || hasAccessToSession === "none" || hasAccessToSession === "read") {
-            res.status(410).json({
-              message: `Your current user ${user.id} does not have access to this session.`
-            });
-            return;
-          }
-        }
-        if (sessionByRoles) {
-          hasAccessToSession = session.RBAC?.roles?.find((x) => x.id === user.role?.id)?.rights || "none";
-          if (!hasAccessToSession || hasAccessToSession === "none" || hasAccessToSession === "read") {
-            res.status(410).json({
-              message: `Your current role ${user.role?.name} does not have access to this session.`
-            });
-            return;
-          }
-        }
-      }
-      if (!hasAccessToAgent || hasAccessToAgent === "none") {
-        res.status(410).json({
+      const hasAccessToAgent = await checkRecordAccess(agentInstance, "read", user);
+      if (!hasAccessToAgent) {
+        res.status(401).json({
           message: "You don't have access to this agent."
         });
         return;
       }
-      if (!hasAccessToSession || hasAccessToSession === "none") {
-        res.status(410).json({
-          message: "You don't have access to this session."
-        });
-        return;
-      }
-      if (headers.session && !hasAccessToSession) {
-        res.status(410).json({
-          message: "You don't have access to this session."
-        });
-        return;
+      if (headers.session) {
+        const session = await db3.from("agent_sessions").where({
+          id: headers.session
+        }).first();
+        let hasAccessToSession = await checkRecordAccess(session, "write", user);
+        if (!hasAccessToSession) {
+          res.status(401).json({
+            message: "You don't have access to this session."
+          });
+          return;
+        }
       }
       if (user.type !== "api" && !user.super_admin && req.body.resourceId !== user.id) {
         res.status(400).json({
@@ -5129,16 +5387,11 @@ Mood: friendly and intelligent.
         });
         return;
       }
-      console.log("[EXULU] agent tools", agentInstance.tools);
-      let enabledTools = agentInstance.tools ? agentInstance.tools.map(
-        ({ config: config2, toolId }) => tools.find(({ id }) => id === toolId)
-      ).filter(Boolean) : [];
-      console.log("[EXULU] available tools", enabledTools?.length);
+      console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
       const disabledTools = req.body.disabledTools ? req.body.disabledTools : [];
-      console.log("[EXULU] disabled tools", disabledTools?.length);
-      enabledTools = enabledTools.filter((tool2) => !disabledTools.includes(tool2.id));
-      console.log("[EXULU] enabled tools", enabledTools?.length);
-      const variableName = agentInstance.providerApiKey;
+      let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
+      console.log("[EXULU] enabled tools", enabledTools?.map((x) => x.name + " (" + x.id + ")"));
+      const variableName = agentInstance.providerapikey;
       const variable = await db3.from("variables").where({ name: variableName }).first();
       if (!variable) {
         res.status(400).json({
@@ -5146,7 +5399,7 @@ Mood: friendly and intelligent.
         });
         return;
       }
-      let providerApiKey = variable.value;
+      let providerapikey = variable.value;
       if (!variable.encrypted) {
         res.status(400).json({
           message: "Provider API key variable not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys."
@@ -5155,7 +5408,7 @@ Mood: friendly and intelligent.
       }
       if (variable.encrypted) {
         const bytes = import_crypto_js3.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-        providerApiKey = bytes.toString(import_crypto_js3.default.enc.Utf8);
+        providerapikey = bytes.toString(import_crypto_js3.default.enc.Utf8);
       }
       if (!!headers.stream) {
         await agent.generateStream({
@@ -5163,13 +5416,17 @@ Mood: friendly and intelligent.
             res,
             req
           },
-          user: user?.id,
-          role: user?.role?.id,
+          contexts,
+          user,
+          instructions: agentInstance.instructions,
           session: headers.session,
           message: req.body.message,
-          tools: enabledTools,
-          providerApiKey,
+          currentTools: enabledTools,
+          allExuluTools: tools,
+          providerapikey,
           toolConfigs: agentInstance.tools,
+          exuluConfig: config,
+          filesContext: filesContext2,
           statistics: {
             label: agent.name,
             trigger: "agent"
@@ -5178,13 +5435,17 @@ Mood: friendly and intelligent.
         return;
       } else {
         const response = await agent.generateSync({
-          user: user?.id,
+          user,
+          instructions: agentInstance.instructions,
           session: headers.session,
-          role: user?.role?.id,
           message: req.body.message,
-          tools: enabledTools,
-          providerApiKey,
+          contexts,
+          currentTools: enabledTools,
+          allExuluTools: tools,
+          providerapikey,
+          exuluConfig: config,
           toolConfigs: agentInstance.tools,
+          filesContext: filesContext2,
           statistics: {
             label: agent.name,
             trigger: "agent"
@@ -5202,8 +5463,9 @@ Mood: friendly and intelligent.
   }
   const TARGET_API = "https://api.anthropic.com";
   app.use("/gateway/anthropic/:id", import_express4.default.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }), async (req, res) => {
-    const path3 = req.url;
-    const url = `${TARGET_API}${path3}`;
+    console.log("[EXULU] Coding request!!!");
+    const path2 = req.url;
+    const url = `${TARGET_API}${path2}`;
     try {
       if (!req.body.tools) {
         req.body.tools = [];
@@ -5214,6 +5476,7 @@ Mood: friendly and intelligent.
         res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
         return;
       }
+      console.log("[EXULU] Authenticated call", authenticationResult.user?.email);
       const { db: db3 } = await postgresClient();
       let query = db3("agents");
       query.select("*");
@@ -5228,20 +5491,30 @@ Mood: friendly and intelligent.
         res.end(Buffer.from(arrayBuffer));
         return;
       }
-      console.log("[EXULU] anthropic proxy called for agent:", agent?.name);
+      console.log("[EXULU] Agent loaded", agent.name);
+      const backend = agents.find((x) => x.id === agent.backend);
+      if (!backend) {
+        const arrayBuffer = createCustomAnthropicStreamingMessage(`
+                    \x1B[41m -- Agent ${agent.name} does not have a exulu backend setup, or the exulu backend that was assigned no longer exists. --
+                    \x1B[0m`);
+        res.setHeader("Content-Type", "application/json");
+        res.end(Buffer.from(arrayBuffer));
+        return;
+      }
+      console.log("[EXULU] Backend loaded", backend.id);
       if (!process.env.NEXTAUTH_SECRET) {
         const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.missing_nextauth_secret);
         res.setHeader("Content-Type", "application/json");
         res.end(Buffer.from(arrayBuffer));
         return;
       }
-      if (!agent.providerApiKey) {
+      if (!agent.providerapikey) {
         const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.not_enabled);
         res.setHeader("Content-Type", "application/json");
         res.end(Buffer.from(arrayBuffer));
         return;
       }
-      const variableName = agent.providerApiKey;
+      const variableName = agent.providerapikey;
       const variable = await db3.from("variables").where({ name: variableName }).first();
       if (!variable) {
         const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.anthropic_token_variable_not_found);
@@ -5249,7 +5522,7 @@ Mood: friendly and intelligent.
         res.end(Buffer.from(arrayBuffer));
         return;
       }
-      let anthropicApiKey = variable.value;
+      let providerapikey = variable.value;
       if (!variable.encrypted) {
         const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.anthropic_token_variable_not_encrypted);
         res.setHeader("Content-Type", "application/json");
@@ -5258,54 +5531,90 @@ Mood: friendly and intelligent.
       }
       if (variable.encrypted) {
         const bytes = import_crypto_js3.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-        anthropicApiKey = bytes.toString(import_crypto_js3.default.enc.Utf8);
+        providerapikey = bytes.toString(import_crypto_js3.default.enc.Utf8);
       }
       const headers = {
-        "x-api-key": anthropicApiKey,
+        "x-api-key": providerapikey,
         "anthropic-version": "2023-06-01",
         "content-type": req.headers["content-type"] || "application/json"
       };
       if (req.headers["accept"]) headers["accept"] = req.headers["accept"];
       if (req.headers["user-agent"]) headers["user-agent"] = req.headers["user-agent"];
-      const response = await fetch(url, {
-        method: req.method,
-        headers,
-        body: req.method !== "GET" ? JSON.stringify(req.body) : void 0
+      console.log("agent", agent.name);
+      const model = backend.model?.create({
+        apiKey: providerapikey
       });
-      await updateStatistic({
-        name: "count",
-        label: "Claude Code",
-        type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-        trigger: "claude-code",
-        count: 1,
-        user: authenticationResult.user?.id,
-        role: authenticationResult.user.role?.id
-      });
-      response.headers.forEach((value, key) => {
-        res.setHeader(key, value);
-      });
-      res.status(response.status);
-      const isStreaming = response.headers.get("content-type")?.includes("text/event-stream");
-      if (isStreaming && !response?.body) {
-        const arrayBuffer = createCustomAnthropicStreamingMessage(CLAUDE_MESSAGES.missing_body);
+      if (!model) {
+        const arrayBuffer = createCustomAnthropicStreamingMessage(`
+                    \x1B[41m -- Could not create language model instance fro agent ${agent.name}. --
+                    \x1B[0m`);
         res.setHeader("Content-Type", "application/json");
         res.end(Buffer.from(arrayBuffer));
         return;
       }
-      if (isStreaming) {
-        const reader = response.body.getReader();
-        const decoder = new TextDecoder();
-        while (true) {
-          const { done, value } = await reader.read();
-          if (done) break;
-          const chunk = decoder.decode(value, { stream: true });
-          res.write(chunk);
+      const systemMessagesConcatenated = req.body.system.map((x) => x.text).join("\n\n\n");
+      let messages = convertClaudeCodeMessagesToVercelAISdkMessages(
+        req.body.messages
+      );
+      const tools2 = convertClaudeCodeToolsToVercelAISdkTools(
+        req.body.tools
+      );
+      console.log("STREAMING TEXT");
+      const result = (0, import_ai2.streamText)({
+        model,
+        // Should be a LanguageModelV1
+        // TIP FOR DEBUGGING IF YOU RUN INTO ISSUES / ERRORS REGARDING THE MESSAGES FORMAT. STORE THE 'raw' VARIABLE TO A FILE (fs.write)
+        // AND COPY THE CONTENT INTO THE messages: BELOW, TYPESCRIPT WILL TELL YOU WHAT IS WRONG WHICH IS USUALLY EASIER TO READ THAN THE
+        // ERROR OUTPUT IN THE LOGS.
+        messages,
+        system: systemMessagesConcatenated || "",
+        // prepareStep could be used here to set the model for the first step or change other params
+        maxOutputTokens: req.body.max_tokens,
+        temperature: req.body.temperature,
+        maxRetries: 2,
+        providerOptions: {
+          metadata: {
+            user_id: req.body.metadata?.user_id
+          }
+        },
+        tools: tools2,
+        onFinish: (data) => console.log("[EXULU] Finished stream"),
+        onError: (error) => console.error("[EXULU] chat stream error.", error)
+        // stopWhen: [stepCountIs(1)],
+      });
+      let allChunks = [];
+      result.consumeStream();
+      const responses = [];
+      try {
+        for await (const uiMessage of (0, import_ai2.readUIMessageStream)({
+          stream: result.toUIMessageStream()
+        })) {
+          console.log("Streaming chunk:", uiMessage);
+          const message = {
+            type: "message",
+            role: uiMessage.role,
+            content: uiMessage.parts.map((part) => {
+              if (part.type.includes("tool-")) {
+                const type = part.type;
+                part.type = "tool_use";
+                part.name = type.replace("tool-", "");
+                part.id = part.toolCallId;
+              }
+              return part;
+            })
+          };
+          responses.push(message);
+          console.log("Wrote message to response", message);
         }
-        res.end();
+      } catch (err) {
+        console.error("Stream error:", err);
+      } finally {
+        const jsonString = JSON.stringify(responses[responses.length - 1]);
+        const arrayBuffer = new TextEncoder().encode(jsonString).buffer;
+        res.setHeader("Content-Type", "application/json");
+        res.end(Buffer.from(arrayBuffer));
         return;
       }
-      const data = await response.arrayBuffer();
-      res.end(Buffer.from(data));
     } catch (error) {
       console.error("[PROXY] Manual proxy error:", error);
       if (!res.headersSent) {
@@ -5320,6 +5629,97 @@ Mood: friendly and intelligent.
   app.use(import_express4.default.static("public"));
   return app;
 };
+var convertClaudeCodeToolsToVercelAISdkTools = (tools) => {
+  const result = {};
+  for (const tool2 of tools) {
+    const mySchema = (0, import_ai2.jsonSchema)(tool2.input_schema);
+    tools[tool2.name] = {
+      id: tool2.name,
+      name: tool2.name,
+      description: tool2.description,
+      inputSchema: mySchema
+    };
+  }
+  return result;
+};
+var convertClaudeCodeMessagesToVercelAISdkMessages = (messages) => {
+  let raw = messages.map((msg) => {
+    if (!msg.role) {
+      msg.role = "assistant";
+    }
+    delete msg.id;
+    if (!Array.isArray(msg.content)) {
+      return {
+        role: msg.role,
+        content: msg.content
+      };
+    }
+    if (msg.content.some((part) => part.type === "tool_result")) {
+      msg.role = "tool";
+    }
+    let parts = msg.content.map((part) => {
+      if (part.type === "step-start") {
+        return void 0;
+      }
+      if (part.type === "reasoning") {
+        const content = part.text?.length > 1 ? part.text : part.content;
+        return {
+          type: "reasoning",
+          text: content || "No reasoning content provided"
+        };
+      }
+      if (part.type === "tool_use") {
+        part.type = "tool-call";
+      }
+      if (part.type === "tool_result") {
+        part.type = "tool-result";
+        part.output = {
+          type: "text",
+          value: part.text || part.content
+        };
+        part.text = null;
+        part.content = null;
+        if (!part.name && part.tool_use_id) {
+          const allParts = raw.map((x) => x.content).flat();
+          const result = allParts.find((x) => {
+            return x.toolCallId === part.tool_use_id && x.name;
+          });
+          console.log("FIND RESULT!!!!", result);
+          if (result) {
+            part.name = result.name;
+          } else {
+            part.name = "...";
+          }
+        }
+      }
+      if (part.tool_use_id) {
+        part.toolCallId = part.tool_use_id;
+        delete part.tool_use_id;
+      }
+      return {
+        type: part.type,
+        ...part.text || part.content ? { text: part.text || part.content } : {},
+        ...part.toolCallId ? { toolCallId: part.toolCallId } : {},
+        ...part.name ? { toolName: part.name } : {},
+        ...part.input ? { input: part.input } : {},
+        ...part.output ? { output: part.output } : {},
+        ...part.cache_control?.type ? {
+          providerOptions: {
+            anthropic: { cacheControl: { type: part.cache_control?.type } }
+          }
+        } : {}
+      };
+    });
+    parts = parts.filter((part) => part !== void 0);
+    return {
+      role: msg.role,
+      id: msg.id,
+      content: parts
+    };
+  });
+  raw = raw.filter((msg) => msg !== void 0);
+  return raw;
+};
 var createCustomAnthropicStreamingMessage = (message) => {
   const responseData = {
     type: "message",
@@ -5338,32 +5738,9 @@ var createCustomAnthropicStreamingMessage = (message) => {
 // src/registry/workers.ts
 var import_ioredis = __toESM(require("ioredis"), 1);
 var import_bullmq5 = require("bullmq");
-
-// src/registry/utils.ts
-var bullmq = {
-  validate: (id, data) => {
-    if (!data) {
-      throw new Error(`Missing job data for job ${id}.`);
-    }
-    if (!data.type) {
-      throw new Error(`Missing property "type" in data for job ${id}.`);
-    }
-    if (!data.inputs) {
-      throw new Error(`Missing property "inputs" in data for job ${id}.`);
-    }
-    if (data.type !== "embedder" && data.type !== "workflow") {
-      throw new Error(`Property "type" in data for job ${id} must be of value "embedder" or "workflow".`);
-    }
-    if (!data.workflow && !data.embedder) {
-      throw new Error(`Either a workflow or embedder must be set for job ${id}.`);
-    }
-  }
-};
-
-// src/registry/workers.ts
-var fs3 = __toESM(require("fs"), 1);
+var fs2 = __toESM(require("fs"), 1);
 var import_path = __toESM(require("path"), 1);
-var import_api3 = require("@opentelemetry/api");
+var import_api2 = require("@opentelemetry/api");
 var defaultLogsDir = import_path.default.join(process.cwd(), "logs");
 var redisConnection;
 var createWorkers = async (queues2, logger, contexts, _logsDir, tracer) => {
@@ -5441,16 +5818,16 @@ var createLogsCleanerWorker = (logsDir) => {
     global_queues.logs_cleaner,
     async (job) => {
       console.log(`[EXULU] recurring job ${job.id}.`);
-      const folder = fs3.readdirSync(logsDir);
+      const folder = fs2.readdirSync(logsDir);
       const files = folder.filter((file) => file.endsWith(".log"));
       const now = /* @__PURE__ */ new Date();
       const daysToKeep = job.data.ttld;
       const dateToKeep = new Date(now.getTime() - daysToKeep * 24 * 60 * 60 * 1e3);
       files.forEach((file) => {
         const filePath = import_path.default.join(logsDir, file);
-        const fileStats = fs3.statSync(filePath);
+        const fileStats = fs2.statSync(filePath);
         if (fileStats.mtime < dateToKeep) {
-          fs3.unlinkSync(filePath);
+          fs2.unlinkSync(filePath);
         }
       });
     },
@@ -5470,12 +5847,12 @@ var createLogsCleanerWorker = (logsDir) => {
 
 // src/mcp/index.ts
 var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
-var import_node_crypto3 = require("crypto");
+var import_node_crypto4 = require("crypto");
 var import_streamableHttp = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
 var import_types = require("@modelcontextprotocol/sdk/types.js");
 var import_zod3 = require("zod");
 var import_express6 = require("express");
-var import_api4 = require("@opentelemetry/api");
+var import_api3 = require("@opentelemetry/api");
 var SESSION_ID_HEADER = "mcp-session-id";
 var ExuluMCP = class {
   server;
@@ -5552,7 +5929,6 @@ ${code}`
     if (!this.server) {
       throw new Error("MCP server not initialized.");
     }
-    console.log("[EXULU] Wiring up MCP server routes to express app.");
     this.express.post("/mcp", async (req, res) => {
       if (!this.server) {
         throw new Error("MCP server not initialized.");
@@ -5563,7 +5939,7 @@ ${code}`
         transport = this.transports[sessionId];
       } else if (!sessionId && (0, import_types.isInitializeRequest)(req.body)) {
         transport = new import_streamableHttp.StreamableHTTPServerTransport({
-          sessionIdGenerator: () => (0, import_node_crypto3.randomUUID)(),
+          sessionIdGenerator: () => (0, import_node_crypto4.randomUUID)(),
           onsessioninitialized: (sessionId2) => {
             this.transports[sessionId2] = transport;
           }
@@ -5605,35 +5981,49 @@ ${code}`
 // src/registry/index.ts
 var import_express8 = __toESM(require("express"), 1);
 
-// src/templates/agents/claude-code.ts
-var claudeCodeAgent = new ExuluAgent2({
+// src/templates/agents/claude-sonnet-4.ts
+var import_anthropic = require("@ai-sdk/anthropic");
+var claudeSonnet4Agent = new ExuluAgent2({
   id: `claude_code_agent`,
   name: `Claude Code Agent`,
   description: `Claude Code agent, enabling the creation of multiple Claude Code Agent instances with different configurations (rate limits, functions, etc).`,
-  type: "custom"
+  type: "agent",
+  config: {
+    name: `Default Claude Code agent`,
+    instructions: "You are a coding assistant.",
+    model: {
+      create: ({ apiKey }) => {
+        const anthropic = (0, import_anthropic.createAnthropic)({
+          apiKey
+        });
+        return anthropic.languageModel("claude-sonnet-4-20250514");
+      }
+    }
+  }
 });
 
 // src/templates/agents/claude-opus-4.ts
-var import_anthropic = require("@ai-sdk/anthropic");
-var defaultAgent = new ExuluAgent2({
+var import_anthropic2 = require("@ai-sdk/anthropic");
+var claudeOpus4Agent = new ExuluAgent2({
   id: `default_claude_4_opus_agent`,
-  name: `Default Claude 4 Opus Agent`,
-  description: `Basic agent without any defined tools, that can support MCP's.`,
+  name: `Default Claude 4 Opus anthropic provider`,
+  description: `Basic agent claude 4 opus agent you can use to chat with.`,
   type: "agent",
   capabilities: {
     text: true,
     images: [".png", ".jpg", ".jpeg", ".webp"],
-    files: [".pdf", ".docx", ".xlsx", ".xls", ".csv", ".pptx", ".ppt"],
+    files: [".pdf", ".docx", ".xlsx", ".xls", ".csv", ".pptx", ".ppt", ".json"],
     audio: [],
     video: []
   },
   evals: [],
+  maxContextLength: 2e5,
   config: {
     name: `Default agent`,
     instructions: "You are a helpful assistant.",
     model: {
       create: ({ apiKey }) => {
-        const anthropic = (0, import_anthropic.createAnthropic)({
+        const anthropic = (0, import_anthropic2.createAnthropic)({
           apiKey
         });
         return anthropic.languageModel("claude-4-opus-20250514");
@@ -5649,8 +6039,79 @@ var defaultAgent = new ExuluAgent2({
   }
 });
 
+// src/templates/agents/gpt-5.ts
+var import_openai2 = require("@ai-sdk/openai");
+var gpt5MiniAgent = new ExuluAgent2({
+  id: `default_gpt_5_mini_agent`,
+  name: `Default GPT 5 Mini OpenAI provider`,
+  description: `Basic agent gpt 5 mini agent you can use to chat with.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [".png", ".jpg", ".jpeg", ".webp"],
+    files: [".pdf", ".docx", ".xlsx", ".xls", ".csv", ".pptx", ".ppt", ".json"],
+    audio: [],
+    video: []
+  },
+  evals: [],
+  maxContextLength: 128e3,
+  config: {
+    name: `Default agent`,
+    instructions: "You are a helpful assistant.",
+    model: {
+      create: ({ apiKey }) => {
+        const openai = (0, import_openai2.createOpenAI)({
+          apiKey
+        });
+        return openai.languageModel("gpt-5-mini");
+      }
+      // todo add a field of type string that adds a dropdown list from which the user can select the model
+      // todo for each model, check which provider is used, and require the admin to add one or multiple
+      //   API keys for the provider (which we can then auto-rotate).
+      // todo also add custom fields for rate limiting, so the admin can set custom rate limits for the agent
+      //   and allow him/her to decide if the rate limit is per user or per agent.
+      // todo finally allow switching on or off immutable audit logs on the agent. Which then enables OTEL
+      //   and stores the logs into the pre-defined storage.
+    }
+  }
+});
+var gpt5agent = new ExuluAgent2({
+  id: `default_gpt_5_agent`,
+  name: `Default GPT 5 OpenAI provider`,
+  description: `Basic agent gpt 5 agent you can use to chat with.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [".png", ".jpg", ".jpeg", ".webp"],
+    files: [".pdf", ".docx", ".xlsx", ".xls", ".csv", ".pptx", ".ppt", ".json"],
+    audio: [],
+    video: []
+  },
+  evals: [],
+  maxContextLength: 128e3,
+  config: {
+    name: `Default agent`,
+    instructions: "You are a helpful assistant.",
+    model: {
+      create: ({ apiKey }) => {
+        const openai = (0, import_openai2.createOpenAI)({
+          apiKey
+        });
+        return openai.languageModel("gpt-5");
+      }
+      // todo add a field of type string that adds a dropdown list from which the user can select the model
+      // todo for each model, check which provider is used, and require the admin to add one or multiple
+      //   API keys for the provider (which we can then auto-rotate).
+      // todo also add custom fields for rate limiting, so the admin can set custom rate limits for the agent
+      //   and allow him/her to decide if the rate limit is per user or per agent.
+      // todo finally allow switching on or off immutable audit logs on the agent. Which then enables OTEL
+      //   and stores the logs into the pre-defined storage.
+    }
+  }
+});
+
 // src/registry/index.ts
-var import_api5 = require("@opentelemetry/api");
+var import_api4 = require("@opentelemetry/api");
 
 // src/registry/logger.ts
 var import_winston_transport = require("@opentelemetry/winston-transport");
@@ -5702,21 +6163,21 @@ var codeStandardsContext = new ExuluContext({
   active: true
 });
 
-// src/templates/contexts/projects.ts
-var projectsContext = new ExuluContext({
-  id: "projects",
-  name: "Projects",
-  description: "Default context that stores files and data related to projects in Exulu.",
+// src/templates/contexts/outputs.ts
+var outputsContext = new ExuluContext({
+  id: "outputs_default_context",
+  name: "Outputs",
+  description: "Outputs from agent sessions that you have saved for re-used later.",
   configuration: {
-    defaultRightsMode: "projects"
+    defaultRightsMode: "private",
+    calculateVectors: "manual"
   },
-  fields: [{
-    name: "Type",
-    type: "text"
-  }, {
-    name: "Summary",
-    type: "longText"
-  }],
+  fields: [
+    {
+      name: "content",
+      type: "longText"
+    }
+  ],
   active: true
 });
 
@@ -5734,14 +6195,6 @@ var filesContext = new ExuluContext({
       name: "type",
       type: "text"
     },
-    {
-      name: "s3bucket",
-      type: "text"
-    },
-    {
-      name: "s3region",
-      type: "text"
-    },
     {
       name: "url",
       type: "text"
@@ -5751,10 +6204,6 @@ var filesContext = new ExuluContext({
       // ID of the file in S3 storage
       type: "text"
     },
-    {
-      name: "s3endpoint",
-      type: "text"
-    },
     {
       name: "content",
       type: "longText"
@@ -5765,7 +6214,6 @@ var filesContext = new ExuluContext({
 
 // src/registry/index.ts
 var isValidPostgresName = (id) => {
-  console.log("[EXULU] validating context id.", id);
   const regex = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
   const isValid = regex.test(id);
   const length = id.length;
@@ -5785,22 +6233,25 @@ var ExuluApp = class {
   create = async ({ contexts, agents, config, tools }) => {
     this._contexts = {
       ...contexts,
-      projectsContext,
       codeStandardsContext,
-      filesContext
+      filesContext,
+      outputsContext
     };
     this._agents = [
-      claudeCodeAgent,
-      defaultAgent,
+      claudeSonnet4Agent,
+      claudeOpus4Agent,
+      gpt5MiniAgent,
+      gpt5agent,
       ...agents ?? []
     ];
     this._config = config;
     this._tools = [
       ...tools ?? [],
       // Add contexts as tools
-      ...Object.values(contexts || {}).map((context) => context.tool()),
-      // Add agents as tools
-      ...(agents || []).map((agent) => agent.tool())
+      ...Object.values(contexts || {}).map((context) => context.tool())
+      // Because agents are stored in the database,  we add those as tools
+      // at request time, not during ExuluApp initialization. We add them
+      // in the grahql tools resolver.
     ];
     const checks = [
       ...Object.keys(this._contexts || {}).map((x) => ({
@@ -5893,8 +6344,7 @@ var ExuluApp = class {
       create: async () => {
         let tracer;
         if (this._config?.telemetry?.enabled) {
-          console.log("[EXULU] telemetry enabled.");
-          tracer = import_api5.trace.getTracer("exulu", "1.0.0");
+          tracer = import_api4.trace.getTracer("exulu", "1.0.0");
         }
         const logger = logger_default({
           enableOtel: this._config?.workers?.telemetry?.enabled ?? false
@@ -5918,8 +6368,7 @@ var ExuluApp = class {
         const app = this._expressApp;
         let tracer;
         if (this._config?.telemetry?.enabled) {
-          console.log("[EXULU] telemetry enabled");
-          tracer = import_api5.trace.getTracer("exulu", "1.0.0");
+          tracer = import_api4.trace.getTracer("exulu", "1.0.0");
         }
         const logger = logger_default({
           enableOtel: this._config?.telemetry?.enabled ?? false
@@ -5931,7 +6380,8 @@ var ExuluApp = class {
           this._tools,
           Object.values(this._contexts ?? {}),
           this._config,
-          tracer
+          tracer,
+          filesContext
         );
         if (this._config?.MCP.enabled) {
           const mcp = new ExuluMCP();
@@ -6211,7 +6661,7 @@ var RecursiveRules = class _RecursiveRules {
    * @param {string} path - The path to the recipe.
    * @returns {Promise<RecursiveRules>} The RecursiveRules object.
    */
-  static async fromRecipe(name = "default", lang = "en", path3) {
+  static async fromRecipe(name = "default", lang = "en", path2) {
     throw new Error("Not implemented");
   }
 };
@@ -7200,7 +7650,20 @@ var generateApiKey = async (name, email) => {
 };
 
 // src/postgres/init-db.ts
-var { agentsSchema: agentsSchema3, evalResultsSchema: evalResultsSchema3, jobsSchema: jobsSchema3, agentSessionsSchema: agentSessionsSchema3, agentMessagesSchema: agentMessagesSchema3, rolesSchema: rolesSchema3, usersSchema: usersSchema3, statisticsSchema: statisticsSchema3, variablesSchema: variablesSchema3, workflowTemplatesSchema: workflowTemplatesSchema3, rbacSchema: rbacSchema3, projectsSchema: projectsSchema3 } = coreSchemas.get();
+var {
+  agentsSchema: agentsSchema3,
+  evalResultsSchema: evalResultsSchema3,
+  jobsSchema: jobsSchema3,
+  agentSessionsSchema: agentSessionsSchema3,
+  agentMessagesSchema: agentMessagesSchema3,
+  rolesSchema: rolesSchema3,
+  usersSchema: usersSchema3,
+  statisticsSchema: statisticsSchema3,
+  variablesSchema: variablesSchema3,
+  workflowTemplatesSchema: workflowTemplatesSchema3,
+  rbacSchema: rbacSchema3,
+  projectsSchema: projectsSchema3
+} = coreSchemas.get();
 var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
   for (const field of fields) {
     const { type, name, default: defaultValue, unique } = field;
@@ -7257,6 +7720,7 @@ var up = async function(knex) {
     }
   };
   for (const schema of schemas) {
+    console.log(`[EXULU] Creating ${schema.name.plural} table.`, schema.fields);
     await createTable(schema);
   }
   if (!await knex.schema.hasTable("verification_token")) {
@@ -7413,6 +7877,9 @@ var create = ({
   return sdk;
 };
 
+// src/index.ts
+var import_crypto_js4 = __toESM(require("crypto-js"), 1);
+
 // types/enums/jobs.ts
 var JOB_STATUS_ENUM = {
   completed: "completed",
@@ -7431,6 +7898,113 @@ var ExuluJobs = {
     validate: validateJob
   }
 };
+var ExuluDefaultContexts = {
+  files: filesContext,
+  codeStandards: codeStandardsContext,
+  outputs: outputsContext
+};
+var ExuluDefaultAgents = {
+  anthropic: {
+    opus4: claudeOpus4Agent,
+    sonnet4: claudeSonnet4Agent
+  },
+  openai: {
+    gpt5Mini: gpt5MiniAgent,
+    gpt5: gpt5agent
+  }
+};
+var ExuluVariables = {
+  get: async (name) => {
+    const { db: db3 } = await postgresClient();
+    let variable = await db3.from("variables").where({ name }).first();
+    if (!variable) {
+      throw new Error(`Variable ${name} not found.`);
+    }
+    if (variable.encrypted) {
+      const bytes = import_crypto_js4.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+      variable.value = bytes.toString(import_crypto_js4.default.enc.Utf8);
+    }
+    return variable.value;
+  }
+};
+var ExuluUtils = {
+  batch: async ({
+    fn,
+    size,
+    inputs,
+    delay,
+    retries
+  }) => {
+    if (!size) {
+      size = 10;
+    }
+    if (!inputs) {
+      throw new Error("Inputs are required.");
+    }
+    if (!delay) {
+      delay = 0;
+    }
+    let results = [];
+    let lastBatchTime = 0;
+    for (let start = 0; start < inputs.length; start += size) {
+      const currentTime = Date.now();
+      const timeSinceLastBatch = currentTime - lastBatchTime;
+      if (timeSinceLastBatch < delay * 1e3) {
+        console.log("[EXULU] Utils function, waiting for", delay - timeSinceLastBatch, "seconds");
+        await new Promise((resolve) => setTimeout(resolve, delay * 1e3 - timeSinceLastBatch));
+      }
+      lastBatchTime = Date.now();
+      console.log(`[EXULU] Utils function, processing batch ${start / size + 1} of ${Math.ceil(inputs.length / size)} (${Math.min(start + 1, inputs.length)}-${Math.min(start + size, inputs.length)} of ${inputs.length})`);
+      const end = start + size > inputs.length ? inputs.length : start + size;
+      const slicedResults = await Promise.all(inputs.slice(start, end).map((data, i) => {
+        if (retries?.max) {
+          return ExuluUtils.retry({
+            fn: async () => {
+              return await fn(data);
+            },
+            retries: retries.max,
+            delays: retries.delays
+          });
+        } else {
+          return fn(data);
+        }
+      }));
+      results = [
+        ...results,
+        ...slicedResults
+      ];
+    }
+    return results;
+  },
+  retry: async ({
+    fn,
+    retries,
+    delays
+  }) => {
+    if (!retries) {
+      retries = 3;
+    }
+    if (!delays) {
+      delays = [1e3, 5e3, 1e4];
+    }
+    for (let i = 0; i < retries; i++) {
+      try {
+        return await fn();
+      } catch (error) {
+        console.error(`[EXULU] Util function, retry attempt ${i + 1} failed:`, error);
+        if (i >= retries - 1) {
+          throw error;
+        }
+        if (!delays[i]) {
+          delays[i] = delays[delays.length - 1] || 1e4;
+        }
+        const delay = delays && delays[i] ? delays[i] : 1e4;
+        console.log(`[EXULU] Util function, retrying in ${delay / 1e3} seconds...`);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+    }
+  }
+};
 var ExuluOtel = {
   create: ({
     SIGNOZ_ACCESS_TOKEN,
@@ -7474,11 +8048,15 @@ var ExuluChunkers = {
   ExuluAuthentication,
   ExuluChunkers,
   ExuluContext,
+  ExuluDefaultAgents,
+  ExuluDefaultContexts,
   ExuluEmbedder,
   ExuluEval,
   ExuluJobs,
   ExuluOtel,
   ExuluQueues,
   ExuluTool,
+  ExuluUtils,
+  ExuluVariables,
   db
 });