@exulu/backend 1.12.0 → 1.14.0

package/dist/index.js

@@ -75,19 +75,19 @@ import "bullmq";
 import { z } from "zod";
 import * as fs from "fs";
 import * as path from "path";
-import { generateObject, generateText, streamText, tool } from "ai";
+import { convertToModelMessages, createIdGenerator, generateObject, generateText, streamText, tool, validateUIMessages, stepCountIs } from "ai";
 
 // types/enums/statistics.ts
 var STATISTICS_TYPE_ENUM = {
-  CONTEXT_RETRIEVE: "context.retrieve",
-  SOURCE_UPDATE: "source.update",
-  EMBEDDER_UPSERT: "embedder.upsert",
-  EMBEDDER_GENERATE: "embedder.generate",
-  EMBEDDER_DELETE: "embedder.delete",
-  WORKFLOW_RUN: "workflow.run",
-  CONTEXT_UPSERT: "context.upsert",
-  TOOL_CALL: "tool.call",
-  AGENT_RUN: "agent.run"
+  CONTEXT_RETRIEVE: "CONTEXT_RETRIEVE",
+  SOURCE_UPDATE: "SOURCE_UPDATE",
+  EMBEDDER_UPSERT: "EMBEDDER_UPSERT",
+  EMBEDDER_GENERATE: "EMBEDDER_GENERATE",
+  EMBEDDER_DELETE: "EMBEDDER_DELETE",
+  WORKFLOW_RUN: "WORKFLOW_RUN",
+  CONTEXT_UPSERT: "CONTEXT_UPSERT",
+  TOOL_CALL: "TOOL_CALL",
+  AGENT_RUN: "AGENT_RUN"
 };
 
 // types/enums/eval-types.ts
@@ -262,7 +262,7 @@ var bullmqDecorator = async ({
 
 // src/registry/utils/map-types.ts
 var mapType = (t, type, name, defaultValue, unique) => {
-  if (type === "text") {
+  if (type === "text" || type === "enum") {
     t.text(name);
     if (unique) t.unique(name);
     if (defaultValue) t.defaultTo(defaultValue);
@@ -403,6 +403,7 @@ var ExuluEvalUtils = {
 
 // src/registry/classes.ts
 import CryptoJS from "crypto-js";
+import "express";
 function sanitizeToolName(name) {
   if (typeof name !== "string") return "";
   let sanitized = name.replace(/[^a-zA-Z0-9_-]+/g, "_");
@@ -421,7 +422,7 @@ var convertToolsArrayToObject = (tools, configs, providerApiKey) => {
   console.log("[EXULU] Sanitized tools", sanitizedTools);
   const askForConfirmation = {
     description: "Ask the user for confirmation.",
-    parameters: z.object({
+    inputSchema: z.object({
       message: z.string().describe("The message to ask for confirmation.")
     })
   };
@@ -532,10 +533,12 @@ var ExuluAgent = class {
     this.model = this.config?.model;
   }
   get providerName() {
-    return this.config?.model?.create({ apiKey: "" })?.provider || "";
+    const model = this.config?.model?.create({ apiKey: "" });
+    return typeof model === "string" ? model : model?.provider || "";
   }
   get modelName() {
-    return this.config?.model?.create({ apiKey: "" })?.modelId || "";
+    const model = this.config?.model?.create({ apiKey: "" });
+    return typeof model === "string" ? model : model?.modelId || "";
   }
   // Exports the agent as a tool that can be used by another agent
   // todo test this
@@ -561,28 +564,44 @@ var ExuluAgent = class {
       }
     });
   };
-  generateSync = async ({ messages, prompt, tools, statistics, toolConfigs, providerApiKey }) => {
+  generateSync = async ({ prompt, user, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
     if (!this.config) {
       throw new Error("Config is required for generating.");
     }
-    if (prompt && messages) {
-      throw new Error("Prompt and messages cannot be provided at the same time.");
+    if (prompt && message) {
+      throw new Error("Message and prompt cannot be provided at the same time.");
     }
     const model = this.model.create({
       apiKey: providerApiKey
     });
+    let messages = [];
+    if (message && session && user) {
+      const previousMessages = await getAgentMessages({
+        session,
+        user,
+        limit: 50,
+        page: 1
+      });
+      const previousMessagesContent = previousMessages.map((message2) => JSON.parse(message2.content));
+      messages = await validateUIMessages({
+        // append the new message to the previous messages:
+        messages: [...previousMessagesContent, message]
+      });
+    }
+    console.log("[EXULU] Model provider key", providerApiKey);
+    console.log("[EXULU] Tool configs", toolConfigs);
     const { text } = await generateText({
       model,
       // Should be a LanguageModelV1
       system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
-      messages,
+      messages: messages ? convertToModelMessages(messages) : void 0,
       prompt,
       maxRetries: 2,
       tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey),
-      maxSteps: 5
+      stopWhen: [stepCountIs(5)]
     });
     if (statistics) {
       await updateStatistic({
@@ -595,36 +614,61 @@ var ExuluAgent = class {
     }
     return text;
   };
-  generateStream = ({ messages, prompt, tools, statistics, toolConfigs, providerApiKey }) => {
+  generateStream = async ({ express: express3, user, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
     if (!this.config) {
       throw new Error("Config is required for generating.");
     }
-    if (prompt && messages) {
-      throw new Error("Prompt and messages cannot be provided at the same time.");
+    if (!message) {
+      throw new Error("Message is required for streaming.");
     }
     const model = this.model.create({
       apiKey: providerApiKey
     });
+    let messages = [];
+    const previousMessages = await getAgentMessages({
+      session,
+      user,
+      limit: 50,
+      page: 1
+    });
+    const previousMessagesContent = previousMessages.map((message2) => JSON.parse(message2.content));
+    messages = await validateUIMessages({
+      // append the new message to the previous messages:
+      messages: [...previousMessagesContent, message]
+    });
     console.log("[EXULU] Model provider key", providerApiKey);
     console.log("[EXULU] Tool configs", toolConfigs);
-    return streamText({
+    const result = streamText({
       model,
       // Should be a LanguageModelV1
-      messages,
-      prompt,
+      messages: messages ? convertToModelMessages(messages) : void 0,
       system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
       maxRetries: 2,
       tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey),
-      maxSteps: 5,
       onError: (error) => console.error("[EXULU] chat stream error.", error),
-      onFinish: async ({ response, usage }) => {
+      stopWhen: [stepCountIs(5)]
+    });
+    result.consumeStream();
+    result.pipeUIMessageStreamToResponse(express3.res, {
+      originalMessages: messages,
+      sendReasoning: true,
+      generateMessageId: createIdGenerator({
+        prefix: "msg_",
+        size: 16
+      }),
+      onFinish: async ({ messages: messages2 }) => {
         console.info(
           "[EXULU] chat stream finished.",
-          usage
+          messages2
         );
+        await saveChat({
+          session,
+          user,
+          messages: messages2
+        });
         if (statistics) {
           await updateStatistic({
             name: "count",
@@ -636,8 +680,26 @@ var ExuluAgent = class {
         }
       }
     });
+    return;
   };
 };
+var getAgentMessages = async ({ session, user, limit, page }) => {
+  const { db: db3 } = await postgresClient();
+  const messages = await db3.from("agent_messages").where({ session, user }).limit(limit).offset(page * limit);
+  return messages;
+};
+var saveChat = async ({ session, user, messages }) => {
+  const { db: db3 } = await postgresClient();
+  const promises2 = messages.map((message) => {
+    return db3.from("agent_messages").insert({
+      session,
+      user,
+      content: JSON.stringify(message),
+      title: message.role === "user" ? "User" : "Assistant"
+    });
+  });
+  await Promise.all(promises2);
+};
 var ExuluEmbedder = class {
   id;
   name;
@@ -888,7 +950,7 @@ var ExuluTool = class {
     this.type = type;
     this.tool = tool({
       description,
-      parameters: inputSchema || z.object({}),
+      inputSchema: inputSchema || z.object({}),
       execute: execute2
     });
   }
@@ -1377,22 +1439,27 @@ var ExuluSource = class {
 var updateStatistic = async (statistic) => {
   const currentDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
   const { db: db3 } = await postgresClient();
-  const existing = await db3.from("statistics").where({
+  const existing = await db3.from("tracking").where({
+    ...statistic.user ? { user: statistic.user } : {},
+    ...statistic.role ? { role: statistic.role } : {},
     name: statistic.name,
     label: statistic.label,
     type: statistic.type,
     createdAt: currentDate
   }).first();
+  console.log("!!! existing !!!", existing);
   if (!existing) {
-    await db3.from("statistics").insert({
+    await db3.from("tracking").insert({
       name: statistic.name,
       label: statistic.label,
       type: statistic.type,
       total: statistic.count ?? 1,
-      createdAt: currentDate
+      createdAt: currentDate,
+      ...statistic.user ? { user: statistic.user } : {},
+      ...statistic.role ? { role: statistic.role } : {}
     });
   } else {
-    await db3.from("statistics").update({
+    await db3.from("tracking").update({
       total: db3.raw("total + ?", [statistic.count ?? 1])
     }).where({
       name: statistic.name,
@@ -1731,25 +1798,25 @@ var requestValidators = {
         message: "Missing body."
       };
     }
-    if (!req.body.threadId) {
+    if (!req.headers["user"]) {
       return {
         error: true,
         code: 400,
-        message: "Missing threadId in body."
+        message: 'Missing "user" property in headers.'
       };
     }
-    if (!req.body.resourceId) {
+    if (!req.headers["session"]) {
       return {
         error: true,
         code: 400,
-        message: "Missing resourceId in body."
+        message: 'Missing "session" property in headers.'
       };
     }
-    if (!req.body.messages) {
+    if (!req.body.message) {
       return {
         error: true,
         code: 400,
-        message: 'Missing "messages" property in body.'
+        message: 'Missing "message" property in body.'
       };
     }
     return {
@@ -1853,6 +1920,9 @@ var map = (field) => {
     case "code":
       type = "String";
       break;
+    case "enum":
+      type = field.enumValues ? `${field.name}Enum` : "String";
+      break;
     case "number":
       type = "Float";
       break;
@@ -1871,6 +1941,16 @@ var map = (field) => {
   return type;
 };
 function createTypeDefs(table) {
+  const enumDefs = table.fields.filter((field) => field.type === "enum" && field.enumValues).map((field) => {
+    const enumValues = field.enumValues.map((value) => {
+      const sanitized = String(value).replace(/[^a-zA-Z0-9_]/g, "_").replace(/^[0-9]/, "_$&").toUpperCase();
+      return `  ${sanitized}`;
+    }).join("\n");
+    return `
+enum ${field.name}Enum {
+${enumValues}
+}`;
+  }).join("\n");
   const fields = table.fields.map((field) => {
     let type;
     type = map(field);
@@ -1882,8 +1962,6 @@ function createTypeDefs(table) {
   type ${table.name.singular} {
   ${fields.join("\n")}
     ${table.fields.find((field) => field.name === "id") ? "" : "id: ID!"}
-    createdAt: Date!
-    updatedAt: Date!
 ${rbacField}
   }
   `;
@@ -1894,39 +1972,62 @@ ${table.fields.map((f) => `  ${f.name}: ${map(f)}`).join("\n")}
 ${rbacInputField}
 }
 `;
-  return typeDef + inputDef;
+  return enumDefs + typeDef + inputDef;
 }
 function createFilterTypeDefs(table) {
   const fieldFilters = table.fields.map((field) => {
     let type;
-    type = map(field);
+    if (field.type === "enum" && field.enumValues) {
+      type = `${field.name}Enum`;
+    } else {
+      type = map(field);
+    }
     return `
   ${field.name}: FilterOperator${type}`;
   });
   const tableNameSingularUpperCaseFirst = table.name.singular.charAt(0).toUpperCase() + table.name.singular.slice(1);
+  const enumFilterOperators = table.fields.filter((field) => field.type === "enum" && field.enumValues).map((field) => {
+    const enumTypeName = `${field.name}Enum`;
+    return `
+input FilterOperator${enumTypeName} {
+  eq: ${enumTypeName}
+  ne: ${enumTypeName}
+  in: [${enumTypeName}]
+  and: [FilterOperator${enumTypeName}]
+  or: [FilterOperator${enumTypeName}]
+}`;
+  }).join("\n");
   const operatorTypes = `
 input FilterOperatorString {
   eq: String
   ne: String
   in: [String]
   contains: String
+  and: [FilterOperatorString]
+  or: [FilterOperatorString]
 }
 
 input FilterOperatorDate {
   lte: Date
   gte: Date
+  and: [FilterOperatorDate]
+  or: [FilterOperatorDate]
 }
 
 input FilterOperatorFloat {
   eq: Float
   ne: Float
   in: [Float]
+  and: [FilterOperatorFloat]
+  or: [FilterOperatorFloat]
 }
 
 input FilterOperatorBoolean {
   eq: Boolean
   ne: Boolean
   in: [Boolean]
+  and: [FilterOperatorBoolean]
+  or: [FilterOperatorBoolean]
 }
 
 input FilterOperatorJSON {
@@ -1945,6 +2046,8 @@ enum SortDirection {
   DESC
 }
 
+${enumFilterOperators}
+
 input Filter${tableNameSingularUpperCaseFirst} {
 ${fieldFilters.join("\n")}
 }`;
@@ -2014,33 +2117,33 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
 function createMutations(table) {
   const tableNamePlural = table.name.plural.toLowerCase();
   const validateWriteAccess = async (id, context) => {
-    const { db: db3, req, user } = context;
-    const hasRBAC = table.RBAC === true;
-    if (!hasRBAC) {
-      return true;
-    }
-    if (user.super_admin === true) {
-      return true;
-    }
-    if (!user.role || !(table.name.plural.includes("agent") && user.role.agents === "write") && !(table.name.plural.includes("workflow") && user.role.workflows === "write") && !(table.name.plural.includes("variable") && user.role.variables === "write") && !(table.name.plural.includes("user") && user.role.users === "write")) {
-      console.error("Access control error: no role found for current user or no access to entity type.");
-      throw new Error("Access control error: no role found for current user or no access to entity type.");
-    }
     try {
-      const authResult = await requestValidators.authenticate(req);
-      if (authResult.error || !authResult.user) {
-        throw new Error("Authentication required");
+      const { db: db3, req, user } = context;
+      if (user.super_admin === true) {
+        return true;
+      }
+      if (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write")) {
+        console.error("Access control error: no role found for current user or no access to entity type.");
+        throw new Error("Access control error: no role found for current user or no access to entity type.");
+      }
+      const hasRBAC = table.RBAC === true;
+      if (!hasRBAC) {
+        return true;
       }
-      const user2 = authResult.user;
       const record = await db3.from(tableNamePlural).select(["rights_mode", "created_by"]).where({ id }).first();
       if (!record) {
         throw new Error("Record not found");
       }
+      if (tableNamePlural === "jobs") {
+        if (!user.super_admin && record.created_by !== user.id) {
+          throw new Error("You are not authorized to edit this record");
+        }
+      }
       if (record.rights_mode === "public") {
         return true;
       }
       if (record.rights_mode === "private") {
-        if (record.created_by === user2.id) {
+        if (record.created_by === user.id) {
           return true;
         }
         throw new Error("Only the creator can edit this private record");
@@ -2050,7 +2153,7 @@ function createMutations(table) {
           entity: table.name.singular,
           target_resource_id: id,
           access_type: "User",
-          user_id: user2.id,
+          user_id: user.id,
           rights: "write"
         }).first();
         if (rbacRecord) {
@@ -2058,12 +2161,12 @@ function createMutations(table) {
         }
         throw new Error("Insufficient user permissions to edit this record");
       }
-      if (record.rights_mode === "roles" && user2.role) {
+      if (record.rights_mode === "roles" && user.role) {
         const rbacRecord = await db3.from("rbac").where({
           entity: table.name.singular,
           target_resource_id: id,
           access_type: "Role",
-          role_id: user2.role,
+          role_id: user.role,
           rights: "write"
         }).first();
         if (rbacRecord) {
@@ -2216,14 +2319,18 @@ function createMutations(table) {
 var applyAccessControl = (table, user, query) => {
   console.log("table", table);
   const tableNamePlural = table.name.plural.toLowerCase();
+  if (!user.super_admin && table.name.plural === "jobs") {
+    query = query.where("created_by", user.id);
+    return query;
+  }
   const hasRBAC = table.RBAC === true;
   if (!hasRBAC) {
     return query;
   }
-  if (user.super_admin === true) {
+  if (table.name.plural !== "agent_sessions" && user.super_admin === true) {
     return query;
   }
-  if (!user.role || !(table.name.plural.includes("agent") && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural.includes("workflow") && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural.includes("variable") && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural.includes("user") && (user.role.users === "read" || user.role.users === "write"))) {
+  if (!user.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write"))) {
     console.error("Access control error: no role found or no access to entity type.");
     return query.where("1", "=", "0");
   }
@@ -2251,6 +2358,27 @@ var applyAccessControl = (table, user, query) => {
   }
   return query;
 };
+var converOperatorToQuery = (query, fieldName, operators) => {
+  if (operators.eq !== void 0) {
+    query = query.where(fieldName, operators.eq);
+  }
+  if (operators.ne !== void 0) {
+    query = query.whereRaw(`?? IS DISTINCT FROM ?`, [fieldName, operators.ne]);
+  }
+  if (operators.in !== void 0) {
+    query = query.whereIn(fieldName, operators.in);
+  }
+  if (operators.contains !== void 0) {
+    query = query.where(fieldName, "like", `%${operators.contains}%`);
+  }
+  if (operators.lte !== void 0) {
+    query = query.where(fieldName, "<=", operators.lte);
+  }
+  if (operators.gte !== void 0) {
+    query = query.where(fieldName, ">=", operators.gte);
+  }
+  return query;
+};
 function createQueries(table) {
   const tableNamePlural = table.name.plural.toLowerCase();
   const tableNameSingular = table.name.singular.toLowerCase();
@@ -2258,18 +2386,19 @@ function createQueries(table) {
     filters.forEach((filter) => {
       Object.entries(filter).forEach(([fieldName, operators]) => {
         if (operators) {
-          if (operators.eq !== void 0) {
-            query = query.where(fieldName, operators.eq);
-          }
-          if (operators.ne !== void 0) {
-            query = query.whereRaw(`?? IS DISTINCT FROM ?`, [fieldName, operators.ne]);
-          }
-          if (operators.in !== void 0) {
-            query = query.whereIn(fieldName, operators.in);
+          if (operators.and !== void 0) {
+            console.log("operators.and", operators.and);
+            operators.and.forEach((operator) => {
+              query = converOperatorToQuery(query, fieldName, operator);
+            });
           }
-          if (operators.contains !== void 0) {
-            query = query.where(fieldName, "like", `%${operators.contains}%`);
+          if (operators.or !== void 0) {
+            operators.or.forEach((operator) => {
+              query = converOperatorToQuery(query, fieldName, operator);
+            });
           }
+          query = converOperatorToQuery(query, fieldName, operators);
+          console.log("query", query);
         }
       });
     });
@@ -2342,57 +2471,36 @@ function createQueries(table) {
       query = applyFilters(query, filters);
       query = applyAccessControl(table, context.user, query);
       if (groupBy) {
-        const results = await query.select(groupBy).count("* as count").groupBy(groupBy);
+        query = query.select(groupBy).groupBy(groupBy);
+        if (tableNamePlural === "tracking") {
+          query = query.sum("total as count");
+        } else {
+          query = query.count("* as count");
+        }
+        const results = await query;
+        console.log("!!! results !!!", results);
         return results.map((r) => ({
           group: r[groupBy],
-          count: Number(r.count)
+          count: r.count ? Number(r.count) : 0
         }));
       } else {
-        const [{ count }] = await query.count("* as count");
-        return [{
-          group: "total",
-          count: Number(count)
-        }];
-      }
-    },
-    // Add jobStatistics query for jobs table (backward compatibility)
-    ...tableNamePlural === "jobs" ? {
-      jobStatistics: async (_, args, context, info) => {
-        const { user, agent, from, to } = args;
-        const { db: db3 } = context;
-        let query = db3("jobs");
-        if (user) {
-          query = query.where("user", user);
-        }
-        if (agent) {
-          query = query.where("agent", agent);
-        }
-        if (from) {
-          query = query.where("createdAt", ">=", from);
-        }
-        if (to) {
-          query = query.where("createdAt", "<=", to);
+        if (tableNamePlural === "tracking") {
+          query = query.sum("total as count");
+          const [{ count }] = await query.sum("total as count");
+          console.log("!!! count !!!", count);
+          return [{
+            group: "total",
+            count: count ? Number(count) : 0
+          }];
+        } else {
+          const [{ count }] = await query.count("* as count");
+          return [{
+            group: "total",
+            count: count ? Number(count) : 0
+          }];
         }
-        query = applyAccessControl(table, context.user, query);
-        const runningQuery = query.clone().whereIn("status", ["active", "waiting", "delayed", "paused"]);
-        const [{ runningCount }] = await runningQuery.count("* as runningCount");
-        const erroredQuery = query.clone().whereIn("status", ["failed", "stuck"]);
-        const [{ erroredCount }] = await erroredQuery.count("* as erroredCount");
-        const completedQuery = query.clone().where("status", "completed");
-        const [{ completedCount }] = await completedQuery.count("* as completedCount");
-        const failedQuery = query.clone().where("status", "failed");
-        const [{ failedCount }] = await failedQuery.count("* as failedCount");
-        const durationQuery = query.clone().where("status", "completed").whereNotNull("duration").select(db3.raw('AVG("duration") as averageDuration'));
-        const [{ averageDuration }] = await durationQuery;
-        return {
-          runningCount: Number(runningCount),
-          erroredCount: Number(erroredCount),
-          completedCount: Number(completedCount),
-          failedCount: Number(failedCount),
-          averageDuration: averageDuration ? Number(averageDuration) : 0
-        };
       }
-    } : {}
+    }
   };
 }
 var RBACResolver = async (db3, table, entityName, resourceId, rights_mode) => {
@@ -2412,6 +2520,16 @@ var RBACResolver = async (db3, table, entityName, resourceId, rights_mode) => {
   };
 };
 function createSDL(tables) {
+  tables.forEach((table) => {
+    table.fields.push({
+      name: "createdAt",
+      type: "date"
+    });
+    table.fields.push({
+      name: "updatedAt",
+      type: "date"
+    });
+  });
   console.log("[EXULU] Creating SDL");
   let typeDefs = `
     scalar JSON
@@ -2468,7 +2586,6 @@ function createSDL(tables) {
       ${tableNamePlural}Pagination(limit: Int, page: Int, filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingularUpperCaseFirst}PaginationResult
       ${tableNameSingular}One(filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingular}
       ${tableNamePlural}Statistics(filters: [Filter${tableNameSingularUpperCaseFirst}], groupBy: String): [StatisticsResult]!
-      ${tableNamePlural === "jobs" ? `jobStatistics(user: ID, agent: String, from: String, to: String): JobStatistics` : ""}
     `;
     mutationDefs += `
       ${tableNamePlural}CreateOne(input: ${tableNameSingular}Input!): ${tableNameSingular}
@@ -2493,17 +2610,6 @@ type PageInfo {
   hasNextPage: Boolean!
 }
 `;
-    if (tableNamePlural === "jobs") {
-      modelDefs += `
-type JobStatistics {
-  runningCount: Int!
-  erroredCount: Int!
-  completedCount: Int!
-  failedCount: Int!
-  averageDuration: Float!
-}
-`;
-    }
     Object.assign(resolvers.Query, createQueries(table));
     Object.assign(resolvers.Mutation, createMutations(table));
     if (table.RBAC) {
@@ -2595,6 +2701,10 @@ var agentMessagesSchema = {
       name: "title",
       type: "text"
     },
+    {
+      name: "user",
+      type: "number"
+    },
     {
       name: "session",
       type: "text"
@@ -2831,6 +2941,10 @@ var rolesSchema = {
       type: "text"
       // write | read access to agents
     },
+    {
+      name: "api",
+      type: "text"
+    },
     {
       name: "workflows",
       type: "text"
@@ -2850,8 +2964,8 @@ var rolesSchema = {
 };
 var statisticsSchema = {
   name: {
-    plural: "statistics",
-    singular: "statistic"
+    plural: "tracking",
+    singular: "tracking"
   },
   fields: [
     {
@@ -2864,11 +2978,20 @@ var statisticsSchema = {
     },
     {
       name: "type",
-      type: "text"
+      type: "enum",
+      enumValues: Object.values(STATISTICS_TYPE_ENUM)
     },
     {
       name: "total",
       type: "number"
+    },
+    {
+      name: "user",
+      type: "number"
+    },
+    {
+      name: "role",
+      type: "uuid"
     }
   ]
 };
@@ -2929,6 +3052,7 @@ var jobsSchema = {
     plural: "jobs",
     singular: "job"
   },
+  RBAC: true,
   fields: [
     {
       name: "redis",
@@ -3580,11 +3704,8 @@ var createExpressRoutes = async (app, agents, tools, contexts) => {
     { route: "/agents/:id", method: "GET", note: "Get specific agent" },
     { route: "/contexts", method: "GET", note: "List all contexts" },
     { route: "/contexts/:id", method: "GET", note: "Get specific context" },
-    { route: "/contexts/statistics", method: "GET", note: "Get context statistics" },
     { route: "/tools", method: "GET", note: "List all tools" },
     { route: "/tools/:id", method: "GET", note: "Get specific tool" },
-    { route: "/statistics/timeseries", method: "POST", note: "Get time series statistics" },
-    { route: "/statistics/totals", method: "POST", note: "Get totals statistics" },
     { route: "/items/:context", method: "POST", note: "Create new item in context" },
     { route: "/items/:context", method: "GET", note: "Get items from context" },
     { route: "/items/export/:context", method: "GET", note: "Export items from context" },
@@ -4117,115 +4238,6 @@ var createExpressRoutes = async (app, agents, tools, contexts) => {
       authenticated: true
     });
   });
-  console.log("[EXULU] statistics timeseries");
-  app.post("/statistics/timeseries", async (req, res) => {
-    const authenticationResult = await requestValidators.authenticate(req);
-    if (!authenticationResult.user?.id) {
-      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-      return;
-    }
-    const { db: db3 } = await postgresClient();
-    const type = req.body.type;
-    if (!Object.values(STATISTICS_TYPE_ENUM).includes(type)) {
-      res.status(400).json({
-        message: "Invalid type, must be one of: " + Object.values(STATISTICS_TYPE_ENUM).join(", ")
-      });
-      return;
-    }
-    let from = new Date(req.body.from);
-    let to = new Date(req.body.to);
-    if (!from || !to) {
-      from = new Date(Date.now() - 7 * 24 * 60 * 60 * 1e3);
-      to = /* @__PURE__ */ new Date();
-    }
-    const query = db3.from("statistics").select("*");
-    query.where("name", "count");
-    query.andWhere("type", type);
-    query.andWhere("createdAt", ">=", from);
-    query.andWhere("createdAt", "<=", to);
-    const results = await query;
-    const dates = [];
-    for (let i = 0; i < (to.getTime() - from.getTime()) / (1e3 * 60 * 60 * 24); i++) {
-      dates.push(new Date(from.getTime() + i * (1e3 * 60 * 60 * 24)));
-    }
-    const data = dates.map((date) => {
-      const result = results.find((result2) => result2.date === date);
-      if (result) {
-        return result;
-      }
-      return {
-        date,
-        count: 0
-      };
-    });
-    res.status(200).json({
-      data,
-      filter: {
-        from,
-        to
-      }
-    });
-  });
-  console.log("[EXULU] statistics totals");
-  app.post("/statistics/totals", async (req, res) => {
-    const authenticationResult = await requestValidators.authenticate(req);
-    if (!authenticationResult.user?.id) {
-      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-      return;
-    }
-    const { db: db3 } = await postgresClient();
-    let from = new Date(req.body.from);
-    let to = new Date(req.body.to);
-    if (!from || !to) {
-      from = new Date(Date.now() - 7 * 24 * 60 * 60 * 1e3);
-      to = /* @__PURE__ */ new Date();
-    }
-    let promises2 = Object.values(STATISTICS_TYPE_ENUM).map(async (type) => {
-      const result = await db3.from("statistics").where("name", "count").andWhere("type", type).andWhere("createdAt", ">=", from).andWhere("createdAt", "<=", to).sum("total as total");
-      return {
-        [type]: result[0]?.total || 0
-      };
-    });
-    const results = await Promise.all(promises2);
-    res.status(200).json({
-      data: { ...Object.assign({}, ...results) },
-      filter: {
-        from,
-        to
-      }
-    });
-  });
-  console.log("[EXULU] contexts statistics");
-  app.get("/contexts/statistics", async (req, res) => {
-    const authenticationResult = await requestValidators.authenticate(req);
-    if (!authenticationResult.user?.id) {
-      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-      return;
-    }
-    const { db: db3 } = await postgresClient();
-    const statistics = await db3("statistics").where("name", "count").andWhere("type", "context.retrieve").sum("total as total").first();
-    const response = await db3("jobs").select(db3.raw(`to_char("createdAt", 'YYYY-MM-DD') as date`)).count("* as count").where("type", "embedder").groupByRaw(`to_char("createdAt", 'YYYY-MM-DD')`).then((rows) => ({
-      jobs: rows
-    }));
-    let jobs = [];
-    if (response[0]) {
-      jobs = response[0].jobs.map((job) => ({
-        date: job.id,
-        count: job.count
-      }));
-    }
-    const embeddingsCountResult = await db3("jobs").where("type", "embedder").count("* as count").first();
-    res.status(200).json({
-      active: contexts.filter((context) => context.active).length,
-      inactive: contexts.filter((context) => !context.active).length,
-      sources: contexts.reduce((acc, context) => acc + context.sources.get().length, 0),
-      queries: statistics?.total || 0,
-      jobs,
-      totals: {
-        embeddings: embeddingsCountResult?.count || 0
-      }
-    });
-  });
   console.log("[EXULU] context by id");
   app.get(`/contexts/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
@@ -4396,7 +4408,11 @@ var createExpressRoutes = async (app, agents, tools, contexts) => {
           return;
         }
       }
-      const stream = req.headers["stream"] || false;
+      const headers = {
+        stream: req.headers["stream"] === "true" || false,
+        user: req.headers["user"] || null,
+        session: req.headers["session"] || null
+      };
       const requestValidationResult = requestValidators.agents(req);
       if (requestValidationResult.error) {
         res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
@@ -4407,6 +4423,13 @@ var createExpressRoutes = async (app, agents, tools, contexts) => {
         res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
         return;
       }
+      const user = authenticationResult.user;
+      if (user.type !== "api" && !user.super_admin && req.body.resourceId !== user.id) {
+        res.status(400).json({
+          message: "The provided user id in the resourceId field is not the same as the authenticated user. Only super admins and API users can impersonate other users."
+        });
+        return;
+      }
       console.log("[EXULU] agent tools", agentInstance.tools);
       const enabledTools = agentInstance.tools.map(({ config, toolId }) => tools.find(({ id }) => id === toolId)).filter(Boolean);
       console.log("[EXULU] enabled tools", enabledTools);
@@ -4429,9 +4452,15 @@ var createExpressRoutes = async (app, agents, tools, contexts) => {
         const bytes = CryptoJS3.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
         providerApiKey = bytes.toString(CryptoJS3.enc.Utf8);
       }
-      if (!!stream) {
-        const result = agent.generateStream({
-          messages: req.body.messages,
+      if (!!headers.stream) {
+        await agent.generateStream({
+          express: {
+            res,
+            req
+          },
+          user: headers.user,
+          session: headers.session,
+          message: req.body.message,
           tools: enabledTools,
           providerApiKey,
           toolConfigs: agentInstance.tools,
@@ -4440,11 +4469,12 @@ var createExpressRoutes = async (app, agents, tools, contexts) => {
             trigger: "agent"
           }
         });
-        result.pipeDataStreamToResponse(res);
         return;
       } else {
         const response = await agent.generateSync({
-          messages: req.body.messages,
+          user: headers.user,
+          session: headers.session,
+          message: req.body.message,
           tools: enabledTools.map((tool2) => tool2.tool),
           providerApiKey,
           toolConfigs: agentInstance.tools,
@@ -4969,7 +4999,6 @@ var defaultAgent = new ExuluAgent({
   description: `Basic agent without any defined tools, that can support MCP's.`,
   type: "agent",
   capabilities: {
-    tools: false,
     images: [],
     files: [],
     audio: [],
@@ -4981,10 +5010,10 @@ var defaultAgent = new ExuluAgent({
     instructions: "You are a helpful assistant.",
     model: {
       create: ({ apiKey }) => {
-        const anthropic2 = createAnthropic({
+        const anthropic = createAnthropic({
           apiKey
         });
-        return anthropic2("claude-4-opus-20250514");
+        return anthropic.languageModel("claude-4-opus-20250514");
       }
       // todo add a field of type string that adds a dropdown list from which the user can select the model
       // todo for each model, check which provider is used, and require the admin to add one or multiple
@@ -6525,6 +6554,13 @@ var ExuluJobs = {
 var db2 = {
   init: async () => {
     await execute();
+  },
+  api: {
+    key: {
+      generate: async (name, email) => {
+        return await generateApiKey(name, email);
+      }
+    }
   }
 };
 var ExuluChunkers = {