@exulu/backend 1.33.0 → 1.34.1

package/.mcp.json

@@ -0,0 +1,11 @@
+{
+  "mcpServers": {
+    "exulu-mcp-server-default-coding-agent": {
+      "type": "http",
+      "url": "https://backend.ai.open.de/mcp/76e6a87c-98be-480b-8081-264eb0bb4ecc",
+      "headers": {
+        "Authorization": "Bearer ${EXULU_TOKEN}"
+      }
+    }
+  }
+}

package/CHANGELOG.md

@@ -1,6 +1,6 @@
-# [1.33.0](https://github.com/Qventu/exulu-backend/compare/v1.32.1...v1.33.0) (2025-11-14)
+## [1.34.1](https://github.com/Qventu/exulu-backend/compare/v1.34.0...v1.34.1) (2025-11-17)
 
 
-### Features
+### Bug Fixes
 
-* add todo system, enhanced roles, session metadata, and scheduling support ([06cb449](https://github.com/Qventu/exulu-backend/commit/06cb44999d8f0fcbf60c8f5c50fc3a84db4131d9))
+* project tracking anthropic passthrough and timeout for worker ([d0c39c8](https://github.com/Qventu/exulu-backend/commit/d0c39c8dec082a098b91958d9b2fa31baa31b59e))

package/dist/index.cjs

@@ -279,7 +279,7 @@ var bullmqDecorator = async ({
     embedder,
     workflow,
     processor,
-    eval
+    evaluation
   ];
   if (types.filter((type2) => type2).length > 1) {
     throw new Error("Cannot have multiple types in the same job, must be one of the following: embedder, workflow, processor or eval.");
@@ -1379,6 +1379,69 @@ var platformConfigurationsSchema = {
     }
   ]
 };
+var promptLibrarySchema = {
+  type: "prompt_library",
+  name: {
+    plural: "prompt_library",
+    singular: "prompt_library_item"
+  },
+  RBAC: true,
+  fields: [
+    {
+      name: "name",
+      type: "text",
+      required: true
+    },
+    {
+      name: "description",
+      type: "text"
+    },
+    {
+      name: "content",
+      type: "longText",
+      required: true
+    },
+    {
+      name: "tags",
+      type: "json"
+    },
+    {
+      name: "usage_count",
+      type: "number",
+      default: 0
+    },
+    {
+      name: "favorite_count",
+      type: "number",
+      default: 0
+    },
+    {
+      name: "assigned_agents",
+      type: "json"
+    }
+  ]
+};
+var promptFavoritesSchema = {
+  type: "prompt_favorites",
+  name: {
+    plural: "prompt_favorites",
+    singular: "prompt_favorite"
+  },
+  fields: [
+    {
+      name: "user_id",
+      type: "number",
+      required: true,
+      index: true
+    },
+    {
+      name: "prompt_id",
+      type: "uuid",
+      required: true,
+      index: true
+    }
+  ]
+};
 var addCoreFields = (schema) => {
   schema.fields.forEach((field) => {
     if (field.type === "file") {
@@ -1422,7 +1485,9 @@ var coreSchemas = {
       rbacSchema: () => addCoreFields(rbacSchema),
       workflowTemplatesSchema: () => addCoreFields(workflowTemplatesSchema),
       platformConfigurationsSchema: () => addCoreFields(platformConfigurationsSchema),
-      jobResultsSchema: () => addCoreFields(jobResultsSchema)
+      jobResultsSchema: () => addCoreFields(jobResultsSchema),
+      promptLibrarySchema: () => addCoreFields(promptLibrarySchema),
+      promptFavoritesSchema: () => addCoreFields(promptFavoritesSchema)
     };
   }
 };
@@ -1882,6 +1947,7 @@ input FilterOperatorJSON {
   eq: JSON
   ne: JSON
   in: [JSON]
+  contains: JSON
 }
 
 input SortBy {
@@ -2383,7 +2449,9 @@ function createMutations(table, agents, contexts, tools, config) {
           source: source.id,
           context: exists.id,
           type: "source",
-          inputs: args.inputs
+          inputs: args.inputs,
+          user: context.user.id,
+          role: context.user.role?.id
         });
         console.log("[EXULU] Source function job scheduled", job.id);
         return {
@@ -2412,6 +2480,15 @@ function createMutations(table, agents, contexts, tools, config) {
           console.log(`[EXULU] created item through source update job ${createdItem.id}`);
         }
       }
+      await updateStatistic({
+        name: "count",
+        label: source.id,
+        type: STATISTICS_TYPE_ENUM.SOURCE_UPDATE,
+        trigger: "api",
+        count: 1,
+        user: context?.user?.id,
+        role: context?.user?.role?.id
+      });
       return {
         message: "Items created successfully.",
         jobs,
@@ -2446,7 +2523,7 @@ function createMutations(table, agents, contexts, tools, config) {
           jobs: jobs2.slice(0, 100)
         };
       }
-      query = applyFilters(query, args.where);
+      query = applyFilters(query, args.where, table);
       const items = await query;
       if (items.length === 0) {
         throw new Error("No items found to generate chunks for.");
@@ -2481,7 +2558,7 @@ function createMutations(table, agents, contexts, tools, config) {
       }
       let query = db3.from(getTableName(id)).select("id");
       if (args.where) {
-        query = applyFilters(query, args.where);
+        query = applyFilters(query, args.where, table);
       }
       const items = await query;
       if (items.length === 0) {
@@ -2538,18 +2615,38 @@ var applyAccessControl = (table, user, query) => {
   }
   return query;
 };
-var converOperatorToQuery = (query, fieldName, operators) => {
+var converOperatorToQuery = (query, fieldName, operators, table) => {
+  const field = table?.fields.find((f) => f.name === fieldName);
+  const isJsonField = field?.type === "json";
   if (operators.eq !== void 0) {
-    query = query.where(fieldName, operators.eq);
+    if (isJsonField) {
+      query = query.whereRaw(`?? = ?::jsonb`, [fieldName, JSON.stringify(operators.eq)]);
+    } else {
+      query = query.where(fieldName, operators.eq);
+    }
   }
   if (operators.ne !== void 0) {
-    query = query.whereRaw(`?? IS DISTINCT FROM ?`, [fieldName, operators.ne]);
+    if (isJsonField) {
+      query = query.whereRaw(`?? IS DISTINCT FROM ?::jsonb`, [fieldName, JSON.stringify(operators.ne)]);
+    } else {
+      query = query.whereRaw(`?? IS DISTINCT FROM ?`, [fieldName, operators.ne]);
+    }
   }
   if (operators.in !== void 0) {
-    query = query.whereIn(fieldName, operators.in);
+    if (isJsonField) {
+      const conditions = operators.in.map((val) => `?? = ?::jsonb`).join(" OR ");
+      const bindings = operators.in.flatMap((val) => [fieldName, JSON.stringify(val)]);
+      query = query.whereRaw(`(${conditions})`, bindings);
+    } else {
+      query = query.whereIn(fieldName, operators.in);
+    }
   }
   if (operators.contains !== void 0) {
-    query = query.where(fieldName, "like", `%${operators.contains}%`);
+    if (isJsonField) {
+      query = query.whereRaw(`?? @> ?::jsonb`, [fieldName, JSON.stringify(operators.contains)]);
+    } else {
+      query = query.where(fieldName, "like", `%${operators.contains}%`);
+    }
   }
   if (operators.lte !== void 0) {
     query = query.where(fieldName, "<=", operators.lte);
@@ -2811,21 +2908,21 @@ var finalizeRequestedFields = async ({
   }
   return result;
 };
-var applyFilters = (query, filters) => {
+var applyFilters = (query, filters, table) => {
   filters.forEach((filter) => {
     Object.entries(filter).forEach(([fieldName, operators]) => {
       if (operators) {
         if (operators.and !== void 0) {
           operators.and.forEach((operator) => {
-            query = converOperatorToQuery(query, fieldName, operator);
+            query = converOperatorToQuery(query, fieldName, operator, table);
           });
         }
         if (operators.or !== void 0) {
           operators.or.forEach((operator) => {
-            query = converOperatorToQuery(query, fieldName, operator);
+            query = converOperatorToQuery(query, fieldName, operator, table);
           });
         }
-        query = converOperatorToQuery(query, fieldName, operators);
+        query = converOperatorToQuery(query, fieldName, operators, table);
       }
     });
   });
@@ -2865,7 +2962,7 @@ function createQueries(table, agents, tools, contexts) {
       const requestedFields = getRequestedFields(info);
       const sanitizedFields = sanitizeRequestedFields(table, requestedFields);
       let query = db3.from(tableNamePlural).select(sanitizedFields);
-      query = applyFilters(query, filters);
+      query = applyFilters(query, filters, table);
       query = applyAccessControl(table, context.user, query);
       query = applySorting(query, sort);
       let result = await query.first();
@@ -2878,7 +2975,7 @@ function createQueries(table, agents, tools, contexts) {
         throw new Error("Limit cannot be greater than 500.");
       }
       let countQuery = db3(tableNamePlural);
-      countQuery = applyFilters(countQuery, filters);
+      countQuery = applyFilters(countQuery, filters, table);
       countQuery = applyAccessControl(table, context.user, countQuery);
       const countResult = await countQuery.count("* as count");
       const itemCount = Number(countResult[0]?.count || 0);
@@ -2887,7 +2984,7 @@ function createQueries(table, agents, tools, contexts) {
       const hasPreviousPage = currentPage > 1;
       const hasNextPage = currentPage < pageCount - 1;
       let dataQuery = db3(tableNamePlural);
-      dataQuery = applyFilters(dataQuery, filters);
+      dataQuery = applyFilters(dataQuery, filters, table);
       dataQuery = applyAccessControl(table, context.user, dataQuery);
       const requestedFields = getRequestedFields(info);
       dataQuery = applySorting(dataQuery, sort);
@@ -2912,7 +3009,7 @@ function createQueries(table, agents, tools, contexts) {
       const { filters = [], groupBy } = args;
       const { db: db3 } = context;
       let query = db3(tableNamePlural);
-      query = applyFilters(query, filters);
+      query = applyFilters(query, filters, table);
       query = applyAccessControl(table, context.user, query);
       if (groupBy) {
         query = query.select(groupBy).groupBy(groupBy);
@@ -3012,11 +3109,11 @@ var vectorSearch = async ({
   const mainTable = getTableName(id);
   const chunksTable = getChunksTableName(id);
   let countQuery = db3(mainTable);
-  countQuery = applyFilters(countQuery, filters);
+  countQuery = applyFilters(countQuery, filters, table);
   countQuery = applyAccessControl(table, user, countQuery);
   const columns = await db3(mainTable).columnInfo();
   let itemsQuery = db3(mainTable).select(Object.keys(columns).map((column) => mainTable + "." + column));
-  itemsQuery = applyFilters(itemsQuery, filters);
+  itemsQuery = applyFilters(itemsQuery, filters, table);
   itemsQuery = applyAccessControl(table, user, itemsQuery);
   itemsQuery = applySorting(itemsQuery, sort);
   if (queryRewriter) {
@@ -3777,7 +3874,7 @@ type PageInfo {
     const {
       jobs,
       count
-    } = await getJobsByQueueAndName(
+    } = await getJobsByQueueName(
       args.queue,
       args.statusses,
       args.page || 1,
@@ -4210,7 +4307,7 @@ var validateCreateOrRemoveSuperAdminPermission = async (tableNamePlural, input,
     }
   }
 };
-async function getJobsByQueueAndName(queueName, statusses, page, limit) {
+async function getJobsByQueueName(queueName, statusses, page, limit) {
   const queue = queues.list.get(queueName);
   if (!queue) {
     throw new Error(`Queue ${queueName} not found`);
@@ -5681,6 +5778,9 @@ var ExuluContext = class {
     };
   };
   createItem = async (item, config, user, role, upsert) => {
+    if (upsert && (!item.id && !item.external_id)) {
+      throw new Error("Item id or external id is required for upsert.");
+    }
     const { db: db3 } = await postgresClient();
     const mutation = db3.from(getTableName(
       this.id
@@ -5965,6 +6065,7 @@ var ExuluContext = class {
   };
 };
 var updateStatistic = async (statistic) => {
+  console.log("[EXULU] updating statistic", statistic);
   const currentDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
   const { db: db3 } = await postgresClient();
   const existing = await db3.from("tracking").where({
@@ -6108,6 +6209,8 @@ var {
   variablesSchema: variablesSchema2,
   workflowTemplatesSchema: workflowTemplatesSchema2,
   rbacSchema: rbacSchema2,
+  promptLibrarySchema: promptLibrarySchema2,
+  promptFavoritesSchema: promptFavoritesSchema2,
   statisticsSchema: statisticsSchema2
 } = coreSchemas.get();
 var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer, queues2) => {
@@ -6138,6 +6241,8 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tr
     agentsSchema2(),
     projectsSchema2(),
     jobResultsSchema2(),
+    promptLibrarySchema2(),
+    promptFavoritesSchema2(),
     evalRunsSchema2(),
     platformConfigurationsSchema2(),
     evalSetsSchema2(),
@@ -6548,7 +6653,7 @@ Mood: friendly and intelligent.
         return;
       }
       let project = null;
-      if (!req.body.project || req.body.project === "DEFAULT") {
+      if (!req.params.project || req.params.project === "DEFAULT") {
         project = null;
       } else {
         let projectQuery = db3("projects");
@@ -6825,7 +6930,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
         }));
         const { db: db3 } = await postgresClient();
         const data = bullmqJob.data;
-        const timeoutMs = data.timeoutInSeconds * 1e3;
+        const timeoutMs = (data.timeoutInSeconds || 300) * 1e3;
         const timeoutPromise = new Promise((_, reject) => {
           setTimeout(() => {
             reject(new Error(`Timeout for job ${bullmqJob.id} reached after ${data.timeoutInSeconds}s`));
@@ -6916,6 +7021,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               const label = `eval-run-${data.eval_run_id}-${data.test_case_id}`;
               const existingResult = await db3.from("job_results").where({ label }).first();
               if (existingResult) {
+                console.log("[EXULU] found existing job result, so ");
                 await db3.from("job_results").where({ label }).update({
                   job_id: bullmqJob.id,
                   label,
@@ -7017,6 +7123,8 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                     test_case_id: testCase.id,
                     eval_run_id: evalRun.id,
                     eval_function_id: evalFunction.id,
+                    eval_function_name: evalFunction.name,
+                    eval_function_config: evalFunction.config || {},
                     result: result2 || 0
                   };
                   console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata2(bullmqJob.name, {
@@ -7044,22 +7152,32 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 }
               }
               const scores = evalFunctionResults.map((result2) => result2.result);
+              console.log("[EXULU] Exulu eval run scores for test case: " + testCase.id, scores);
               let score = 0;
-              switch (data.scoring_method) {
+              switch (data.scoring_method?.toLowerCase()) {
                 case "median":
+                  console.log("[EXULU] Calculating median score");
                   score = getMedian(scores);
                   break;
                 case "average":
+                  console.log("[EXULU] Calculating average score");
                   score = getAverage(scores);
                   break;
                 case "sum":
+                  console.log("[EXULU] Calculating sum score");
                   score = getSum(scores);
                   break;
+                default:
+                  console.log("[EXULU] Calculating average score");
+                  score = getAverage(scores);
               }
               return {
                 result: score,
                 metadata: {
-                  ...evalFunctionResults,
+                  messages,
+                  function_results: [
+                    ...evalFunctionResults
+                  ],
                   ...metadata
                 }
               };
@@ -7155,6 +7273,15 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                   }));
                 }
               }
+              await updateStatistic({
+                name: "count",
+                label: source.id,
+                type: STATISTICS_TYPE_ENUM.SOURCE_UPDATE,
+                trigger: "api",
+                count: 1,
+                user: data?.user,
+                role: data?.role
+              });
               return {
                 result,
                 metadata: {
@@ -7592,6 +7719,92 @@ var ExuluMCP = class {
       });
       server.tools[tool2.id] = tool2.name;
     }
+    const getListOfPromptTemplatesName = "getListOfPromptTemplates";
+    if (!server.tools[getListOfPromptTemplatesName]) {
+      server.mcp.registerTool(getListOfPromptTemplatesName, {
+        title: "Get List of Prompt Templates",
+        description: "Retrieves a list of prompt templates available for this agent. Returns the name, description, and ID of each template.",
+        inputSchema: {
+          inputs: import_zod2.z.object({})
+        }
+      }, async ({ inputs }, args) => {
+        console.log("[EXULU] Getting list of prompt templates for agent", agentInstance.id);
+        const { db: db4 } = await postgresClient();
+        const prompts = await db4.from("prompt_library").select("id", "name", "description").whereRaw("assigned_agents @> ?::jsonb", [JSON.stringify(agentInstance.id)]).orderBy("updatedAt", "desc");
+        console.log("[EXULU] Found", prompts.length, "prompt templates");
+        return {
+          content: [{
+            type: "text",
+            text: JSON.stringify({
+              prompts: prompts.map((p) => ({
+                id: p.id,
+                name: p.name,
+                description: p.description || "No description provided"
+              })),
+              count: prompts.length
+            }, null, 2)
+          }],
+          structuredContent: {
+            prompts: prompts.map((p) => ({
+              id: p.id,
+              name: p.name,
+              description: p.description || "No description provided"
+            })),
+            count: prompts.length
+          }
+        };
+      });
+    }
+    const getPromptTemplateDetailsName = "getPromptTemplateDetails";
+    if (!server.tools[getPromptTemplateDetailsName]) {
+      server.mcp.registerTool(getPromptTemplateDetailsName, {
+        title: "Get Prompt Template Details",
+        description: "Retrieves the full details of a specific prompt template by ID, including the actual template content with variables.",
+        inputSchema: {
+          inputs: import_zod2.z.object({
+            id: import_zod2.z.string().describe("The ID of the prompt template to retrieve")
+          })
+        }
+      }, async ({ inputs }, args) => {
+        console.log("[EXULU] Getting prompt template details for ID", inputs.id);
+        const { db: db4 } = await postgresClient();
+        const prompt = await db4.from("prompt_library").select("id", "name", "description", "content", "createdAt", "updatedAt", "usage_count", "favorite_count").where({ id: inputs.id }).first();
+        if (!prompt) {
+          throw new Error(`Prompt template with ID ${inputs.id} not found`);
+        }
+        const isAssignedToAgent = await db4.from("prompt_library").select("id").where({ id: inputs.id }).whereRaw("assigned_agents @> ?::jsonb", [JSON.stringify(agentInstance.id)]).first();
+        console.log("[EXULU] Prompt template found:", prompt.name);
+        return {
+          content: [{
+            type: "text",
+            text: JSON.stringify({
+              id: prompt.id,
+              name: prompt.name,
+              description: prompt.description || "No description provided",
+              content: prompt.content,
+              createdAt: prompt.createdAt,
+              updatedAt: prompt.updatedAt,
+              usageCount: prompt.usage_count || 0,
+              favoriteCount: prompt.favorite_count || 0,
+              isAssignedToThisAgent: !!isAssignedToAgent
+            }, null, 2)
+          }],
+          structuredContent: {
+            id: prompt.id,
+            name: prompt.name,
+            description: prompt.description || "No description provided",
+            content: prompt.content,
+            createdAt: prompt.createdAt,
+            updatedAt: prompt.updatedAt,
+            usageCount: prompt.usage_count || 0,
+            favoriteCount: prompt.favorite_count || 0,
+            isAssignedToThisAgent: !!isAssignedToAgent
+          }
+        };
+      });
+    }
+    server.tools[getListOfPromptTemplatesName] = getListOfPromptTemplatesName;
+    server.tools[getPromptTemplateDetailsName] = getPromptTemplateDetailsName;
     return server.mcp;
   };
   create = async ({ express: express3, allTools, allAgents, allContexts, config }) => {
@@ -8206,8 +8419,8 @@ var llmAsJudgeEval = () => {
         console.log("[EXULU] running llm as judge eval", { agent, backend, messages, testCase, config });
         let prompt = config?.prompt;
         if (!prompt) {
-          console.error("[EXULU] prompt is required.");
-          throw new Error("Prompt is required.");
+          console.error("[EXULU] prompt is required for llm as judge eval but none is provided.");
+          throw new Error("Prompt is required for llm as judge eval but none is provided.");
         }
         const lastMessage = messages[messages.length - 1]?.parts?.filter((part) => part.type === "text").map((part) => part.text).join("\n");
         console.log("[EXULU] last message", lastMessage);
@@ -10672,7 +10885,9 @@ var {
   workflowTemplatesSchema: workflowTemplatesSchema3,
   rbacSchema: rbacSchema3,
   projectsSchema: projectsSchema3,
-  jobResultsSchema: jobResultsSchema3
+  jobResultsSchema: jobResultsSchema3,
+  promptLibrarySchema: promptLibrarySchema3,
+  promptFavoritesSchema: promptFavoritesSchema3
 } = coreSchemas.get();
 var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
   for (const field of fields) {
@@ -10707,6 +10922,8 @@ var up = async function(knex) {
     statisticsSchema3(),
     projectsSchema3(),
     jobResultsSchema3(),
+    promptLibrarySchema3(),
+    promptFavoritesSchema3(),
     rbacSchema3(),
     agentsSchema3(),
     variablesSchema3(),

package/dist/index.js

@@ -227,7 +227,7 @@ var bullmqDecorator = async ({
     embedder,
     workflow,
     processor,
-    eval
+    evaluation
   ];
   if (types.filter((type2) => type2).length > 1) {
     throw new Error("Cannot have multiple types in the same job, must be one of the following: embedder, workflow, processor or eval.");
@@ -1327,6 +1327,69 @@ var platformConfigurationsSchema = {
     }
   ]
 };
+var promptLibrarySchema = {
+  type: "prompt_library",
+  name: {
+    plural: "prompt_library",
+    singular: "prompt_library_item"
+  },
+  RBAC: true,
+  fields: [
+    {
+      name: "name",
+      type: "text",
+      required: true
+    },
+    {
+      name: "description",
+      type: "text"
+    },
+    {
+      name: "content",
+      type: "longText",
+      required: true
+    },
+    {
+      name: "tags",
+      type: "json"
+    },
+    {
+      name: "usage_count",
+      type: "number",
+      default: 0
+    },
+    {
+      name: "favorite_count",
+      type: "number",
+      default: 0
+    },
+    {
+      name: "assigned_agents",
+      type: "json"
+    }
+  ]
+};
+var promptFavoritesSchema = {
+  type: "prompt_favorites",
+  name: {
+    plural: "prompt_favorites",
+    singular: "prompt_favorite"
+  },
+  fields: [
+    {
+      name: "user_id",
+      type: "number",
+      required: true,
+      index: true
+    },
+    {
+      name: "prompt_id",
+      type: "uuid",
+      required: true,
+      index: true
+    }
+  ]
+};
 var addCoreFields = (schema) => {
   schema.fields.forEach((field) => {
     if (field.type === "file") {
@@ -1370,7 +1433,9 @@ var coreSchemas = {
       rbacSchema: () => addCoreFields(rbacSchema),
       workflowTemplatesSchema: () => addCoreFields(workflowTemplatesSchema),
       platformConfigurationsSchema: () => addCoreFields(platformConfigurationsSchema),
-      jobResultsSchema: () => addCoreFields(jobResultsSchema)
+      jobResultsSchema: () => addCoreFields(jobResultsSchema),
+      promptLibrarySchema: () => addCoreFields(promptLibrarySchema),
+      promptFavoritesSchema: () => addCoreFields(promptFavoritesSchema)
     };
   }
 };
@@ -1830,6 +1895,7 @@ input FilterOperatorJSON {
   eq: JSON
   ne: JSON
   in: [JSON]
+  contains: JSON
 }
 
 input SortBy {
@@ -2331,7 +2397,9 @@ function createMutations(table, agents, contexts, tools, config) {
           source: source.id,
           context: exists.id,
           type: "source",
-          inputs: args.inputs
+          inputs: args.inputs,
+          user: context.user.id,
+          role: context.user.role?.id
         });
         console.log("[EXULU] Source function job scheduled", job.id);
         return {
@@ -2360,6 +2428,15 @@ function createMutations(table, agents, contexts, tools, config) {
           console.log(`[EXULU] created item through source update job ${createdItem.id}`);
         }
       }
+      await updateStatistic({
+        name: "count",
+        label: source.id,
+        type: STATISTICS_TYPE_ENUM.SOURCE_UPDATE,
+        trigger: "api",
+        count: 1,
+        user: context?.user?.id,
+        role: context?.user?.role?.id
+      });
       return {
         message: "Items created successfully.",
         jobs,
@@ -2394,7 +2471,7 @@ function createMutations(table, agents, contexts, tools, config) {
           jobs: jobs2.slice(0, 100)
         };
       }
-      query = applyFilters(query, args.where);
+      query = applyFilters(query, args.where, table);
       const items = await query;
       if (items.length === 0) {
         throw new Error("No items found to generate chunks for.");
@@ -2429,7 +2506,7 @@ function createMutations(table, agents, contexts, tools, config) {
       }
       let query = db3.from(getTableName(id)).select("id");
       if (args.where) {
-        query = applyFilters(query, args.where);
+        query = applyFilters(query, args.where, table);
       }
       const items = await query;
       if (items.length === 0) {
@@ -2486,18 +2563,38 @@ var applyAccessControl = (table, user, query) => {
   }
   return query;
 };
-var converOperatorToQuery = (query, fieldName, operators) => {
+var converOperatorToQuery = (query, fieldName, operators, table) => {
+  const field = table?.fields.find((f) => f.name === fieldName);
+  const isJsonField = field?.type === "json";
   if (operators.eq !== void 0) {
-    query = query.where(fieldName, operators.eq);
+    if (isJsonField) {
+      query = query.whereRaw(`?? = ?::jsonb`, [fieldName, JSON.stringify(operators.eq)]);
+    } else {
+      query = query.where(fieldName, operators.eq);
+    }
   }
   if (operators.ne !== void 0) {
-    query = query.whereRaw(`?? IS DISTINCT FROM ?`, [fieldName, operators.ne]);
+    if (isJsonField) {
+      query = query.whereRaw(`?? IS DISTINCT FROM ?::jsonb`, [fieldName, JSON.stringify(operators.ne)]);
+    } else {
+      query = query.whereRaw(`?? IS DISTINCT FROM ?`, [fieldName, operators.ne]);
+    }
   }
   if (operators.in !== void 0) {
-    query = query.whereIn(fieldName, operators.in);
+    if (isJsonField) {
+      const conditions = operators.in.map((val) => `?? = ?::jsonb`).join(" OR ");
+      const bindings = operators.in.flatMap((val) => [fieldName, JSON.stringify(val)]);
+      query = query.whereRaw(`(${conditions})`, bindings);
+    } else {
+      query = query.whereIn(fieldName, operators.in);
+    }
   }
   if (operators.contains !== void 0) {
-    query = query.where(fieldName, "like", `%${operators.contains}%`);
+    if (isJsonField) {
+      query = query.whereRaw(`?? @> ?::jsonb`, [fieldName, JSON.stringify(operators.contains)]);
+    } else {
+      query = query.where(fieldName, "like", `%${operators.contains}%`);
+    }
   }
   if (operators.lte !== void 0) {
     query = query.where(fieldName, "<=", operators.lte);
@@ -2759,21 +2856,21 @@ var finalizeRequestedFields = async ({
   }
   return result;
 };
-var applyFilters = (query, filters) => {
+var applyFilters = (query, filters, table) => {
   filters.forEach((filter) => {
     Object.entries(filter).forEach(([fieldName, operators]) => {
       if (operators) {
         if (operators.and !== void 0) {
           operators.and.forEach((operator) => {
-            query = converOperatorToQuery(query, fieldName, operator);
+            query = converOperatorToQuery(query, fieldName, operator, table);
           });
         }
         if (operators.or !== void 0) {
           operators.or.forEach((operator) => {
-            query = converOperatorToQuery(query, fieldName, operator);
+            query = converOperatorToQuery(query, fieldName, operator, table);
           });
         }
-        query = converOperatorToQuery(query, fieldName, operators);
+        query = converOperatorToQuery(query, fieldName, operators, table);
       }
     });
   });
@@ -2813,7 +2910,7 @@ function createQueries(table, agents, tools, contexts) {
       const requestedFields = getRequestedFields(info);
       const sanitizedFields = sanitizeRequestedFields(table, requestedFields);
       let query = db3.from(tableNamePlural).select(sanitizedFields);
-      query = applyFilters(query, filters);
+      query = applyFilters(query, filters, table);
       query = applyAccessControl(table, context.user, query);
       query = applySorting(query, sort);
       let result = await query.first();
@@ -2826,7 +2923,7 @@ function createQueries(table, agents, tools, contexts) {
         throw new Error("Limit cannot be greater than 500.");
       }
       let countQuery = db3(tableNamePlural);
-      countQuery = applyFilters(countQuery, filters);
+      countQuery = applyFilters(countQuery, filters, table);
       countQuery = applyAccessControl(table, context.user, countQuery);
       const countResult = await countQuery.count("* as count");
       const itemCount = Number(countResult[0]?.count || 0);
@@ -2835,7 +2932,7 @@ function createQueries(table, agents, tools, contexts) {
       const hasPreviousPage = currentPage > 1;
       const hasNextPage = currentPage < pageCount - 1;
       let dataQuery = db3(tableNamePlural);
-      dataQuery = applyFilters(dataQuery, filters);
+      dataQuery = applyFilters(dataQuery, filters, table);
       dataQuery = applyAccessControl(table, context.user, dataQuery);
       const requestedFields = getRequestedFields(info);
       dataQuery = applySorting(dataQuery, sort);
@@ -2860,7 +2957,7 @@ function createQueries(table, agents, tools, contexts) {
       const { filters = [], groupBy } = args;
       const { db: db3 } = context;
       let query = db3(tableNamePlural);
-      query = applyFilters(query, filters);
+      query = applyFilters(query, filters, table);
       query = applyAccessControl(table, context.user, query);
       if (groupBy) {
         query = query.select(groupBy).groupBy(groupBy);
@@ -2960,11 +3057,11 @@ var vectorSearch = async ({
   const mainTable = getTableName(id);
   const chunksTable = getChunksTableName(id);
   let countQuery = db3(mainTable);
-  countQuery = applyFilters(countQuery, filters);
+  countQuery = applyFilters(countQuery, filters, table);
   countQuery = applyAccessControl(table, user, countQuery);
   const columns = await db3(mainTable).columnInfo();
   let itemsQuery = db3(mainTable).select(Object.keys(columns).map((column) => mainTable + "." + column));
-  itemsQuery = applyFilters(itemsQuery, filters);
+  itemsQuery = applyFilters(itemsQuery, filters, table);
   itemsQuery = applyAccessControl(table, user, itemsQuery);
   itemsQuery = applySorting(itemsQuery, sort);
   if (queryRewriter) {
@@ -3725,7 +3822,7 @@ type PageInfo {
     const {
       jobs,
       count
-    } = await getJobsByQueueAndName(
+    } = await getJobsByQueueName(
       args.queue,
       args.statusses,
       args.page || 1,
@@ -4158,7 +4255,7 @@ var validateCreateOrRemoveSuperAdminPermission = async (tableNamePlural, input,
     }
   }
 };
-async function getJobsByQueueAndName(queueName, statusses, page, limit) {
+async function getJobsByQueueName(queueName, statusses, page, limit) {
   const queue = queues.list.get(queueName);
   if (!queue) {
     throw new Error(`Queue ${queueName} not found`);
@@ -5648,6 +5745,9 @@ var ExuluContext = class {
     };
   };
   createItem = async (item, config, user, role, upsert) => {
+    if (upsert && (!item.id && !item.external_id)) {
+      throw new Error("Item id or external id is required for upsert.");
+    }
     const { db: db3 } = await postgresClient();
     const mutation = db3.from(getTableName(
       this.id
@@ -5932,6 +6032,7 @@ var ExuluContext = class {
   };
 };
 var updateStatistic = async (statistic) => {
+  console.log("[EXULU] updating statistic", statistic);
   const currentDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
   const { db: db3 } = await postgresClient();
   const existing = await db3.from("tracking").where({
@@ -6075,6 +6176,8 @@ var {
   variablesSchema: variablesSchema2,
   workflowTemplatesSchema: workflowTemplatesSchema2,
   rbacSchema: rbacSchema2,
+  promptLibrarySchema: promptLibrarySchema2,
+  promptFavoritesSchema: promptFavoritesSchema2,
   statisticsSchema: statisticsSchema2
 } = coreSchemas.get();
 var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer, queues2) => {
@@ -6105,6 +6208,8 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tr
     agentsSchema2(),
     projectsSchema2(),
     jobResultsSchema2(),
+    promptLibrarySchema2(),
+    promptFavoritesSchema2(),
     evalRunsSchema2(),
     platformConfigurationsSchema2(),
     evalSetsSchema2(),
@@ -6515,7 +6620,7 @@ Mood: friendly and intelligent.
         return;
       }
       let project = null;
-      if (!req.body.project || req.body.project === "DEFAULT") {
+      if (!req.params.project || req.params.project === "DEFAULT") {
         project = null;
       } else {
         let projectQuery = db3("projects");
@@ -6792,7 +6897,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
         }));
         const { db: db3 } = await postgresClient();
         const data = bullmqJob.data;
-        const timeoutMs = data.timeoutInSeconds * 1e3;
+        const timeoutMs = (data.timeoutInSeconds || 300) * 1e3;
         const timeoutPromise = new Promise((_, reject) => {
           setTimeout(() => {
             reject(new Error(`Timeout for job ${bullmqJob.id} reached after ${data.timeoutInSeconds}s`));
@@ -6883,6 +6988,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               const label = `eval-run-${data.eval_run_id}-${data.test_case_id}`;
               const existingResult = await db3.from("job_results").where({ label }).first();
               if (existingResult) {
+                console.log("[EXULU] found existing job result, so ");
                 await db3.from("job_results").where({ label }).update({
                   job_id: bullmqJob.id,
                   label,
@@ -6984,6 +7090,8 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                     test_case_id: testCase.id,
                     eval_run_id: evalRun.id,
                     eval_function_id: evalFunction.id,
+                    eval_function_name: evalFunction.name,
+                    eval_function_config: evalFunction.config || {},
                     result: result2 || 0
                   };
                   console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata2(bullmqJob.name, {
@@ -7011,22 +7119,32 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 }
               }
               const scores = evalFunctionResults.map((result2) => result2.result);
+              console.log("[EXULU] Exulu eval run scores for test case: " + testCase.id, scores);
               let score = 0;
-              switch (data.scoring_method) {
+              switch (data.scoring_method?.toLowerCase()) {
                 case "median":
+                  console.log("[EXULU] Calculating median score");
                   score = getMedian(scores);
                   break;
                 case "average":
+                  console.log("[EXULU] Calculating average score");
                   score = getAverage(scores);
                   break;
                 case "sum":
+                  console.log("[EXULU] Calculating sum score");
                   score = getSum(scores);
                   break;
+                default:
+                  console.log("[EXULU] Calculating average score");
+                  score = getAverage(scores);
               }
               return {
                 result: score,
                 metadata: {
-                  ...evalFunctionResults,
+                  messages,
+                  function_results: [
+                    ...evalFunctionResults
+                  ],
                   ...metadata
                 }
               };
@@ -7122,6 +7240,15 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                   }));
                 }
               }
+              await updateStatistic({
+                name: "count",
+                label: source.id,
+                type: STATISTICS_TYPE_ENUM.SOURCE_UPDATE,
+                trigger: "api",
+                count: 1,
+                user: data?.user,
+                role: data?.role
+              });
               return {
                 result,
                 metadata: {
@@ -7559,6 +7686,92 @@ var ExuluMCP = class {
       });
       server.tools[tool2.id] = tool2.name;
     }
+    const getListOfPromptTemplatesName = "getListOfPromptTemplates";
+    if (!server.tools[getListOfPromptTemplatesName]) {
+      server.mcp.registerTool(getListOfPromptTemplatesName, {
+        title: "Get List of Prompt Templates",
+        description: "Retrieves a list of prompt templates available for this agent. Returns the name, description, and ID of each template.",
+        inputSchema: {
+          inputs: z2.object({})
+        }
+      }, async ({ inputs }, args) => {
+        console.log("[EXULU] Getting list of prompt templates for agent", agentInstance.id);
+        const { db: db4 } = await postgresClient();
+        const prompts = await db4.from("prompt_library").select("id", "name", "description").whereRaw("assigned_agents @> ?::jsonb", [JSON.stringify(agentInstance.id)]).orderBy("updatedAt", "desc");
+        console.log("[EXULU] Found", prompts.length, "prompt templates");
+        return {
+          content: [{
+            type: "text",
+            text: JSON.stringify({
+              prompts: prompts.map((p) => ({
+                id: p.id,
+                name: p.name,
+                description: p.description || "No description provided"
+              })),
+              count: prompts.length
+            }, null, 2)
+          }],
+          structuredContent: {
+            prompts: prompts.map((p) => ({
+              id: p.id,
+              name: p.name,
+              description: p.description || "No description provided"
+            })),
+            count: prompts.length
+          }
+        };
+      });
+    }
+    const getPromptTemplateDetailsName = "getPromptTemplateDetails";
+    if (!server.tools[getPromptTemplateDetailsName]) {
+      server.mcp.registerTool(getPromptTemplateDetailsName, {
+        title: "Get Prompt Template Details",
+        description: "Retrieves the full details of a specific prompt template by ID, including the actual template content with variables.",
+        inputSchema: {
+          inputs: z2.object({
+            id: z2.string().describe("The ID of the prompt template to retrieve")
+          })
+        }
+      }, async ({ inputs }, args) => {
+        console.log("[EXULU] Getting prompt template details for ID", inputs.id);
+        const { db: db4 } = await postgresClient();
+        const prompt = await db4.from("prompt_library").select("id", "name", "description", "content", "createdAt", "updatedAt", "usage_count", "favorite_count").where({ id: inputs.id }).first();
+        if (!prompt) {
+          throw new Error(`Prompt template with ID ${inputs.id} not found`);
+        }
+        const isAssignedToAgent = await db4.from("prompt_library").select("id").where({ id: inputs.id }).whereRaw("assigned_agents @> ?::jsonb", [JSON.stringify(agentInstance.id)]).first();
+        console.log("[EXULU] Prompt template found:", prompt.name);
+        return {
+          content: [{
+            type: "text",
+            text: JSON.stringify({
+              id: prompt.id,
+              name: prompt.name,
+              description: prompt.description || "No description provided",
+              content: prompt.content,
+              createdAt: prompt.createdAt,
+              updatedAt: prompt.updatedAt,
+              usageCount: prompt.usage_count || 0,
+              favoriteCount: prompt.favorite_count || 0,
+              isAssignedToThisAgent: !!isAssignedToAgent
+            }, null, 2)
+          }],
+          structuredContent: {
+            id: prompt.id,
+            name: prompt.name,
+            description: prompt.description || "No description provided",
+            content: prompt.content,
+            createdAt: prompt.createdAt,
+            updatedAt: prompt.updatedAt,
+            usageCount: prompt.usage_count || 0,
+            favoriteCount: prompt.favorite_count || 0,
+            isAssignedToThisAgent: !!isAssignedToAgent
+          }
+        };
+      });
+    }
+    server.tools[getListOfPromptTemplatesName] = getListOfPromptTemplatesName;
+    server.tools[getPromptTemplateDetailsName] = getPromptTemplateDetailsName;
     return server.mcp;
   };
   create = async ({ express: express3, allTools, allAgents, allContexts, config }) => {
@@ -8173,8 +8386,8 @@ var llmAsJudgeEval = () => {
         console.log("[EXULU] running llm as judge eval", { agent, backend, messages, testCase, config });
         let prompt = config?.prompt;
         if (!prompt) {
-          console.error("[EXULU] prompt is required.");
-          throw new Error("Prompt is required.");
+          console.error("[EXULU] prompt is required for llm as judge eval but none is provided.");
+          throw new Error("Prompt is required for llm as judge eval but none is provided.");
         }
         const lastMessage = messages[messages.length - 1]?.parts?.filter((part) => part.type === "text").map((part) => part.text).join("\n");
         console.log("[EXULU] last message", lastMessage);
@@ -10639,7 +10852,9 @@ var {
   workflowTemplatesSchema: workflowTemplatesSchema3,
   rbacSchema: rbacSchema3,
   projectsSchema: projectsSchema3,
-  jobResultsSchema: jobResultsSchema3
+  jobResultsSchema: jobResultsSchema3,
+  promptLibrarySchema: promptLibrarySchema3,
+  promptFavoritesSchema: promptFavoritesSchema3
 } = coreSchemas.get();
 var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
   for (const field of fields) {
@@ -10674,6 +10889,8 @@ var up = async function(knex) {
     statisticsSchema3(),
     projectsSchema3(),
     jobResultsSchema3(),
+    promptLibrarySchema3(),
+    promptFavoritesSchema3(),
     rbacSchema3(),
     agentsSchema3(),
     variablesSchema3(),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
   "author": "Qventu Bv.",
-  "version": "1.33.0",
+  "version": "1.34.1",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {

package/types/models/eval-run.ts

@@ -1,15 +1,18 @@
 export type ScoringMethod = "median" | "sum" | "average"
 
+export interface EvalRunEvalFunction {
+  id: string
+  name: string,
+  config: Record<string, any>
+}
+
 export interface EvalRun {
   id: string
   name: string
   eval_set_id: string
   agent_id: string
   timeout_in_seconds: number
-  eval_functions: {
-    id: string
-    config: Record<string, any>
-  }[]
+  eval_functions: EvalRunEvalFunction[]
   config?: Record<string, any> // Optional config for eval functions
   scoring_method: ScoringMethod
   pass_threshold: number // 0-100 percentage