@exulu/backend 1.42.2 → 1.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1435,6 +1435,10 @@ var addCoreFields = (schema) => {
1435
1435
  name: "last_processed_at",
1436
1436
  type: "date"
1437
1437
  });
1438
+ schema.fields.push({
1439
+ name: "embeddings_updated_at",
1440
+ type: "date"
1441
+ });
1438
1442
  if (schema.RBAC) {
1439
1443
  if (!schema.fields.some((field) => field.name === "rights_mode")) {
1440
1444
  schema.fields.push({
@@ -1648,7 +1652,8 @@ var checkRecordAccess = async (record, request, user) => {
1648
1652
  const isPublic = record.rights_mode === "public";
1649
1653
  const byUsers = record.rights_mode === "users";
1650
1654
  const byRoles = record.rights_mode === "roles";
1651
- const isCreator = user ? record.created_by === user.id.toString() : false;
1655
+ const createdBy = typeof record.created_by === "string" ? record.created_by : record.created_by?.toString();
1656
+ const isCreator = user ? createdBy === user.id.toString() : false;
1652
1657
  const isAdmin = user ? user.super_admin : false;
1653
1658
  const isApi = user ? user.type === "api" : false;
1654
1659
  let hasAccess = "none";
@@ -1855,6 +1860,7 @@ ${enumValues}
1855
1860
  fields.push(" maxContextLength: Int");
1856
1861
  fields.push(" provider: String");
1857
1862
  fields.push(" authenticationInformation: String");
1863
+ fields.push(" systemInstructions: String");
1858
1864
  fields.push(" slug: String");
1859
1865
  }
1860
1866
  const rbacField = table.RBAC ? " RBAC: RBACData" : "";
@@ -1919,6 +1925,8 @@ input FilterOperatorDate {
1919
1925
  input FilterOperatorFloat {
1920
1926
  eq: Float
1921
1927
  ne: Float
1928
+ lte: Float
1929
+ gte: Float
1922
1930
  in: [Float]
1923
1931
  and: [FilterOperatorFloat]
1924
1932
  or: [FilterOperatorFloat]
@@ -1982,7 +1990,11 @@ var getRequestedFields = (info) => {
1982
1990
  return fields.filter((field) => field !== "pageInfo" && field !== "items" && field !== "RBAC");
1983
1991
  };
1984
1992
  var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRbacRecords) => {
1985
- const { users = [], roles = [], projects = [] } = rbacData;
1993
+ const {
1994
+ users = [],
1995
+ roles = []
1996
+ /* projects = [] */
1997
+ } = rbacData;
1986
1998
  if (!existingRbacRecords) {
1987
1999
  existingRbacRecords = await db3.from("rbac").where({
1988
2000
  entity: entityName,
@@ -1991,25 +2003,19 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
1991
2003
  }
1992
2004
  const newUserRecords = new Set(users.map((u) => `${u.id}:${u.rights}`));
1993
2005
  const newRoleRecords = new Set(roles.map((r) => `${r.id}:${r.rights}`));
1994
- const newProjectRecords = new Set(projects.map((p) => `${p.id}:${p.rights}`));
1995
2006
  const existingUserRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "User").map((r) => `${r.user_id}:${r.rights}`));
1996
2007
  const existingRoleRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "Role").map((r) => `${r.role_id}:${r.rights}`));
1997
2008
  const existingProjectRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "Project").map((r) => `${r.project_id}:${r.rights}`));
1998
2009
  const usersToCreate = users.filter((u) => !existingUserRecords.has(`${u.id}:${u.rights}`));
1999
2010
  const rolesToCreate = roles.filter((r) => !existingRoleRecords.has(`${r.id}:${r.rights}`));
2000
- const projectsToCreate = projects.filter((p) => !existingProjectRecords.has(`${p.id}:${p.rights}`));
2001
2011
  const usersToRemove = existingRbacRecords.filter((r) => r.access_type === "User" && !newUserRecords.has(`${r.user_id}:${r.rights}`));
2002
2012
  const rolesToRemove = existingRbacRecords.filter((r) => r.access_type === "Role" && !newRoleRecords.has(`${r.role_id}:${r.rights}`));
2003
- const projectsToRemove = existingRbacRecords.filter((r) => r.access_type === "Project" && !newProjectRecords.has(`${r.project_id}:${r.rights}`));
2004
2013
  if (usersToRemove.length > 0) {
2005
2014
  await db3.from("rbac").whereIn("id", usersToRemove.map((r) => r.id)).del();
2006
2015
  }
2007
2016
  if (rolesToRemove.length > 0) {
2008
2017
  await db3.from("rbac").whereIn("id", rolesToRemove.map((r) => r.id)).del();
2009
2018
  }
2010
- if (projectsToRemove.length > 0) {
2011
- await db3.from("rbac").whereIn("id", projectsToRemove.map((r) => r.id)).del();
2012
- }
2013
2019
  const recordsToInsert = [];
2014
2020
  usersToCreate.forEach((user) => {
2015
2021
  recordsToInsert.push({
@@ -2033,17 +2039,6 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
2033
2039
  updatedAt: /* @__PURE__ */ new Date()
2034
2040
  });
2035
2041
  });
2036
- projectsToCreate.forEach((project) => {
2037
- recordsToInsert.push({
2038
- entity: entityName,
2039
- access_type: "Project",
2040
- target_resource_id: resourceId,
2041
- project_id: project.id,
2042
- rights: project.rights,
2043
- createdAt: /* @__PURE__ */ new Date(),
2044
- updatedAt: /* @__PURE__ */ new Date()
2045
- });
2046
- });
2047
2042
  if (recordsToInsert.length > 0) {
2048
2043
  await db3.from("rbac").insert(recordsToInsert);
2049
2044
  }
@@ -2057,7 +2052,7 @@ function createMutations(table, agents, contexts, tools, config) {
2057
2052
  if (user.super_admin === true) {
2058
2053
  return true;
2059
2054
  }
2060
- if (!user.super_admin && (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write") && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && user.role.evals === "write"))) {
2055
+ if (!user.super_admin && (table.name.plural === "agents" || table.name.plural === "workflow_templates" || table.name.plural === "variables" || table.name.plural === "users" || table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write") && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && user.role.evals === "write"))) {
2061
2056
  console.error("Access control error: no role found for current user or no access to entity type.");
2062
2057
  throw new Error("Access control error: no role found for current user or no access to entity type.");
2063
2058
  }
@@ -2111,6 +2106,61 @@ function createMutations(table, agents, contexts, tools, config) {
2111
2106
  }
2112
2107
  };
2113
2108
  const mutations = {
2109
+ [`${tableNamePlural}CopyOneById`]: async (_, args, context, info) => {
2110
+ const { db: db3 } = context;
2111
+ const requestedFields = getRequestedFields(info);
2112
+ let { id } = args;
2113
+ if (!id) {
2114
+ throw new Error("ID is required for copying a record.");
2115
+ }
2116
+ await validateWriteAccess(id, context);
2117
+ const item = await db3.from(tableNamePlural).select("*").where({ id }).first();
2118
+ if (!item) {
2119
+ throw new Error("Record not found");
2120
+ }
2121
+ if (item.rights_mode) {
2122
+ item.rights_mode = "private";
2123
+ }
2124
+ if (item.created_at) {
2125
+ item.created_at = /* @__PURE__ */ new Date();
2126
+ }
2127
+ if (item.createdAt) {
2128
+ item.createdAt = /* @__PURE__ */ new Date();
2129
+ }
2130
+ if (item.updated_at) {
2131
+ item.updated_at = /* @__PURE__ */ new Date();
2132
+ }
2133
+ if (item.updatedAt) {
2134
+ item.updatedAt = /* @__PURE__ */ new Date();
2135
+ }
2136
+ if (item.created_by) {
2137
+ item.created_by = context.user.id;
2138
+ }
2139
+ if (item.createdBy) {
2140
+ item.createdBy = context.user.id;
2141
+ }
2142
+ if (item.name) {
2143
+ item.name = item.name + " (Copy)";
2144
+ }
2145
+ Object.keys(item).forEach((key) => {
2146
+ if (table.fields.find((field) => field.name === key)?.type === "json") {
2147
+ if (typeof item[key] === "object" || Array.isArray(item[key])) {
2148
+ item[key] = JSON.stringify(item[key]);
2149
+ }
2150
+ }
2151
+ });
2152
+ const insert = db3(tableNamePlural).insert({
2153
+ ...item,
2154
+ id: db3.fn.uuid()
2155
+ }).returning("*");
2156
+ const result = await insert;
2157
+ if (!result[0]) {
2158
+ throw new Error("Failed to copy record.");
2159
+ }
2160
+ return {
2161
+ item: finalizeRequestedFields({ args, table, requestedFields, agents, contexts, tools, result: result[0], user: context.user })
2162
+ };
2163
+ },
2114
2164
  [`${tableNamePlural}CreateOne`]: async (_, args, context, info) => {
2115
2165
  const { db: db3 } = context;
2116
2166
  const requestedFields = getRequestedFields(info);
@@ -2368,7 +2418,7 @@ function createMutations(table, agents, contexts, tools, config) {
2368
2418
  }
2369
2419
  const { limit = 10, filters = [], sort } = args;
2370
2420
  const { db: db3 } = context;
2371
- const { items } = await paginationRequest({
2421
+ const { items } = await itemsPaginationRequest({
2372
2422
  db: db3,
2373
2423
  limit,
2374
2424
  page: 0,
@@ -2467,9 +2517,6 @@ function createMutations(table, agents, contexts, tools, config) {
2467
2517
  };
2468
2518
  };
2469
2519
  mutations[`${tableNameSingular}GenerateChunks`] = async (_, args, context, info) => {
2470
- if (!context.user?.super_admin) {
2471
- throw new Error("You are not authorized to generate chunks via API, user must be super admin.");
2472
- }
2473
2520
  const { db: db3 } = await postgresClient();
2474
2521
  const exists = contexts.find((context2) => context2.id === table.id);
2475
2522
  if (!exists) {
@@ -2480,13 +2527,17 @@ function createMutations(table, agents, contexts, tools, config) {
2480
2527
  const columns = await db3(mainTable).columnInfo();
2481
2528
  let query = db3.from(mainTable).select(Object.keys(columns));
2482
2529
  if (!args.where) {
2530
+ if (!context.user?.super_admin) {
2531
+ throw new Error("You are not authorized to generate all chunks via API, user must be super admin.");
2532
+ }
2483
2533
  const {
2484
2534
  jobs: jobs2,
2485
2535
  items: items2
2486
2536
  } = await embeddings.generate.all(
2487
2537
  config,
2488
2538
  context.user.id,
2489
- context.user.role?.id
2539
+ context.user.role?.id,
2540
+ args.limit
2490
2541
  );
2491
2542
  return {
2492
2543
  message: "Chunks generated successfully.",
@@ -2495,6 +2546,9 @@ function createMutations(table, agents, contexts, tools, config) {
2495
2546
  };
2496
2547
  }
2497
2548
  query = applyFilters(query, args.where, table);
2549
+ if (args.limit) {
2550
+ query = query.limit(args.limit);
2551
+ }
2498
2552
  const items = await query;
2499
2553
  if (items.length === 0) {
2500
2554
  throw new Error("No items found to generate chunks for.");
@@ -2519,9 +2573,6 @@ function createMutations(table, agents, contexts, tools, config) {
2519
2573
  };
2520
2574
  };
2521
2575
  mutations[`${tableNameSingular}DeleteChunks`] = async (_, args, context, info) => {
2522
- if (!context.user?.super_admin) {
2523
- throw new Error("You are not authorized to delete chunks via API, user must be super admin.");
2524
- }
2525
2576
  const { db: db3 } = await postgresClient();
2526
2577
  const id = contexts.find((context2) => context2.id === table.id)?.id;
2527
2578
  if (!id) {
@@ -2530,6 +2581,10 @@ function createMutations(table, agents, contexts, tools, config) {
2530
2581
  if (args.where) {
2531
2582
  let query = db3.from(getTableName(id)).select("id");
2532
2583
  query = applyFilters(query, args.where, table);
2584
+ query = applyAccessControl(table, query, context.user);
2585
+ if (args.limit) {
2586
+ query = query.limit(args.limit);
2587
+ }
2533
2588
  const items = await query;
2534
2589
  if (items.length === 0) {
2535
2590
  throw new Error("No items found to delete chunks for.");
@@ -2543,11 +2598,20 @@ function createMutations(table, agents, contexts, tools, config) {
2543
2598
  jobs: []
2544
2599
  };
2545
2600
  } else {
2546
- const count = await db3.from(getChunksTableName(id)).count();
2547
- await db3.from(getChunksTableName(id)).truncate();
2601
+ if (!context.user?.super_admin) {
2602
+ throw new Error("You are not authorized to delete all chunks via API, user must be super admin.");
2603
+ }
2604
+ let count = 0;
2605
+ if (!args.limit) {
2606
+ const result = await db3.from(getChunksTableName(id)).count();
2607
+ count = parseInt(result[0].count);
2608
+ await db3.from(getChunksTableName(id)).truncate();
2609
+ } else {
2610
+ count = await db3.from(getChunksTableName(id)).limit(args.limit).delete();
2611
+ }
2548
2612
  return {
2549
2613
  message: "Chunks deleted successfully.",
2550
- items: parseInt(count[0].count),
2614
+ items: count,
2551
2615
  jobs: []
2552
2616
  };
2553
2617
  }
@@ -2562,8 +2626,8 @@ var applyAccessControl = (table, query, user, field_prefix) => {
2562
2626
  }
2563
2627
  console.log("[EXULU] user.role", user?.role);
2564
2628
  console.log("[EXULU] table.name.plural", table.name.plural);
2565
- if (user && !user?.super_admin && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
2566
- console.error("==== Access control error: no role found or no access to entity type. ====");
2629
+ if (user && !user?.super_admin && (table.name.plural === "agents" || table.name.plural === "workflow_templates" || table.name.plural === "variables" || table.name.plural === "users" || table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
2630
+ console.error("==== Access control error: no role found or no access to entity type. ====", user, table.name.plural);
2567
2631
  throw new Error("Access control error: no role found or no access to entity type.");
2568
2632
  }
2569
2633
  const hasRBAC = table.RBAC === true;
@@ -2606,6 +2670,7 @@ var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) =
2606
2670
  const isJsonField = field?.type === "json";
2607
2671
  const prefix = field_prefix ? field_prefix + "." : "";
2608
2672
  fieldName = prefix + fieldName;
2673
+ console.log("[EXULU] operators", operators);
2609
2674
  if (operators.eq !== void 0) {
2610
2675
  if (isJsonField) {
2611
2676
  query = query.whereRaw(`?? = ?::jsonb`, [fieldName, JSON.stringify(operators.eq)]);
@@ -2637,7 +2702,13 @@ var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) =
2637
2702
  }
2638
2703
  }
2639
2704
  if (operators.lte !== void 0) {
2640
- query = query.where(fieldName, "<=", operators.lte);
2705
+ console.log("[EXULU] operators.lte", operators.lte);
2706
+ console.log("[EXULU] fieldName", fieldName);
2707
+ if (operators.lte === 0 || operators.lte === "0") {
2708
+ query = query.whereNull(fieldName).orWhere(fieldName, "=", 0);
2709
+ } else {
2710
+ query = query.where(fieldName, "<=", operators.lte);
2711
+ }
2641
2712
  }
2642
2713
  if (operators.gte !== void 0) {
2643
2714
  query = query.where(fieldName, ">=", operators.gte);
@@ -2653,7 +2724,8 @@ var backendAgentFields = [
2653
2724
  "capabilities",
2654
2725
  "maxContextLength",
2655
2726
  "provider",
2656
- "authenticationInformation"
2727
+ "authenticationInformation",
2728
+ "systemInstructions"
2657
2729
  ];
2658
2730
  var removeAgentFields = (requestedFields) => {
2659
2731
  const filtered = requestedFields.filter((field) => !backendAgentFields.includes(field));
@@ -2738,6 +2810,9 @@ var addAgentFields = async (args, requestedFields, agents, result, tools, user,
2738
2810
  if (requestedFields.includes("provider")) {
2739
2811
  result.provider = backend?.provider || "";
2740
2812
  }
2813
+ if (requestedFields.includes("systemInstructions")) {
2814
+ result.systemInstructions = backend?.config?.instructions || void 0;
2815
+ }
2741
2816
  if (!requestedFields.includes("backend")) {
2742
2817
  delete result.backend;
2743
2818
  }
@@ -2785,13 +2860,16 @@ var postprocessUpdate = async ({
2785
2860
  if (!context.embedder) {
2786
2861
  return result;
2787
2862
  }
2788
- const { db: db3 } = await postgresClient();
2789
- console.log("[EXULU] Deleting chunks for item", result.id);
2790
- await db3.from(getChunksTableName(context.id)).where({ source: result.id }).delete();
2791
- console.log("[EXULU] Deleted chunks for item", result.id);
2792
- console.log("[EXULU] Embedder", context.embedder);
2793
- console.log("[EXULU] Configuration", context.configuration);
2794
2863
  if (context.embedder && (context.configuration.calculateVectors === "onUpdate" || context.configuration.calculateVectors === "always")) {
2864
+ const { db: db3 } = await postgresClient();
2865
+ console.log("[EXULU] Deleting chunks for item", result.id);
2866
+ const exists = await context.chunksTableExists();
2867
+ if (exists) {
2868
+ await db3.from(getChunksTableName(context.id)).where({ source: result.id }).delete();
2869
+ console.log("[EXULU] Deleted chunks for item", result.id);
2870
+ }
2871
+ console.log("[EXULU] Embedder", context.embedder);
2872
+ console.log("[EXULU] Configuration", context.configuration);
2795
2873
  console.log("[EXULU] Generating embeddings for item", result.id);
2796
2874
  const { job } = await context.embeddings.generate.one({
2797
2875
  item: result,
@@ -2823,7 +2901,14 @@ var postprocessDeletion = async ({
2823
2901
  }
2824
2902
  if (Array.isArray(result)) {
2825
2903
  result = result.map((item) => {
2826
- return postprocessDeletion({ table, requestedFields, agents, contexts, tools, result: item });
2904
+ return postprocessDeletion({
2905
+ table,
2906
+ requestedFields,
2907
+ agents,
2908
+ contexts,
2909
+ tools,
2910
+ result: item
2911
+ });
2827
2912
  });
2828
2913
  } else {
2829
2914
  if (table.type === "items") {
@@ -2852,6 +2937,14 @@ var postprocessDeletion = async ({
2852
2937
  const { db: db3 } = await postgresClient();
2853
2938
  await db3.from("agent_messages").where({ session: result.id }).where({ session: result.id }).delete();
2854
2939
  }
2940
+ if (table.type === "eval_runs") {
2941
+ if (!result.id) {
2942
+ return result;
2943
+ }
2944
+ const { db: db3 } = await postgresClient();
2945
+ await db3.from("job_results").where({ label: { contains: result.id } }).del();
2946
+ await db3.from("eval_runs").where({ id: result.id }).del();
2947
+ }
2855
2948
  }
2856
2949
  return result;
2857
2950
  };
@@ -2962,7 +3055,7 @@ var applySorting = (query, sort, field_prefix) => {
2962
3055
  }
2963
3056
  return query;
2964
3057
  };
2965
- var paginationRequest = async ({
3058
+ var itemsPaginationRequest = async ({
2966
3059
  db: db3,
2967
3060
  limit,
2968
3061
  page,
@@ -2992,7 +3085,8 @@ var paginationRequest = async ({
2992
3085
  if (page > 1) {
2993
3086
  dataQuery = dataQuery.offset((page - 1) * limit);
2994
3087
  }
2995
- let items = await dataQuery.select(fields ? fields : "*").limit(limit);
3088
+ dataQuery = dataQuery.select(fields ? fields : "*").limit(limit);
3089
+ let items = await dataQuery;
2996
3090
  return {
2997
3091
  items,
2998
3092
  pageInfo: {
@@ -3043,7 +3137,7 @@ function createQueries(table, agents, tools, contexts) {
3043
3137
  const { limit = 10, page = 0, filters = [], sort } = args;
3044
3138
  const requestedFields = getRequestedFields(info);
3045
3139
  const sanitizedFields = sanitizeRequestedFields(table, requestedFields);
3046
- const { items, pageInfo } = await paginationRequest({
3140
+ const { items, pageInfo } = await itemsPaginationRequest({
3047
3141
  db: db3,
3048
3142
  limit,
3049
3143
  page,
@@ -3118,6 +3212,46 @@ function createQueries(table, agents, tools, contexts) {
3118
3212
  expand: args.expand
3119
3213
  });
3120
3214
  };
3215
+ queries[`${tableNameSingular}ChunkById`] = async (_, args, context, info) => {
3216
+ const exists = contexts.find((ctx) => ctx.id === table.id);
3217
+ if (!exists) {
3218
+ throw new Error("Context " + table.id + " not found in registry.");
3219
+ }
3220
+ const { db: db3 } = context;
3221
+ const chunksTable = getChunksTableName(exists.id);
3222
+ const mainTable = getTableName(exists.id);
3223
+ const chunk = await db3(chunksTable + " as chunks").select([
3224
+ "chunks.id as chunk_id",
3225
+ "chunks.source as chunk_source",
3226
+ "chunks.content as chunk_content",
3227
+ "chunks.chunk_index",
3228
+ "chunks.metadata as chunk_metadata",
3229
+ db3.raw('chunks."createdAt" as chunk_created_at'),
3230
+ db3.raw('chunks."updatedAt" as chunk_updated_at'),
3231
+ "items.id as item_id",
3232
+ "items.name as item_name",
3233
+ "items.external_id as item_external_id",
3234
+ db3.raw('items."updatedAt" as item_updated_at'),
3235
+ db3.raw('items."createdAt" as item_created_at')
3236
+ ]).leftJoin(mainTable + " as items", "chunks.source", "items.id").where("chunks.id", args.id).first();
3237
+ if (!chunk) {
3238
+ return null;
3239
+ }
3240
+ return {
3241
+ chunk_content: chunk.chunk_content,
3242
+ chunk_index: chunk.chunk_index,
3243
+ chunk_id: chunk.chunk_id,
3244
+ chunk_source: chunk.chunk_source,
3245
+ chunk_metadata: chunk.chunk_metadata,
3246
+ chunk_created_at: chunk.chunk_created_at,
3247
+ chunk_updated_at: chunk.chunk_updated_at,
3248
+ item_id: chunk.item_id,
3249
+ item_name: chunk.item_name,
3250
+ item_external_id: chunk.item_external_id,
3251
+ item_updated_at: chunk.item_updated_at,
3252
+ item_created_at: chunk.item_created_at
3253
+ };
3254
+ };
3121
3255
  }
3122
3256
  return queries;
3123
3257
  }
@@ -3250,120 +3384,52 @@ var vectorSearch = async ({
3250
3384
  const fullTextWeight = 2;
3251
3385
  const semanticWeight = 1;
3252
3386
  const rrfK = 50;
3253
- const hybridSQL = `
3254
- WITH full_text AS (
3255
- SELECT
3256
- chunks.id,
3257
- chunks.source,
3258
- row_number() OVER (
3259
- ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC
3260
- ) AS rank_ix
3261
- FROM ${chunksTable} as chunks
3262
- LEFT JOIN ${mainTable} as items ON items.id = chunks.source
3263
- WHERE chunks.fts @@ plainto_tsquery(?, ?)
3264
- AND ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?
3265
- AND (items.archived IS FALSE OR items.archived IS NULL)
3266
- ORDER BY rank_ix
3267
- LIMIT LEAST(?, 250) * 2
3268
- ),
3269
- semantic AS (
3270
- SELECT
3271
- chunks.id,
3272
- chunks.source,
3273
- row_number() OVER (
3274
- ORDER BY chunks.embedding <=> ${vectorExpr} ASC
3275
- ) AS rank_ix
3276
- FROM ${chunksTable} as chunks
3277
- LEFT JOIN ${mainTable} as items ON items.id = chunks.source
3278
- WHERE chunks.embedding IS NOT NULL
3279
- AND (1 - (chunks.embedding <=> ${vectorExpr})) >= ?
3280
- AND (items.archived IS FALSE OR items.archived IS NULL)
3281
- ORDER BY rank_ix
3282
- LIMIT LEAST(?, 250) * 2
3283
- )
3284
- SELECT
3285
- items.id as item_id,
3286
- items.name as item_name,
3287
- items.external_id as item_external_id,
3288
- chunks.id AS chunk_id,
3289
- chunks.source,
3290
- chunks.content,
3291
- chunks.chunk_index,
3292
- chunks.metadata,
3293
- chunks."createdAt" as chunk_created_at,
3294
- chunks."updatedAt" as chunk_updated_at,
3295
- items."updatedAt" as item_updated_at,
3296
- items."createdAt" as item_created_at,
3297
- /* Per-signal scores for introspection */
3298
- ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank,
3299
- (1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance,
3300
-
3301
- /* Hybrid RRF score */
3302
- (
3303
- COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
3304
- +
3305
- COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
3306
- )::float AS hybrid_score
3307
-
3308
- FROM full_text ft
3309
- FULL OUTER JOIN semantic se
3310
- ON ft.id = se.id
3311
- JOIN ${chunksTable} as chunks
3312
- ON COALESCE(ft.id, se.id) = chunks.id
3313
- JOIN ${mainTable} as items
3314
- ON items.id = chunks.source
3315
- WHERE (
3316
- COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
3317
- +
3318
- COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
3319
- ) >= ?
3320
- AND (chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)
3321
- AND (chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)
3322
- ORDER BY hybrid_score DESC
3323
- LIMIT LEAST(?, 250)
3324
- OFFSET 0
3325
- `;
3326
- const bindings = [
3327
- // full_text: plainto_tsquery(lang, query) in rank and where
3328
- language,
3329
- query,
3330
- language,
3331
- query,
3332
- language,
3333
- query,
3334
- cutoffs?.tsvector || 0,
3335
- // full_text tsvector cutoff
3336
- matchCount,
3337
- // full_text limit
3338
- cutoffs?.cosineDistance || 0,
3339
- // semantic cosine distance cutoff
3340
- matchCount,
3341
- // semantic limit
3342
- // fts_rank (ts_rank) call
3343
- language,
3344
- query,
3345
- // RRF fusion parameters
3346
- rrfK,
3347
- fullTextWeight,
3348
- rrfK,
3349
- semanticWeight,
3350
- // WHERE clause hybrid_score filter
3351
- rrfK,
3352
- fullTextWeight,
3353
- rrfK,
3354
- semanticWeight,
3355
- cutoffs?.hybrid || 0,
3356
- // Additional cutoff filters in main WHERE clause
3357
- language,
3358
- query,
3359
- cutoffs?.tsvector || 0,
3360
- // tsvector cutoff for results from semantic CTE
3361
- cutoffs?.cosineDistance || 0,
3362
- // cosine distance cutoff for results from full_text CTE
3363
- matchCount
3364
- // final limit
3365
- ];
3366
- resultChunks = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
3387
+ let fullTextQuery = db3(chunksTable + " as chunks").select([
3388
+ "chunks.id",
3389
+ "chunks.source",
3390
+ db3.raw(`row_number() OVER (ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC) AS rank_ix`, [language, query])
3391
+ ]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereRaw(`chunks.fts @@ plainto_tsquery(?, ?)`, [language, query]).whereRaw(`ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?`, [language, query, cutoffs?.tsvector || 0]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
3392
+ fullTextQuery = applyFilters(fullTextQuery, filters, table, "items");
3393
+ fullTextQuery = applyAccessControl(table, fullTextQuery, user, "items");
3394
+ let semanticQuery = db3(chunksTable + " as chunks").select([
3395
+ "chunks.id",
3396
+ "chunks.source",
3397
+ db3.raw(`row_number() OVER (ORDER BY chunks.embedding <=> ${vectorExpr} ASC) AS rank_ix`)
3398
+ ]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereNotNull("chunks.embedding").whereRaw(`(1 - (chunks.embedding <=> ${vectorExpr})) >= ?`, [cutoffs?.cosineDistance || 0]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
3399
+ semanticQuery = applyFilters(semanticQuery, filters, table, "items");
3400
+ semanticQuery = applyAccessControl(table, semanticQuery, user, "items");
3401
+ let hybridQuery = db3.with("full_text", fullTextQuery).with("semantic", semanticQuery).select([
3402
+ "items.id as item_id",
3403
+ "items.name as item_name",
3404
+ "items.external_id as item_external_id",
3405
+ "chunks.id as chunk_id",
3406
+ "chunks.source",
3407
+ "chunks.content",
3408
+ "chunks.chunk_index",
3409
+ "chunks.metadata",
3410
+ db3.raw('chunks."createdAt" as chunk_created_at'),
3411
+ db3.raw('chunks."updatedAt" as chunk_updated_at'),
3412
+ db3.raw('items."updatedAt" as item_updated_at'),
3413
+ db3.raw('items."createdAt" as item_created_at'),
3414
+ db3.raw(`ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank`, [language, query]),
3415
+ db3.raw(`(1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance`),
3416
+ db3.raw(`
3417
+ (
3418
+ COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
3419
+ +
3420
+ COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
3421
+ )::float AS hybrid_score
3422
+ `, [rrfK, fullTextWeight, rrfK, semanticWeight])
3423
+ ]).from("full_text as ft").fullOuterJoin("semantic as se", "ft.id", "se.id").join(chunksTable + " as chunks", function() {
3424
+ this.on(db3.raw("COALESCE(ft.id, se.id)"), "=", "chunks.id");
3425
+ }).join(mainTable + " as items", "items.id", "chunks.source").whereRaw(`
3426
+ (
3427
+ COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
3428
+ +
3429
+ COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
3430
+ ) >= ?
3431
+ `, [rrfK, fullTextWeight, rrfK, semanticWeight, cutoffs?.hybrid || 0]).whereRaw(`(chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)`, [language, query, cutoffs?.tsvector || 0]).whereRaw(`(chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)`, [cutoffs?.cosineDistance || 0]).orderByRaw("hybrid_score DESC").limit(Math.min(matchCount, 250));
3432
+ resultChunks = await hybridQuery;
3367
3433
  }
3368
3434
  console.log("[EXULU] Vector search chunk results:", resultChunks?.length);
3369
3435
  let results = resultChunks.map((chunk) => ({
@@ -3599,8 +3665,8 @@ var contextToTableDefinition = (context) => {
3599
3665
  type: "text"
3600
3666
  });
3601
3667
  definition.fields.push({
3602
- name: "embeddings_updated_at",
3603
- type: "date"
3668
+ name: "chunks_count",
3669
+ type: "number"
3604
3670
  });
3605
3671
  definition.fields.push({
3606
3672
  name: "name",
@@ -3703,10 +3769,13 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
3703
3769
  if (table.type === "items") {
3704
3770
  typeDefs += `
3705
3771
  ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}], cutoffs: SearchCutoffs, expand: SearchExpand): ${tableNameSingular}VectorSearchResult
3772
+ ${tableNameSingular}ChunkById(id: ID!): ${tableNameSingular}VectorSearchChunk
3706
3773
  `;
3707
3774
  }
3708
3775
  mutationDefs += `
3709
3776
  ${tableNamePlural}CreateOne(input: ${tableNameSingular}Input!, upsert: Boolean): ${tableNameSingular}MutationPayload
3777
+ ${tableNamePlural}CopyOneById(id: ID!): ${tableNameSingular}MutationPayload
3778
+
3710
3779
  ${tableNamePlural}UpdateOne(where: [Filter${tableNameSingularUpperCaseFirst}], input: ${tableNameSingular}Input!): ${tableNameSingular}MutationPayload
3711
3780
  ${tableNamePlural}UpdateOneById(id: ID!, input: ${tableNameSingular}Input!): ${tableNameSingular}MutationPayload
3712
3781
  ${tableNamePlural}RemoveOneById(id: ID!): ${tableNameSingular}
@@ -3714,9 +3783,9 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
3714
3783
  `;
3715
3784
  if (table.type === "items") {
3716
3785
  mutationDefs += `
3717
- ${tableNameSingular}GenerateChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}GenerateChunksReturnPayload
3786
+ ${tableNameSingular}GenerateChunks(where: [Filter${tableNameSingularUpperCaseFirst}], limit: Int): ${tableNameSingular}GenerateChunksReturnPayload
3718
3787
  ${tableNameSingular}ExecuteSource(source: ID!, inputs: JSON!): ${tableNameSingular}ExecuteSourceReturnPayload
3719
- ${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}DeleteChunksReturnPayload
3788
+ ${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}], limit: Int): ${tableNameSingular}DeleteChunksReturnPayload
3720
3789
  `;
3721
3790
  if (table.processor) {
3722
3791
  mutationDefs += `
@@ -3850,8 +3919,11 @@ type PageInfo {
3850
3919
  typeDefs += `
3851
3920
  contextById(id: ID!): Context
3852
3921
  `;
3922
+ typeDefs += `
3923
+ getUniquePromptTags: [String!]!
3924
+ `;
3853
3925
  mutationDefs += `
3854
- runEval(id: ID!, test_case_ids: [ID!]): RunEvalReturnPayload
3926
+ runEval(id: ID!, cases: [ID!]): RunEvalReturnPayload
3855
3927
  `;
3856
3928
  mutationDefs += `
3857
3929
  drainQueue(queue: QueueEnum!): JobActionReturnPayload
@@ -4307,6 +4379,38 @@ type PageInfo {
4307
4379
  array.push("agents");
4308
4380
  return [...new Set(array)].sort();
4309
4381
  };
4382
+ resolvers.Query["getUniquePromptTags"] = async (_, args, context, info) => {
4383
+ const { db: db3 } = context;
4384
+ const user = context.user;
4385
+ const promptTable = tables.find((t) => t.name.plural === "prompt_library");
4386
+ if (!promptTable) {
4387
+ throw new Error("Prompt library table not found");
4388
+ }
4389
+ let query = db3.from("prompt_library").select("tags");
4390
+ query = applyAccessControl(promptTable, query, user);
4391
+ const results = await query;
4392
+ const allTags = [];
4393
+ for (const row of results) {
4394
+ if (row.tags) {
4395
+ let tags = [];
4396
+ if (typeof row.tags === "string") {
4397
+ try {
4398
+ tags = JSON.parse(row.tags);
4399
+ } catch (e) {
4400
+ tags = [row.tags];
4401
+ }
4402
+ } else if (Array.isArray(row.tags)) {
4403
+ tags = row.tags;
4404
+ }
4405
+ tags.forEach((tag) => {
4406
+ if (tag && typeof tag === "string" && tag.trim()) {
4407
+ allTags.push(tag.trim().toLowerCase());
4408
+ }
4409
+ });
4410
+ }
4411
+ }
4412
+ return [...new Set(allTags)].sort();
4413
+ };
4310
4414
  modelDefs += `
4311
4415
  type ProviderPaginationResult {
4312
4416
  items: [Provider]!
@@ -4697,14 +4801,16 @@ var addBucketPrefixToKey = (key, bucket) => {
4697
4801
  }
4698
4802
  return `${bucket}/${key}`;
4699
4803
  };
4700
- var uploadFile = async (file, fileName, config, options = {}, user, customBucket) => {
4804
+ var uploadFile = async (file, fileName, config, options = {}, user, customBucket, global) => {
4701
4805
  if (!config.fileUploads) {
4702
4806
  throw new Error("File uploads are not configured (in the exported uploadFile function)");
4703
4807
  }
4704
4808
  const client2 = getS3Client(config);
4705
4809
  let defaultBucket = config.fileUploads.s3Bucket;
4706
4810
  let key = fileName;
4707
- key = addUserPrefixToKey(key, user || "api");
4811
+ if (!global) {
4812
+ key = addUserPrefixToKey(key, user || "api");
4813
+ }
4708
4814
  key = addGeneralPrefixToKey(key, config);
4709
4815
  const sanitizedMetadata = sanitizeMetadata(options.metadata);
4710
4816
  const command = new PutObjectCommand({
@@ -4855,7 +4961,7 @@ var createUppyRoutes = async (app, contexts, config) => {
4855
4961
  return;
4856
4962
  }
4857
4963
  let allowed = false;
4858
- if (user.type === "api" || user.super_admin || key.includes(`user_${user.id}/`)) {
4964
+ if (user.type === "api" || user.super_admin || !key.includes(`user_`) || key.includes(`user_${user.id}/`)) {
4859
4965
  allowed = true;
4860
4966
  }
4861
4967
  if (!allowed) {
@@ -4933,9 +5039,16 @@ var createUppyRoutes = async (app, contexts, config) => {
4933
5039
  return;
4934
5040
  }
4935
5041
  const client2 = getS3Client(config);
5042
+ let prefix = `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}`;
5043
+ if (!req.headers.global) {
5044
+ prefix += `user_${authenticationResult.user.id}`;
5045
+ } else {
5046
+ prefix += "global";
5047
+ }
5048
+ console.log("[EXULU] prefix", prefix);
4936
5049
  const command = new ListObjectsV2Command({
4937
5050
  Bucket: config.fileUploads.s3Bucket,
4938
- Prefix: `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}user_${authenticationResult.user.id}`,
5051
+ Prefix: prefix,
4939
5052
  MaxKeys: 9,
4940
5053
  ...req.query.continuationToken && { ContinuationToken: req.query.continuationToken }
4941
5054
  });
@@ -5018,7 +5131,13 @@ var createUppyRoutes = async (app, contexts, config) => {
5018
5131
  const { filename, contentType } = extractFileParameters(req);
5019
5132
  validateFileParameters(filename, contentType);
5020
5133
  const key = generateS3Key2(filename);
5021
- let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
5134
+ let fullKey = key;
5135
+ console.log("[EXULU] global", req.headers.global);
5136
+ if (!req.headers.global) {
5137
+ fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
5138
+ } else {
5139
+ fullKey = "global/" + key;
5140
+ }
5022
5141
  fullKey = addGeneralPrefixToKey(fullKey, config);
5023
5142
  console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
5024
5143
  getSignedUrl(
@@ -5074,7 +5193,13 @@ var createUppyRoutes = async (app, contexts, config) => {
5074
5193
  return res.status(400).json({ error: "s3: content type must be a string" });
5075
5194
  }
5076
5195
  const key = `${randomUUID()}-_EXULU_${filename}`;
5077
- let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
5196
+ let fullKey = key;
5197
+ console.log("[EXULU] global", req.headers.global);
5198
+ if (!req.headers.global) {
5199
+ fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
5200
+ } else {
5201
+ fullKey = "global/" + key;
5202
+ }
5078
5203
  fullKey = addGeneralPrefixToKey(fullKey, config);
5079
5204
  console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
5080
5205
  const params = {
@@ -5722,6 +5847,31 @@ var ExuluAgent2 = class {
5722
5847
  they are talking with the current date in mind as a reference.`;
5723
5848
  let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
5724
5849
  system += "\n\n" + genericContext;
5850
+ const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
5851
+ console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
5852
+ console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
5853
+ if (includesContextSearchTool) {
5854
+ system += `
5855
+
5856
+
5857
+
5858
+ When you use a context search tool, you will include references to the items
5859
+ retrieved from the context search tool inline in the response using this exact JSON format
5860
+ (all on one line, no line breaks):
5861
+ {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
5862
+
5863
+ IMPORTANT formatting rules:
5864
+ - Use the exact format shown above, all on ONE line
5865
+ - Do NOT use quotes around field names or values
5866
+ - Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
5867
+ - Include the file/item name, not the full path
5868
+ - Separate multiple citations with spaces
5869
+
5870
+ Example: {item_name: document.pdf, item_id: abc123, context: my-context-id, chunk_id: chunk_456, chunk_index: 0}
5871
+
5872
+ The citations will be rendered as interactive badges in the UI.
5873
+ `;
5874
+ }
5725
5875
  if (prompt) {
5726
5876
  let result = { object: null, text: "" };
5727
5877
  let inputTokens = 0;
@@ -5756,7 +5906,7 @@ var ExuluAgent2 = class {
5756
5906
  req,
5757
5907
  project
5758
5908
  ),
5759
- stopWhen: [stepCountIs(2)]
5909
+ stopWhen: [stepCountIs(5)]
5760
5910
  });
5761
5911
  result.text = text;
5762
5912
  inputTokens = totalUsage?.inputTokens || 0;
@@ -5817,7 +5967,7 @@ var ExuluAgent2 = class {
5817
5967
  req,
5818
5968
  project
5819
5969
  ),
5820
- stopWhen: [stepCountIs(2)]
5970
+ stopWhen: [stepCountIs(5)]
5821
5971
  });
5822
5972
  if (statistics) {
5823
5973
  await Promise.all([
@@ -5982,6 +6132,31 @@ ${extractedText}
5982
6132
  const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
5983
6133
  let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
5984
6134
  system += "\n\n" + genericContext;
6135
+ const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
6136
+ console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
6137
+ console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
6138
+ if (includesContextSearchTool) {
6139
+ system += `
6140
+
6141
+
6142
+
6143
+ When you use a context search tool, you will include references to the items
6144
+ retrieved from the context search tool inline in the response using this exact JSON format
6145
+ (all on one line, no line breaks):
6146
+ {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
6147
+
6148
+ IMPORTANT formatting rules:
6149
+ - Use the exact format shown above, all on ONE line
6150
+ - Do NOT use quotes around field names or values
6151
+ - Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
6152
+ - Include the file/item name, not the full path
6153
+ - Separate multiple citations with spaces
6154
+
6155
+ Example: {item_name: document.pdf, item_id: abc123, context: my-context-id, chunk_id: chunk_456, chunk_index: 0}
6156
+
6157
+ The citations will be rendered as interactive badges in the UI.
6158
+ `;
6159
+ }
5985
6160
  const result = streamText({
5986
6161
  model,
5987
6162
  // Should be a LanguageModelV1
@@ -6012,8 +6187,8 @@ ${extractedText}
6012
6187
  onError: (error) => {
6013
6188
  console.error("[EXULU] chat stream error.", error);
6014
6189
  throw new Error(`Chat stream error: ${error instanceof Error ? error.message : String(error)}`);
6015
- }
6016
- // stopWhen: [stepCountIs(1)],
6190
+ },
6191
+ stopWhen: [stepCountIs(5)]
6017
6192
  });
6018
6193
  return {
6019
6194
  stream: result,
@@ -6525,6 +6700,7 @@ var ExuluContext = class {
6525
6700
  })));
6526
6701
  }
6527
6702
  await db3.from(getTableName(this.id)).where({ id: item.id }).update({
6703
+ chunks_count: chunks?.length || 0,
6528
6704
  embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
6529
6705
  }).returning("id");
6530
6706
  return {
@@ -6540,6 +6716,13 @@ var ExuluContext = class {
6540
6716
  throw new Error("Item id or external id is required for upsert.");
6541
6717
  }
6542
6718
  const { db: db3 } = await postgresClient();
6719
+ Object.keys(item).forEach((key) => {
6720
+ if (this.fields.find((field) => field.name === key)?.type === "json") {
6721
+ if (typeof item[key] === "object" || Array.isArray(item[key])) {
6722
+ item[key] = JSON.stringify(item[key]);
6723
+ }
6724
+ }
6725
+ });
6543
6726
  const mutation = db3.from(getTableName(
6544
6727
  this.id
6545
6728
  )).insert(
@@ -6629,6 +6812,13 @@ var ExuluContext = class {
6629
6812
  if (!record) {
6630
6813
  throw new Error("Item not found.");
6631
6814
  }
6815
+ Object.keys(item).forEach((key) => {
6816
+ if (this.fields.find((field) => field.name === key)?.type === "json") {
6817
+ if (typeof item[key] === "object" || Array.isArray(item[key])) {
6818
+ item[key] = JSON.stringify(item[key]);
6819
+ }
6820
+ }
6821
+ });
6632
6822
  const mutation = db3.from(
6633
6823
  getTableName(this.id)
6634
6824
  ).where(
@@ -6794,9 +6984,13 @@ var ExuluContext = class {
6794
6984
  trigger: trigger || "agent"
6795
6985
  }, role, void 0);
6796
6986
  },
6797
- all: async (config, userId, roleId) => {
6987
+ all: async (config, userId, roleId, limit) => {
6798
6988
  const { db: db3 } = await postgresClient();
6799
- const items = await db3.from(getTableName(this.id)).select("*");
6989
+ let query = db3.from(getTableName(this.id)).select("*");
6990
+ if (limit) {
6991
+ query = query.limit(limit);
6992
+ }
6993
+ const items = await query;
6800
6994
  const jobs = [];
6801
6995
  const queue = await this.embedder?.queue;
6802
6996
  if (!queue?.queue.name && items.length > 2e3) {
@@ -6839,9 +7033,11 @@ var ExuluContext = class {
6839
7033
  table.text("created_by");
6840
7034
  table.text("ttl");
6841
7035
  table.text("rights_mode").defaultTo(this.configuration?.defaultRightsMode ?? "private");
7036
+ table.timestamp("embeddings_updated_at").defaultTo(null);
7037
+ table.timestamp("last_processed_at").defaultTo(null);
6842
7038
  table.integer("textlength");
6843
7039
  table.text("source");
6844
- table.timestamp("embeddings_updated_at");
7040
+ table.integer("chunks_count").defaultTo(0);
6845
7041
  for (const field of this.fields) {
6846
7042
  let { type, name, unique } = field;
6847
7043
  if (!type || !name) {
@@ -7370,12 +7566,6 @@ Mood: friendly and intelligent.
7370
7566
  return;
7371
7567
  }
7372
7568
  }
7373
- if (user?.type !== "api" && !user?.super_admin && req.body.resourceId !== user?.id) {
7374
- res.status(400).json({
7375
- message: "The provided user id in the resourceId field is not the same as the authenticated user. Only super admins and API users can impersonate other users."
7376
- });
7377
- return;
7378
- }
7379
7569
  console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
7380
7570
  const disabledTools = req.body.disabledTools ? req.body.disabledTools : [];
7381
7571
  let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
@@ -7838,6 +8028,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
7838
8028
  console.log("[EXULU] creating workers for " + queues2?.length + " queues.");
7839
8029
  console.log("[EXULU] queues", queues2.map((q) => q.queue.name));
7840
8030
  installGlobalErrorHandlers();
8031
+ process.setMaxListeners(Math.max(queues2.length * 2 + 5, 15));
7841
8032
  if (!redisServer.host || !redisServer.port) {
7842
8033
  console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
7843
8034
  throw new Error("No redis server configured in the environment, so cannot start worker.");
@@ -7870,7 +8061,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
7870
8061
  }));
7871
8062
  const { db: db3 } = await postgresClient();
7872
8063
  const data = bullmqJob.data;
7873
- const timeoutInSeconds = data.timeoutInSeconds || 600;
8064
+ const timeoutInSeconds = data.timeoutInSeconds || queue.timeoutInSeconds || 600;
7874
8065
  const timeoutMs = timeoutInSeconds * 1e3;
7875
8066
  let timeoutHandle;
7876
8067
  const timeoutPromise = new Promise((_, reject) => {
@@ -8074,7 +8265,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
8074
8265
  attempts: queue2.retries || 3,
8075
8266
  // todo make this configurable?
8076
8267
  removeOnComplete: 5e3,
8077
- removeOnFail: 1e4,
8268
+ removeOnFail: 5e3,
8078
8269
  backoff: queue2.backoff || {
8079
8270
  type: "exponential",
8080
8271
  delay: 2e3
@@ -8296,8 +8487,8 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
8296
8487
  const { db: db3 } = await postgresClient();
8297
8488
  await db3.from("job_results").where({ job_id: job.id }).update({
8298
8489
  state: JOB_STATUS_ENUM.completed,
8299
- result: returnvalue.result ? JSON.stringify(returnvalue.result) : null,
8300
- metadata: returnvalue.metadata ? JSON.stringify(returnvalue.metadata) : null
8490
+ result: returnvalue.result != null ? JSON.stringify(returnvalue.result) : null,
8491
+ metadata: returnvalue.metadata != null ? JSON.stringify(returnvalue.metadata) : null
8301
8492
  });
8302
8493
  });
8303
8494
  worker.on("failed", async (job, error, prev) => {
@@ -8408,9 +8599,11 @@ var pollJobResult = async ({ queue, jobId }) => {
8408
8599
  console.log(`[EXULU] eval function job ${job.name} completed, getting result from database...`);
8409
8600
  const { db: db3 } = await postgresClient();
8410
8601
  const entry = await db3.from("job_results").where({ job_id: job.id }).first();
8602
+ console.log("[EXULU] eval function job ${job.name} result", entry);
8411
8603
  result = entry?.result;
8412
8604
  if (result === void 0 || result === null || result === "") {
8413
- throw new Error(`Eval function ${job.id} result not found in database for job eval function job ${job.name}.`);
8605
+ throw new Error(`Eval function ${job.id} result not found in database
8606
+ for job eval function job ${job.name}. Entry data from DB: ${JSON.stringify(entry)}.`);
8414
8607
  }
8415
8608
  console.log(`[EXULU] eval function ${job.id} result: ${result}`);
8416
8609
  break;
@@ -9420,45 +9613,6 @@ var createLogger = ({
9420
9613
  };
9421
9614
  var logger_default = createLogger;
9422
9615
 
9423
- // src/templates/contexts/code-standards.ts
9424
- var codeStandardsContext = new ExuluContext({
9425
- id: "code_standards",
9426
- name: "Code Standards",
9427
- description: "Code standards that can be used with the Exulu CLI.",
9428
- configuration: {
9429
- defaultRightsMode: "public"
9430
- },
9431
- fields: [{
9432
- name: "Best practices",
9433
- type: "longText"
9434
- }, {
9435
- name: "Code style",
9436
- type: "longText"
9437
- }, {
9438
- name: "Tech stack",
9439
- type: "longText"
9440
- }],
9441
- active: true
9442
- });
9443
-
9444
- // src/templates/contexts/outputs.ts
9445
- var outputsContext = new ExuluContext({
9446
- id: "outputs_default_context",
9447
- name: "Outputs",
9448
- description: "Outputs from agent sessions that you have saved for re-used later.",
9449
- configuration: {
9450
- defaultRightsMode: "private",
9451
- calculateVectors: "manual"
9452
- },
9453
- fields: [
9454
- {
9455
- name: "content",
9456
- type: "longText"
9457
- }
9458
- ],
9459
- active: true
9460
- });
9461
-
9462
9616
  // src/registry/index.ts
9463
9617
  import winston2 from "winston";
9464
9618
  import util from "util";
@@ -9572,8 +9726,14 @@ var llmAsJudgeEval = () => {
9572
9726
  console.error("[EXULU] prompt is required for llm as judge eval but none is provided.");
9573
9727
  throw new Error("Prompt is required for llm as judge eval but none is provided.");
9574
9728
  }
9729
+ console.log("[EXULU] messages", messages);
9730
+ const lastTypes = messages[messages.length - 1]?.parts?.map((part) => ({
9731
+ type: part.type,
9732
+ text: part.type === "text" ? part.text?.slice(0, 100) : void 0
9733
+ }));
9575
9734
  const lastMessage = messages[messages.length - 1]?.parts?.filter((part) => part.type === "text").map((part) => part.text).join("\n");
9576
9735
  console.log("[EXULU] last message", lastMessage);
9736
+ console.log("[EXULU] last types", lastTypes);
9577
9737
  if (!lastMessage) {
9578
9738
  return 0;
9579
9739
  }
@@ -10585,9 +10745,7 @@ var ExuluApp = class {
10585
10745
  ...evals ?? []
10586
10746
  ] : [];
10587
10747
  this._contexts = {
10588
- ...contexts,
10589
- codeStandardsContext,
10590
- outputsContext
10748
+ ...contexts
10591
10749
  };
10592
10750
  this._agents = [
10593
10751
  claudeSonnet4Agent,
@@ -10644,8 +10802,8 @@ var ExuluApp = class {
10644
10802
  const queueSet = /* @__PURE__ */ new Set();
10645
10803
  if (redisServer.host?.length && redisServer.port?.length) {
10646
10804
  queues.register(global_queues.eval_runs, {
10647
- worker: 1,
10648
- queue: 1
10805
+ worker: 10,
10806
+ queue: 10
10649
10807
  }, 1);
10650
10808
  for (const queue of queues.list.values()) {
10651
10809
  const config2 = await queue.use();
@@ -12323,10 +12481,6 @@ var ExuluJobs = {
12323
12481
  validate: validateJob
12324
12482
  }
12325
12483
  };
12326
- var ExuluDefaultContexts = {
12327
- codeStandards: codeStandardsContext,
12328
- outputs: outputsContext
12329
- };
12330
12484
  var ExuluDefaultAgents = {
12331
12485
  anthropic: {
12332
12486
  opus4: claudeOpus4Agent,
@@ -12484,7 +12638,6 @@ export {
12484
12638
  ExuluChunkers,
12485
12639
  ExuluContext,
12486
12640
  ExuluDefaultAgents,
12487
- ExuluDefaultContexts,
12488
12641
  ExuluEmbedder,
12489
12642
  ExuluEval2 as ExuluEval,
12490
12643
  ExuluJobs,