@exulu/backend 1.42.2 → 1.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -38,7 +38,6 @@ __export(index_exports, {
38
38
  ExuluChunkers: () => ExuluChunkers,
39
39
  ExuluContext: () => ExuluContext,
40
40
  ExuluDefaultAgents: () => ExuluDefaultAgents,
41
- ExuluDefaultContexts: () => ExuluDefaultContexts,
42
41
  ExuluEmbedder: () => ExuluEmbedder,
43
42
  ExuluEval: () => ExuluEval2,
44
43
  ExuluJobs: () => ExuluJobs,
@@ -1487,6 +1486,10 @@ var addCoreFields = (schema) => {
1487
1486
  name: "last_processed_at",
1488
1487
  type: "date"
1489
1488
  });
1489
+ schema.fields.push({
1490
+ name: "embeddings_updated_at",
1491
+ type: "date"
1492
+ });
1490
1493
  if (schema.RBAC) {
1491
1494
  if (!schema.fields.some((field) => field.name === "rights_mode")) {
1492
1495
  schema.fields.push({
@@ -1700,7 +1703,8 @@ var checkRecordAccess = async (record, request, user) => {
1700
1703
  const isPublic = record.rights_mode === "public";
1701
1704
  const byUsers = record.rights_mode === "users";
1702
1705
  const byRoles = record.rights_mode === "roles";
1703
- const isCreator = user ? record.created_by === user.id.toString() : false;
1706
+ const createdBy = typeof record.created_by === "string" ? record.created_by : record.created_by?.toString();
1707
+ const isCreator = user ? createdBy === user.id.toString() : false;
1704
1708
  const isAdmin = user ? user.super_admin : false;
1705
1709
  const isApi = user ? user.type === "api" : false;
1706
1710
  let hasAccess = "none";
@@ -1907,6 +1911,7 @@ ${enumValues}
1907
1911
  fields.push(" maxContextLength: Int");
1908
1912
  fields.push(" provider: String");
1909
1913
  fields.push(" authenticationInformation: String");
1914
+ fields.push(" systemInstructions: String");
1910
1915
  fields.push(" slug: String");
1911
1916
  }
1912
1917
  const rbacField = table.RBAC ? " RBAC: RBACData" : "";
@@ -1971,6 +1976,8 @@ input FilterOperatorDate {
1971
1976
  input FilterOperatorFloat {
1972
1977
  eq: Float
1973
1978
  ne: Float
1979
+ lte: Float
1980
+ gte: Float
1974
1981
  in: [Float]
1975
1982
  and: [FilterOperatorFloat]
1976
1983
  or: [FilterOperatorFloat]
@@ -2034,7 +2041,11 @@ var getRequestedFields = (info) => {
2034
2041
  return fields.filter((field) => field !== "pageInfo" && field !== "items" && field !== "RBAC");
2035
2042
  };
2036
2043
  var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRbacRecords) => {
2037
- const { users = [], roles = [], projects = [] } = rbacData;
2044
+ const {
2045
+ users = [],
2046
+ roles = []
2047
+ /* projects = [] */
2048
+ } = rbacData;
2038
2049
  if (!existingRbacRecords) {
2039
2050
  existingRbacRecords = await db3.from("rbac").where({
2040
2051
  entity: entityName,
@@ -2043,25 +2054,19 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
2043
2054
  }
2044
2055
  const newUserRecords = new Set(users.map((u) => `${u.id}:${u.rights}`));
2045
2056
  const newRoleRecords = new Set(roles.map((r) => `${r.id}:${r.rights}`));
2046
- const newProjectRecords = new Set(projects.map((p) => `${p.id}:${p.rights}`));
2047
2057
  const existingUserRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "User").map((r) => `${r.user_id}:${r.rights}`));
2048
2058
  const existingRoleRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "Role").map((r) => `${r.role_id}:${r.rights}`));
2049
2059
  const existingProjectRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "Project").map((r) => `${r.project_id}:${r.rights}`));
2050
2060
  const usersToCreate = users.filter((u) => !existingUserRecords.has(`${u.id}:${u.rights}`));
2051
2061
  const rolesToCreate = roles.filter((r) => !existingRoleRecords.has(`${r.id}:${r.rights}`));
2052
- const projectsToCreate = projects.filter((p) => !existingProjectRecords.has(`${p.id}:${p.rights}`));
2053
2062
  const usersToRemove = existingRbacRecords.filter((r) => r.access_type === "User" && !newUserRecords.has(`${r.user_id}:${r.rights}`));
2054
2063
  const rolesToRemove = existingRbacRecords.filter((r) => r.access_type === "Role" && !newRoleRecords.has(`${r.role_id}:${r.rights}`));
2055
- const projectsToRemove = existingRbacRecords.filter((r) => r.access_type === "Project" && !newProjectRecords.has(`${r.project_id}:${r.rights}`));
2056
2064
  if (usersToRemove.length > 0) {
2057
2065
  await db3.from("rbac").whereIn("id", usersToRemove.map((r) => r.id)).del();
2058
2066
  }
2059
2067
  if (rolesToRemove.length > 0) {
2060
2068
  await db3.from("rbac").whereIn("id", rolesToRemove.map((r) => r.id)).del();
2061
2069
  }
2062
- if (projectsToRemove.length > 0) {
2063
- await db3.from("rbac").whereIn("id", projectsToRemove.map((r) => r.id)).del();
2064
- }
2065
2070
  const recordsToInsert = [];
2066
2071
  usersToCreate.forEach((user) => {
2067
2072
  recordsToInsert.push({
@@ -2085,17 +2090,6 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
2085
2090
  updatedAt: /* @__PURE__ */ new Date()
2086
2091
  });
2087
2092
  });
2088
- projectsToCreate.forEach((project) => {
2089
- recordsToInsert.push({
2090
- entity: entityName,
2091
- access_type: "Project",
2092
- target_resource_id: resourceId,
2093
- project_id: project.id,
2094
- rights: project.rights,
2095
- createdAt: /* @__PURE__ */ new Date(),
2096
- updatedAt: /* @__PURE__ */ new Date()
2097
- });
2098
- });
2099
2093
  if (recordsToInsert.length > 0) {
2100
2094
  await db3.from("rbac").insert(recordsToInsert);
2101
2095
  }
@@ -2109,7 +2103,7 @@ function createMutations(table, agents, contexts, tools, config) {
2109
2103
  if (user.super_admin === true) {
2110
2104
  return true;
2111
2105
  }
2112
- if (!user.super_admin && (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write") && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && user.role.evals === "write"))) {
2106
+ if (!user.super_admin && (table.name.plural === "agents" || table.name.plural === "workflow_templates" || table.name.plural === "variables" || table.name.plural === "users" || table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write") && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && user.role.evals === "write"))) {
2113
2107
  console.error("Access control error: no role found for current user or no access to entity type.");
2114
2108
  throw new Error("Access control error: no role found for current user or no access to entity type.");
2115
2109
  }
@@ -2163,6 +2157,61 @@ function createMutations(table, agents, contexts, tools, config) {
2163
2157
  }
2164
2158
  };
2165
2159
  const mutations = {
2160
+ [`${tableNamePlural}CopyOneById`]: async (_, args, context, info) => {
2161
+ const { db: db3 } = context;
2162
+ const requestedFields = getRequestedFields(info);
2163
+ let { id } = args;
2164
+ if (!id) {
2165
+ throw new Error("ID is required for copying a record.");
2166
+ }
2167
+ await validateWriteAccess(id, context);
2168
+ const item = await db3.from(tableNamePlural).select("*").where({ id }).first();
2169
+ if (!item) {
2170
+ throw new Error("Record not found");
2171
+ }
2172
+ if (item.rights_mode) {
2173
+ item.rights_mode = "private";
2174
+ }
2175
+ if (item.created_at) {
2176
+ item.created_at = /* @__PURE__ */ new Date();
2177
+ }
2178
+ if (item.createdAt) {
2179
+ item.createdAt = /* @__PURE__ */ new Date();
2180
+ }
2181
+ if (item.updated_at) {
2182
+ item.updated_at = /* @__PURE__ */ new Date();
2183
+ }
2184
+ if (item.updatedAt) {
2185
+ item.updatedAt = /* @__PURE__ */ new Date();
2186
+ }
2187
+ if (item.created_by) {
2188
+ item.created_by = context.user.id;
2189
+ }
2190
+ if (item.createdBy) {
2191
+ item.createdBy = context.user.id;
2192
+ }
2193
+ if (item.name) {
2194
+ item.name = item.name + " (Copy)";
2195
+ }
2196
+ Object.keys(item).forEach((key) => {
2197
+ if (table.fields.find((field) => field.name === key)?.type === "json") {
2198
+ if (typeof item[key] === "object" || Array.isArray(item[key])) {
2199
+ item[key] = JSON.stringify(item[key]);
2200
+ }
2201
+ }
2202
+ });
2203
+ const insert = db3(tableNamePlural).insert({
2204
+ ...item,
2205
+ id: db3.fn.uuid()
2206
+ }).returning("*");
2207
+ const result = await insert;
2208
+ if (!result[0]) {
2209
+ throw new Error("Failed to copy record.");
2210
+ }
2211
+ return {
2212
+ item: finalizeRequestedFields({ args, table, requestedFields, agents, contexts, tools, result: result[0], user: context.user })
2213
+ };
2214
+ },
2166
2215
  [`${tableNamePlural}CreateOne`]: async (_, args, context, info) => {
2167
2216
  const { db: db3 } = context;
2168
2217
  const requestedFields = getRequestedFields(info);
@@ -2420,7 +2469,7 @@ function createMutations(table, agents, contexts, tools, config) {
2420
2469
  }
2421
2470
  const { limit = 10, filters = [], sort } = args;
2422
2471
  const { db: db3 } = context;
2423
- const { items } = await paginationRequest({
2472
+ const { items } = await itemsPaginationRequest({
2424
2473
  db: db3,
2425
2474
  limit,
2426
2475
  page: 0,
@@ -2519,9 +2568,6 @@ function createMutations(table, agents, contexts, tools, config) {
2519
2568
  };
2520
2569
  };
2521
2570
  mutations[`${tableNameSingular}GenerateChunks`] = async (_, args, context, info) => {
2522
- if (!context.user?.super_admin) {
2523
- throw new Error("You are not authorized to generate chunks via API, user must be super admin.");
2524
- }
2525
2571
  const { db: db3 } = await postgresClient();
2526
2572
  const exists = contexts.find((context2) => context2.id === table.id);
2527
2573
  if (!exists) {
@@ -2532,13 +2578,17 @@ function createMutations(table, agents, contexts, tools, config) {
2532
2578
  const columns = await db3(mainTable).columnInfo();
2533
2579
  let query = db3.from(mainTable).select(Object.keys(columns));
2534
2580
  if (!args.where) {
2581
+ if (!context.user?.super_admin) {
2582
+ throw new Error("You are not authorized to generate all chunks via API, user must be super admin.");
2583
+ }
2535
2584
  const {
2536
2585
  jobs: jobs2,
2537
2586
  items: items2
2538
2587
  } = await embeddings.generate.all(
2539
2588
  config,
2540
2589
  context.user.id,
2541
- context.user.role?.id
2590
+ context.user.role?.id,
2591
+ args.limit
2542
2592
  );
2543
2593
  return {
2544
2594
  message: "Chunks generated successfully.",
@@ -2547,6 +2597,9 @@ function createMutations(table, agents, contexts, tools, config) {
2547
2597
  };
2548
2598
  }
2549
2599
  query = applyFilters(query, args.where, table);
2600
+ if (args.limit) {
2601
+ query = query.limit(args.limit);
2602
+ }
2550
2603
  const items = await query;
2551
2604
  if (items.length === 0) {
2552
2605
  throw new Error("No items found to generate chunks for.");
@@ -2571,9 +2624,6 @@ function createMutations(table, agents, contexts, tools, config) {
2571
2624
  };
2572
2625
  };
2573
2626
  mutations[`${tableNameSingular}DeleteChunks`] = async (_, args, context, info) => {
2574
- if (!context.user?.super_admin) {
2575
- throw new Error("You are not authorized to delete chunks via API, user must be super admin.");
2576
- }
2577
2627
  const { db: db3 } = await postgresClient();
2578
2628
  const id = contexts.find((context2) => context2.id === table.id)?.id;
2579
2629
  if (!id) {
@@ -2582,6 +2632,10 @@ function createMutations(table, agents, contexts, tools, config) {
2582
2632
  if (args.where) {
2583
2633
  let query = db3.from(getTableName(id)).select("id");
2584
2634
  query = applyFilters(query, args.where, table);
2635
+ query = applyAccessControl(table, query, context.user);
2636
+ if (args.limit) {
2637
+ query = query.limit(args.limit);
2638
+ }
2585
2639
  const items = await query;
2586
2640
  if (items.length === 0) {
2587
2641
  throw new Error("No items found to delete chunks for.");
@@ -2595,11 +2649,20 @@ function createMutations(table, agents, contexts, tools, config) {
2595
2649
  jobs: []
2596
2650
  };
2597
2651
  } else {
2598
- const count = await db3.from(getChunksTableName(id)).count();
2599
- await db3.from(getChunksTableName(id)).truncate();
2652
+ if (!context.user?.super_admin) {
2653
+ throw new Error("You are not authorized to delete all chunks via API, user must be super admin.");
2654
+ }
2655
+ let count = 0;
2656
+ if (!args.limit) {
2657
+ const result = await db3.from(getChunksTableName(id)).count();
2658
+ count = parseInt(result[0].count);
2659
+ await db3.from(getChunksTableName(id)).truncate();
2660
+ } else {
2661
+ count = await db3.from(getChunksTableName(id)).limit(args.limit).delete();
2662
+ }
2600
2663
  return {
2601
2664
  message: "Chunks deleted successfully.",
2602
- items: parseInt(count[0].count),
2665
+ items: count,
2603
2666
  jobs: []
2604
2667
  };
2605
2668
  }
@@ -2614,8 +2677,8 @@ var applyAccessControl = (table, query, user, field_prefix) => {
2614
2677
  }
2615
2678
  console.log("[EXULU] user.role", user?.role);
2616
2679
  console.log("[EXULU] table.name.plural", table.name.plural);
2617
- if (user && !user?.super_admin && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
2618
- console.error("==== Access control error: no role found or no access to entity type. ====");
2680
+ if (user && !user?.super_admin && (table.name.plural === "agents" || table.name.plural === "workflow_templates" || table.name.plural === "variables" || table.name.plural === "users" || table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
2681
+ console.error("==== Access control error: no role found or no access to entity type. ====", user, table.name.plural);
2619
2682
  throw new Error("Access control error: no role found or no access to entity type.");
2620
2683
  }
2621
2684
  const hasRBAC = table.RBAC === true;
@@ -2658,6 +2721,7 @@ var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) =
2658
2721
  const isJsonField = field?.type === "json";
2659
2722
  const prefix = field_prefix ? field_prefix + "." : "";
2660
2723
  fieldName = prefix + fieldName;
2724
+ console.log("[EXULU] operators", operators);
2661
2725
  if (operators.eq !== void 0) {
2662
2726
  if (isJsonField) {
2663
2727
  query = query.whereRaw(`?? = ?::jsonb`, [fieldName, JSON.stringify(operators.eq)]);
@@ -2689,7 +2753,13 @@ var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) =
2689
2753
  }
2690
2754
  }
2691
2755
  if (operators.lte !== void 0) {
2692
- query = query.where(fieldName, "<=", operators.lte);
2756
+ console.log("[EXULU] operators.lte", operators.lte);
2757
+ console.log("[EXULU] fieldName", fieldName);
2758
+ if (operators.lte === 0 || operators.lte === "0") {
2759
+ query = query.whereNull(fieldName).orWhere(fieldName, "=", 0);
2760
+ } else {
2761
+ query = query.where(fieldName, "<=", operators.lte);
2762
+ }
2693
2763
  }
2694
2764
  if (operators.gte !== void 0) {
2695
2765
  query = query.where(fieldName, ">=", operators.gte);
@@ -2705,7 +2775,8 @@ var backendAgentFields = [
2705
2775
  "capabilities",
2706
2776
  "maxContextLength",
2707
2777
  "provider",
2708
- "authenticationInformation"
2778
+ "authenticationInformation",
2779
+ "systemInstructions"
2709
2780
  ];
2710
2781
  var removeAgentFields = (requestedFields) => {
2711
2782
  const filtered = requestedFields.filter((field) => !backendAgentFields.includes(field));
@@ -2790,6 +2861,9 @@ var addAgentFields = async (args, requestedFields, agents, result, tools, user,
2790
2861
  if (requestedFields.includes("provider")) {
2791
2862
  result.provider = backend?.provider || "";
2792
2863
  }
2864
+ if (requestedFields.includes("systemInstructions")) {
2865
+ result.systemInstructions = backend?.config?.instructions || void 0;
2866
+ }
2793
2867
  if (!requestedFields.includes("backend")) {
2794
2868
  delete result.backend;
2795
2869
  }
@@ -2837,13 +2911,16 @@ var postprocessUpdate = async ({
2837
2911
  if (!context.embedder) {
2838
2912
  return result;
2839
2913
  }
2840
- const { db: db3 } = await postgresClient();
2841
- console.log("[EXULU] Deleting chunks for item", result.id);
2842
- await db3.from(getChunksTableName(context.id)).where({ source: result.id }).delete();
2843
- console.log("[EXULU] Deleted chunks for item", result.id);
2844
- console.log("[EXULU] Embedder", context.embedder);
2845
- console.log("[EXULU] Configuration", context.configuration);
2846
2914
  if (context.embedder && (context.configuration.calculateVectors === "onUpdate" || context.configuration.calculateVectors === "always")) {
2915
+ const { db: db3 } = await postgresClient();
2916
+ console.log("[EXULU] Deleting chunks for item", result.id);
2917
+ const exists = await context.chunksTableExists();
2918
+ if (exists) {
2919
+ await db3.from(getChunksTableName(context.id)).where({ source: result.id }).delete();
2920
+ console.log("[EXULU] Deleted chunks for item", result.id);
2921
+ }
2922
+ console.log("[EXULU] Embedder", context.embedder);
2923
+ console.log("[EXULU] Configuration", context.configuration);
2847
2924
  console.log("[EXULU] Generating embeddings for item", result.id);
2848
2925
  const { job } = await context.embeddings.generate.one({
2849
2926
  item: result,
@@ -2875,7 +2952,14 @@ var postprocessDeletion = async ({
2875
2952
  }
2876
2953
  if (Array.isArray(result)) {
2877
2954
  result = result.map((item) => {
2878
- return postprocessDeletion({ table, requestedFields, agents, contexts, tools, result: item });
2955
+ return postprocessDeletion({
2956
+ table,
2957
+ requestedFields,
2958
+ agents,
2959
+ contexts,
2960
+ tools,
2961
+ result: item
2962
+ });
2879
2963
  });
2880
2964
  } else {
2881
2965
  if (table.type === "items") {
@@ -2904,6 +2988,14 @@ var postprocessDeletion = async ({
2904
2988
  const { db: db3 } = await postgresClient();
2905
2989
  await db3.from("agent_messages").where({ session: result.id }).where({ session: result.id }).delete();
2906
2990
  }
2991
+ if (table.type === "eval_runs") {
2992
+ if (!result.id) {
2993
+ return result;
2994
+ }
2995
+ const { db: db3 } = await postgresClient();
2996
+ await db3.from("job_results").where({ label: { contains: result.id } }).del();
2997
+ await db3.from("eval_runs").where({ id: result.id }).del();
2998
+ }
2907
2999
  }
2908
3000
  return result;
2909
3001
  };
@@ -3014,7 +3106,7 @@ var applySorting = (query, sort, field_prefix) => {
3014
3106
  }
3015
3107
  return query;
3016
3108
  };
3017
- var paginationRequest = async ({
3109
+ var itemsPaginationRequest = async ({
3018
3110
  db: db3,
3019
3111
  limit,
3020
3112
  page,
@@ -3044,7 +3136,8 @@ var paginationRequest = async ({
3044
3136
  if (page > 1) {
3045
3137
  dataQuery = dataQuery.offset((page - 1) * limit);
3046
3138
  }
3047
- let items = await dataQuery.select(fields ? fields : "*").limit(limit);
3139
+ dataQuery = dataQuery.select(fields ? fields : "*").limit(limit);
3140
+ let items = await dataQuery;
3048
3141
  return {
3049
3142
  items,
3050
3143
  pageInfo: {
@@ -3095,7 +3188,7 @@ function createQueries(table, agents, tools, contexts) {
3095
3188
  const { limit = 10, page = 0, filters = [], sort } = args;
3096
3189
  const requestedFields = getRequestedFields(info);
3097
3190
  const sanitizedFields = sanitizeRequestedFields(table, requestedFields);
3098
- const { items, pageInfo } = await paginationRequest({
3191
+ const { items, pageInfo } = await itemsPaginationRequest({
3099
3192
  db: db3,
3100
3193
  limit,
3101
3194
  page,
@@ -3170,6 +3263,46 @@ function createQueries(table, agents, tools, contexts) {
3170
3263
  expand: args.expand
3171
3264
  });
3172
3265
  };
3266
+ queries[`${tableNameSingular}ChunkById`] = async (_, args, context, info) => {
3267
+ const exists = contexts.find((ctx) => ctx.id === table.id);
3268
+ if (!exists) {
3269
+ throw new Error("Context " + table.id + " not found in registry.");
3270
+ }
3271
+ const { db: db3 } = context;
3272
+ const chunksTable = getChunksTableName(exists.id);
3273
+ const mainTable = getTableName(exists.id);
3274
+ const chunk = await db3(chunksTable + " as chunks").select([
3275
+ "chunks.id as chunk_id",
3276
+ "chunks.source as chunk_source",
3277
+ "chunks.content as chunk_content",
3278
+ "chunks.chunk_index",
3279
+ "chunks.metadata as chunk_metadata",
3280
+ db3.raw('chunks."createdAt" as chunk_created_at'),
3281
+ db3.raw('chunks."updatedAt" as chunk_updated_at'),
3282
+ "items.id as item_id",
3283
+ "items.name as item_name",
3284
+ "items.external_id as item_external_id",
3285
+ db3.raw('items."updatedAt" as item_updated_at'),
3286
+ db3.raw('items."createdAt" as item_created_at')
3287
+ ]).leftJoin(mainTable + " as items", "chunks.source", "items.id").where("chunks.id", args.id).first();
3288
+ if (!chunk) {
3289
+ return null;
3290
+ }
3291
+ return {
3292
+ chunk_content: chunk.chunk_content,
3293
+ chunk_index: chunk.chunk_index,
3294
+ chunk_id: chunk.chunk_id,
3295
+ chunk_source: chunk.chunk_source,
3296
+ chunk_metadata: chunk.chunk_metadata,
3297
+ chunk_created_at: chunk.chunk_created_at,
3298
+ chunk_updated_at: chunk.chunk_updated_at,
3299
+ item_id: chunk.item_id,
3300
+ item_name: chunk.item_name,
3301
+ item_external_id: chunk.item_external_id,
3302
+ item_updated_at: chunk.item_updated_at,
3303
+ item_created_at: chunk.item_created_at
3304
+ };
3305
+ };
3173
3306
  }
3174
3307
  return queries;
3175
3308
  }
@@ -3302,120 +3435,52 @@ var vectorSearch = async ({
3302
3435
  const fullTextWeight = 2;
3303
3436
  const semanticWeight = 1;
3304
3437
  const rrfK = 50;
3305
- const hybridSQL = `
3306
- WITH full_text AS (
3307
- SELECT
3308
- chunks.id,
3309
- chunks.source,
3310
- row_number() OVER (
3311
- ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC
3312
- ) AS rank_ix
3313
- FROM ${chunksTable} as chunks
3314
- LEFT JOIN ${mainTable} as items ON items.id = chunks.source
3315
- WHERE chunks.fts @@ plainto_tsquery(?, ?)
3316
- AND ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?
3317
- AND (items.archived IS FALSE OR items.archived IS NULL)
3318
- ORDER BY rank_ix
3319
- LIMIT LEAST(?, 250) * 2
3320
- ),
3321
- semantic AS (
3322
- SELECT
3323
- chunks.id,
3324
- chunks.source,
3325
- row_number() OVER (
3326
- ORDER BY chunks.embedding <=> ${vectorExpr} ASC
3327
- ) AS rank_ix
3328
- FROM ${chunksTable} as chunks
3329
- LEFT JOIN ${mainTable} as items ON items.id = chunks.source
3330
- WHERE chunks.embedding IS NOT NULL
3331
- AND (1 - (chunks.embedding <=> ${vectorExpr})) >= ?
3332
- AND (items.archived IS FALSE OR items.archived IS NULL)
3333
- ORDER BY rank_ix
3334
- LIMIT LEAST(?, 250) * 2
3335
- )
3336
- SELECT
3337
- items.id as item_id,
3338
- items.name as item_name,
3339
- items.external_id as item_external_id,
3340
- chunks.id AS chunk_id,
3341
- chunks.source,
3342
- chunks.content,
3343
- chunks.chunk_index,
3344
- chunks.metadata,
3345
- chunks."createdAt" as chunk_created_at,
3346
- chunks."updatedAt" as chunk_updated_at,
3347
- items."updatedAt" as item_updated_at,
3348
- items."createdAt" as item_created_at,
3349
- /* Per-signal scores for introspection */
3350
- ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank,
3351
- (1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance,
3352
-
3353
- /* Hybrid RRF score */
3354
- (
3355
- COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
3356
- +
3357
- COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
3358
- )::float AS hybrid_score
3359
-
3360
- FROM full_text ft
3361
- FULL OUTER JOIN semantic se
3362
- ON ft.id = se.id
3363
- JOIN ${chunksTable} as chunks
3364
- ON COALESCE(ft.id, se.id) = chunks.id
3365
- JOIN ${mainTable} as items
3366
- ON items.id = chunks.source
3367
- WHERE (
3368
- COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
3369
- +
3370
- COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
3371
- ) >= ?
3372
- AND (chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)
3373
- AND (chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)
3374
- ORDER BY hybrid_score DESC
3375
- LIMIT LEAST(?, 250)
3376
- OFFSET 0
3377
- `;
3378
- const bindings = [
3379
- // full_text: plainto_tsquery(lang, query) in rank and where
3380
- language,
3381
- query,
3382
- language,
3383
- query,
3384
- language,
3385
- query,
3386
- cutoffs?.tsvector || 0,
3387
- // full_text tsvector cutoff
3388
- matchCount,
3389
- // full_text limit
3390
- cutoffs?.cosineDistance || 0,
3391
- // semantic cosine distance cutoff
3392
- matchCount,
3393
- // semantic limit
3394
- // fts_rank (ts_rank) call
3395
- language,
3396
- query,
3397
- // RRF fusion parameters
3398
- rrfK,
3399
- fullTextWeight,
3400
- rrfK,
3401
- semanticWeight,
3402
- // WHERE clause hybrid_score filter
3403
- rrfK,
3404
- fullTextWeight,
3405
- rrfK,
3406
- semanticWeight,
3407
- cutoffs?.hybrid || 0,
3408
- // Additional cutoff filters in main WHERE clause
3409
- language,
3410
- query,
3411
- cutoffs?.tsvector || 0,
3412
- // tsvector cutoff for results from semantic CTE
3413
- cutoffs?.cosineDistance || 0,
3414
- // cosine distance cutoff for results from full_text CTE
3415
- matchCount
3416
- // final limit
3417
- ];
3418
- resultChunks = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
3438
+ let fullTextQuery = db3(chunksTable + " as chunks").select([
3439
+ "chunks.id",
3440
+ "chunks.source",
3441
+ db3.raw(`row_number() OVER (ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC) AS rank_ix`, [language, query])
3442
+ ]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereRaw(`chunks.fts @@ plainto_tsquery(?, ?)`, [language, query]).whereRaw(`ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?`, [language, query, cutoffs?.tsvector || 0]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
3443
+ fullTextQuery = applyFilters(fullTextQuery, filters, table, "items");
3444
+ fullTextQuery = applyAccessControl(table, fullTextQuery, user, "items");
3445
+ let semanticQuery = db3(chunksTable + " as chunks").select([
3446
+ "chunks.id",
3447
+ "chunks.source",
3448
+ db3.raw(`row_number() OVER (ORDER BY chunks.embedding <=> ${vectorExpr} ASC) AS rank_ix`)
3449
+ ]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereNotNull("chunks.embedding").whereRaw(`(1 - (chunks.embedding <=> ${vectorExpr})) >= ?`, [cutoffs?.cosineDistance || 0]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
3450
+ semanticQuery = applyFilters(semanticQuery, filters, table, "items");
3451
+ semanticQuery = applyAccessControl(table, semanticQuery, user, "items");
3452
+ let hybridQuery = db3.with("full_text", fullTextQuery).with("semantic", semanticQuery).select([
3453
+ "items.id as item_id",
3454
+ "items.name as item_name",
3455
+ "items.external_id as item_external_id",
3456
+ "chunks.id as chunk_id",
3457
+ "chunks.source",
3458
+ "chunks.content",
3459
+ "chunks.chunk_index",
3460
+ "chunks.metadata",
3461
+ db3.raw('chunks."createdAt" as chunk_created_at'),
3462
+ db3.raw('chunks."updatedAt" as chunk_updated_at'),
3463
+ db3.raw('items."updatedAt" as item_updated_at'),
3464
+ db3.raw('items."createdAt" as item_created_at'),
3465
+ db3.raw(`ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank`, [language, query]),
3466
+ db3.raw(`(1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance`),
3467
+ db3.raw(`
3468
+ (
3469
+ COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
3470
+ +
3471
+ COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
3472
+ )::float AS hybrid_score
3473
+ `, [rrfK, fullTextWeight, rrfK, semanticWeight])
3474
+ ]).from("full_text as ft").fullOuterJoin("semantic as se", "ft.id", "se.id").join(chunksTable + " as chunks", function() {
3475
+ this.on(db3.raw("COALESCE(ft.id, se.id)"), "=", "chunks.id");
3476
+ }).join(mainTable + " as items", "items.id", "chunks.source").whereRaw(`
3477
+ (
3478
+ COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
3479
+ +
3480
+ COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
3481
+ ) >= ?
3482
+ `, [rrfK, fullTextWeight, rrfK, semanticWeight, cutoffs?.hybrid || 0]).whereRaw(`(chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)`, [language, query, cutoffs?.tsvector || 0]).whereRaw(`(chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)`, [cutoffs?.cosineDistance || 0]).orderByRaw("hybrid_score DESC").limit(Math.min(matchCount, 250));
3483
+ resultChunks = await hybridQuery;
3419
3484
  }
3420
3485
  console.log("[EXULU] Vector search chunk results:", resultChunks?.length);
3421
3486
  let results = resultChunks.map((chunk) => ({
@@ -3651,8 +3716,8 @@ var contextToTableDefinition = (context) => {
3651
3716
  type: "text"
3652
3717
  });
3653
3718
  definition.fields.push({
3654
- name: "embeddings_updated_at",
3655
- type: "date"
3719
+ name: "chunks_count",
3720
+ type: "number"
3656
3721
  });
3657
3722
  definition.fields.push({
3658
3723
  name: "name",
@@ -3755,10 +3820,13 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
3755
3820
  if (table.type === "items") {
3756
3821
  typeDefs += `
3757
3822
  ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}], cutoffs: SearchCutoffs, expand: SearchExpand): ${tableNameSingular}VectorSearchResult
3823
+ ${tableNameSingular}ChunkById(id: ID!): ${tableNameSingular}VectorSearchChunk
3758
3824
  `;
3759
3825
  }
3760
3826
  mutationDefs += `
3761
3827
  ${tableNamePlural}CreateOne(input: ${tableNameSingular}Input!, upsert: Boolean): ${tableNameSingular}MutationPayload
3828
+ ${tableNamePlural}CopyOneById(id: ID!): ${tableNameSingular}MutationPayload
3829
+
3762
3830
  ${tableNamePlural}UpdateOne(where: [Filter${tableNameSingularUpperCaseFirst}], input: ${tableNameSingular}Input!): ${tableNameSingular}MutationPayload
3763
3831
  ${tableNamePlural}UpdateOneById(id: ID!, input: ${tableNameSingular}Input!): ${tableNameSingular}MutationPayload
3764
3832
  ${tableNamePlural}RemoveOneById(id: ID!): ${tableNameSingular}
@@ -3766,9 +3834,9 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
3766
3834
  `;
3767
3835
  if (table.type === "items") {
3768
3836
  mutationDefs += `
3769
- ${tableNameSingular}GenerateChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}GenerateChunksReturnPayload
3837
+ ${tableNameSingular}GenerateChunks(where: [Filter${tableNameSingularUpperCaseFirst}], limit: Int): ${tableNameSingular}GenerateChunksReturnPayload
3770
3838
  ${tableNameSingular}ExecuteSource(source: ID!, inputs: JSON!): ${tableNameSingular}ExecuteSourceReturnPayload
3771
- ${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}DeleteChunksReturnPayload
3839
+ ${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}], limit: Int): ${tableNameSingular}DeleteChunksReturnPayload
3772
3840
  `;
3773
3841
  if (table.processor) {
3774
3842
  mutationDefs += `
@@ -3902,8 +3970,11 @@ type PageInfo {
3902
3970
  typeDefs += `
3903
3971
  contextById(id: ID!): Context
3904
3972
  `;
3973
+ typeDefs += `
3974
+ getUniquePromptTags: [String!]!
3975
+ `;
3905
3976
  mutationDefs += `
3906
- runEval(id: ID!, test_case_ids: [ID!]): RunEvalReturnPayload
3977
+ runEval(id: ID!, cases: [ID!]): RunEvalReturnPayload
3907
3978
  `;
3908
3979
  mutationDefs += `
3909
3980
  drainQueue(queue: QueueEnum!): JobActionReturnPayload
@@ -4359,6 +4430,38 @@ type PageInfo {
4359
4430
  array.push("agents");
4360
4431
  return [...new Set(array)].sort();
4361
4432
  };
4433
+ resolvers.Query["getUniquePromptTags"] = async (_, args, context, info) => {
4434
+ const { db: db3 } = context;
4435
+ const user = context.user;
4436
+ const promptTable = tables.find((t) => t.name.plural === "prompt_library");
4437
+ if (!promptTable) {
4438
+ throw new Error("Prompt library table not found");
4439
+ }
4440
+ let query = db3.from("prompt_library").select("tags");
4441
+ query = applyAccessControl(promptTable, query, user);
4442
+ const results = await query;
4443
+ const allTags = [];
4444
+ for (const row of results) {
4445
+ if (row.tags) {
4446
+ let tags = [];
4447
+ if (typeof row.tags === "string") {
4448
+ try {
4449
+ tags = JSON.parse(row.tags);
4450
+ } catch (e) {
4451
+ tags = [row.tags];
4452
+ }
4453
+ } else if (Array.isArray(row.tags)) {
4454
+ tags = row.tags;
4455
+ }
4456
+ tags.forEach((tag) => {
4457
+ if (tag && typeof tag === "string" && tag.trim()) {
4458
+ allTags.push(tag.trim().toLowerCase());
4459
+ }
4460
+ });
4461
+ }
4462
+ }
4463
+ return [...new Set(allTags)].sort();
4464
+ };
4362
4465
  modelDefs += `
4363
4466
  type ProviderPaginationResult {
4364
4467
  items: [Provider]!
@@ -4730,14 +4833,16 @@ var addBucketPrefixToKey = (key, bucket) => {
4730
4833
  }
4731
4834
  return `${bucket}/${key}`;
4732
4835
  };
4733
- var uploadFile = async (file, fileName, config, options = {}, user, customBucket) => {
4836
+ var uploadFile = async (file, fileName, config, options = {}, user, customBucket, global) => {
4734
4837
  if (!config.fileUploads) {
4735
4838
  throw new Error("File uploads are not configured (in the exported uploadFile function)");
4736
4839
  }
4737
4840
  const client2 = getS3Client(config);
4738
4841
  let defaultBucket = config.fileUploads.s3Bucket;
4739
4842
  let key = fileName;
4740
- key = addUserPrefixToKey(key, user || "api");
4843
+ if (!global) {
4844
+ key = addUserPrefixToKey(key, user || "api");
4845
+ }
4741
4846
  key = addGeneralPrefixToKey(key, config);
4742
4847
  const sanitizedMetadata = sanitizeMetadata(options.metadata);
4743
4848
  const command = new import_client_s3.PutObjectCommand({
@@ -4888,7 +4993,7 @@ var createUppyRoutes = async (app, contexts, config) => {
4888
4993
  return;
4889
4994
  }
4890
4995
  let allowed = false;
4891
- if (user.type === "api" || user.super_admin || key.includes(`user_${user.id}/`)) {
4996
+ if (user.type === "api" || user.super_admin || !key.includes(`user_`) || key.includes(`user_${user.id}/`)) {
4892
4997
  allowed = true;
4893
4998
  }
4894
4999
  if (!allowed) {
@@ -4966,9 +5071,16 @@ var createUppyRoutes = async (app, contexts, config) => {
4966
5071
  return;
4967
5072
  }
4968
5073
  const client2 = getS3Client(config);
5074
+ let prefix = `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}`;
5075
+ if (!req.headers.global) {
5076
+ prefix += `user_${authenticationResult.user.id}`;
5077
+ } else {
5078
+ prefix += "global";
5079
+ }
5080
+ console.log("[EXULU] prefix", prefix);
4969
5081
  const command = new import_client_s3.ListObjectsV2Command({
4970
5082
  Bucket: config.fileUploads.s3Bucket,
4971
- Prefix: `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}user_${authenticationResult.user.id}`,
5083
+ Prefix: prefix,
4972
5084
  MaxKeys: 9,
4973
5085
  ...req.query.continuationToken && { ContinuationToken: req.query.continuationToken }
4974
5086
  });
@@ -5051,7 +5163,13 @@ var createUppyRoutes = async (app, contexts, config) => {
5051
5163
  const { filename, contentType } = extractFileParameters(req);
5052
5164
  validateFileParameters(filename, contentType);
5053
5165
  const key = generateS3Key2(filename);
5054
- let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
5166
+ let fullKey = key;
5167
+ console.log("[EXULU] global", req.headers.global);
5168
+ if (!req.headers.global) {
5169
+ fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
5170
+ } else {
5171
+ fullKey = "global/" + key;
5172
+ }
5055
5173
  fullKey = addGeneralPrefixToKey(fullKey, config);
5056
5174
  console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
5057
5175
  (0, import_s3_request_presigner.getSignedUrl)(
@@ -5107,7 +5225,13 @@ var createUppyRoutes = async (app, contexts, config) => {
5107
5225
  return res.status(400).json({ error: "s3: content type must be a string" });
5108
5226
  }
5109
5227
  const key = `${(0, import_node_crypto.randomUUID)()}-_EXULU_${filename}`;
5110
- let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
5228
+ let fullKey = key;
5229
+ console.log("[EXULU] global", req.headers.global);
5230
+ if (!req.headers.global) {
5231
+ fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
5232
+ } else {
5233
+ fullKey = "global/" + key;
5234
+ }
5111
5235
  fullKey = addGeneralPrefixToKey(fullKey, config);
5112
5236
  console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
5113
5237
  const params = {
@@ -5755,6 +5879,31 @@ var ExuluAgent2 = class {
5755
5879
  they are talking with the current date in mind as a reference.`;
5756
5880
  let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
5757
5881
  system += "\n\n" + genericContext;
5882
+ const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
5883
+ console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
5884
+ console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
5885
+ if (includesContextSearchTool) {
5886
+ system += `
5887
+
5888
+
5889
+
5890
+ When you use a context search tool, you will include references to the items
5891
+ retrieved from the context search tool inline in the response using this exact JSON format
5892
+ (all on one line, no line breaks):
5893
+ {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
5894
+
5895
+ IMPORTANT formatting rules:
5896
+ - Use the exact format shown above, all on ONE line
5897
+ - Do NOT use quotes around field names or values
5898
+ - Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
5899
+ - Include the file/item name, not the full path
5900
+ - Separate multiple citations with spaces
5901
+
5902
+ Example: {item_name: document.pdf, item_id: abc123, context: my-context-id, chunk_id: chunk_456, chunk_index: 0}
5903
+
5904
+ The citations will be rendered as interactive badges in the UI.
5905
+ `;
5906
+ }
5758
5907
  if (prompt) {
5759
5908
  let result = { object: null, text: "" };
5760
5909
  let inputTokens = 0;
@@ -5789,7 +5938,7 @@ var ExuluAgent2 = class {
5789
5938
  req,
5790
5939
  project
5791
5940
  ),
5792
- stopWhen: [(0, import_ai.stepCountIs)(2)]
5941
+ stopWhen: [(0, import_ai.stepCountIs)(5)]
5793
5942
  });
5794
5943
  result.text = text;
5795
5944
  inputTokens = totalUsage?.inputTokens || 0;
@@ -5850,7 +5999,7 @@ var ExuluAgent2 = class {
5850
5999
  req,
5851
6000
  project
5852
6001
  ),
5853
- stopWhen: [(0, import_ai.stepCountIs)(2)]
6002
+ stopWhen: [(0, import_ai.stepCountIs)(5)]
5854
6003
  });
5855
6004
  if (statistics) {
5856
6005
  await Promise.all([
@@ -6015,6 +6164,31 @@ ${extractedText}
6015
6164
  const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
6016
6165
  let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
6017
6166
  system += "\n\n" + genericContext;
6167
+ const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
6168
+ console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
6169
+ console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
6170
+ if (includesContextSearchTool) {
6171
+ system += `
6172
+
6173
+
6174
+
6175
+ When you use a context search tool, you will include references to the items
6176
+ retrieved from the context search tool inline in the response using this exact JSON format
6177
+ (all on one line, no line breaks):
6178
+ {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
6179
+
6180
+ IMPORTANT formatting rules:
6181
+ - Use the exact format shown above, all on ONE line
6182
+ - Do NOT use quotes around field names or values
6183
+ - Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
6184
+ - Include the file/item name, not the full path
6185
+ - Separate multiple citations with spaces
6186
+
6187
+ Example: {item_name: document.pdf, item_id: abc123, context: my-context-id, chunk_id: chunk_456, chunk_index: 0}
6188
+
6189
+ The citations will be rendered as interactive badges in the UI.
6190
+ `;
6191
+ }
6018
6192
  const result = (0, import_ai.streamText)({
6019
6193
  model,
6020
6194
  // Should be a LanguageModelV1
@@ -6045,8 +6219,8 @@ ${extractedText}
6045
6219
  onError: (error) => {
6046
6220
  console.error("[EXULU] chat stream error.", error);
6047
6221
  throw new Error(`Chat stream error: ${error instanceof Error ? error.message : String(error)}`);
6048
- }
6049
- // stopWhen: [stepCountIs(1)],
6222
+ },
6223
+ stopWhen: [(0, import_ai.stepCountIs)(5)]
6050
6224
  });
6051
6225
  return {
6052
6226
  stream: result,
@@ -6558,6 +6732,7 @@ var ExuluContext = class {
6558
6732
  })));
6559
6733
  }
6560
6734
  await db3.from(getTableName(this.id)).where({ id: item.id }).update({
6735
+ chunks_count: chunks?.length || 0,
6561
6736
  embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
6562
6737
  }).returning("id");
6563
6738
  return {
@@ -6573,6 +6748,13 @@ var ExuluContext = class {
6573
6748
  throw new Error("Item id or external id is required for upsert.");
6574
6749
  }
6575
6750
  const { db: db3 } = await postgresClient();
6751
+ Object.keys(item).forEach((key) => {
6752
+ if (this.fields.find((field) => field.name === key)?.type === "json") {
6753
+ if (typeof item[key] === "object" || Array.isArray(item[key])) {
6754
+ item[key] = JSON.stringify(item[key]);
6755
+ }
6756
+ }
6757
+ });
6576
6758
  const mutation = db3.from(getTableName(
6577
6759
  this.id
6578
6760
  )).insert(
@@ -6662,6 +6844,13 @@ var ExuluContext = class {
6662
6844
  if (!record) {
6663
6845
  throw new Error("Item not found.");
6664
6846
  }
6847
+ Object.keys(item).forEach((key) => {
6848
+ if (this.fields.find((field) => field.name === key)?.type === "json") {
6849
+ if (typeof item[key] === "object" || Array.isArray(item[key])) {
6850
+ item[key] = JSON.stringify(item[key]);
6851
+ }
6852
+ }
6853
+ });
6665
6854
  const mutation = db3.from(
6666
6855
  getTableName(this.id)
6667
6856
  ).where(
@@ -6827,9 +7016,13 @@ var ExuluContext = class {
6827
7016
  trigger: trigger || "agent"
6828
7017
  }, role, void 0);
6829
7018
  },
6830
- all: async (config, userId, roleId) => {
7019
+ all: async (config, userId, roleId, limit) => {
6831
7020
  const { db: db3 } = await postgresClient();
6832
- const items = await db3.from(getTableName(this.id)).select("*");
7021
+ let query = db3.from(getTableName(this.id)).select("*");
7022
+ if (limit) {
7023
+ query = query.limit(limit);
7024
+ }
7025
+ const items = await query;
6833
7026
  const jobs = [];
6834
7027
  const queue = await this.embedder?.queue;
6835
7028
  if (!queue?.queue.name && items.length > 2e3) {
@@ -6872,9 +7065,11 @@ var ExuluContext = class {
6872
7065
  table.text("created_by");
6873
7066
  table.text("ttl");
6874
7067
  table.text("rights_mode").defaultTo(this.configuration?.defaultRightsMode ?? "private");
7068
+ table.timestamp("embeddings_updated_at").defaultTo(null);
7069
+ table.timestamp("last_processed_at").defaultTo(null);
6875
7070
  table.integer("textlength");
6876
7071
  table.text("source");
6877
- table.timestamp("embeddings_updated_at");
7072
+ table.integer("chunks_count").defaultTo(0);
6878
7073
  for (const field of this.fields) {
6879
7074
  let { type, name, unique } = field;
6880
7075
  if (!type || !name) {
@@ -7403,12 +7598,6 @@ Mood: friendly and intelligent.
7403
7598
  return;
7404
7599
  }
7405
7600
  }
7406
- if (user?.type !== "api" && !user?.super_admin && req.body.resourceId !== user?.id) {
7407
- res.status(400).json({
7408
- message: "The provided user id in the resourceId field is not the same as the authenticated user. Only super admins and API users can impersonate other users."
7409
- });
7410
- return;
7411
- }
7412
7601
  console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
7413
7602
  const disabledTools = req.body.disabledTools ? req.body.disabledTools : [];
7414
7603
  let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
@@ -7871,6 +8060,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
7871
8060
  console.log("[EXULU] creating workers for " + queues2?.length + " queues.");
7872
8061
  console.log("[EXULU] queues", queues2.map((q) => q.queue.name));
7873
8062
  installGlobalErrorHandlers();
8063
+ process.setMaxListeners(Math.max(queues2.length * 2 + 5, 15));
7874
8064
  if (!redisServer.host || !redisServer.port) {
7875
8065
  console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
7876
8066
  throw new Error("No redis server configured in the environment, so cannot start worker.");
@@ -7903,7 +8093,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
7903
8093
  }));
7904
8094
  const { db: db3 } = await postgresClient();
7905
8095
  const data = bullmqJob.data;
7906
- const timeoutInSeconds = data.timeoutInSeconds || 600;
8096
+ const timeoutInSeconds = data.timeoutInSeconds || queue.timeoutInSeconds || 600;
7907
8097
  const timeoutMs = timeoutInSeconds * 1e3;
7908
8098
  let timeoutHandle;
7909
8099
  const timeoutPromise = new Promise((_, reject) => {
@@ -8107,7 +8297,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
8107
8297
  attempts: queue2.retries || 3,
8108
8298
  // todo make this configurable?
8109
8299
  removeOnComplete: 5e3,
8110
- removeOnFail: 1e4,
8300
+ removeOnFail: 5e3,
8111
8301
  backoff: queue2.backoff || {
8112
8302
  type: "exponential",
8113
8303
  delay: 2e3
@@ -8329,8 +8519,8 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
8329
8519
  const { db: db3 } = await postgresClient();
8330
8520
  await db3.from("job_results").where({ job_id: job.id }).update({
8331
8521
  state: JOB_STATUS_ENUM.completed,
8332
- result: returnvalue.result ? JSON.stringify(returnvalue.result) : null,
8333
- metadata: returnvalue.metadata ? JSON.stringify(returnvalue.metadata) : null
8522
+ result: returnvalue.result != null ? JSON.stringify(returnvalue.result) : null,
8523
+ metadata: returnvalue.metadata != null ? JSON.stringify(returnvalue.metadata) : null
8334
8524
  });
8335
8525
  });
8336
8526
  worker.on("failed", async (job, error, prev) => {
@@ -8441,9 +8631,11 @@ var pollJobResult = async ({ queue, jobId }) => {
8441
8631
  console.log(`[EXULU] eval function job ${job.name} completed, getting result from database...`);
8442
8632
  const { db: db3 } = await postgresClient();
8443
8633
  const entry = await db3.from("job_results").where({ job_id: job.id }).first();
8634
+ console.log("[EXULU] eval function job ${job.name} result", entry);
8444
8635
  result = entry?.result;
8445
8636
  if (result === void 0 || result === null || result === "") {
8446
- throw new Error(`Eval function ${job.id} result not found in database for job eval function job ${job.name}.`);
8637
+ throw new Error(`Eval function ${job.id} result not found in database
8638
+ for job eval function job ${job.name}. Entry data from DB: ${JSON.stringify(entry)}.`);
8447
8639
  }
8448
8640
  console.log(`[EXULU] eval function ${job.id} result: ${result}`);
8449
8641
  break;
@@ -9453,45 +9645,6 @@ var createLogger = ({
9453
9645
  };
9454
9646
  var logger_default = createLogger;
9455
9647
 
9456
- // src/templates/contexts/code-standards.ts
9457
- var codeStandardsContext = new ExuluContext({
9458
- id: "code_standards",
9459
- name: "Code Standards",
9460
- description: "Code standards that can be used with the Exulu CLI.",
9461
- configuration: {
9462
- defaultRightsMode: "public"
9463
- },
9464
- fields: [{
9465
- name: "Best practices",
9466
- type: "longText"
9467
- }, {
9468
- name: "Code style",
9469
- type: "longText"
9470
- }, {
9471
- name: "Tech stack",
9472
- type: "longText"
9473
- }],
9474
- active: true
9475
- });
9476
-
9477
- // src/templates/contexts/outputs.ts
9478
- var outputsContext = new ExuluContext({
9479
- id: "outputs_default_context",
9480
- name: "Outputs",
9481
- description: "Outputs from agent sessions that you have saved for re-used later.",
9482
- configuration: {
9483
- defaultRightsMode: "private",
9484
- calculateVectors: "manual"
9485
- },
9486
- fields: [
9487
- {
9488
- name: "content",
9489
- type: "longText"
9490
- }
9491
- ],
9492
- active: true
9493
- });
9494
-
9495
9648
  // src/registry/index.ts
9496
9649
  var import_winston2 = __toESM(require("winston"), 1);
9497
9650
  var import_util = __toESM(require("util"), 1);
@@ -9605,8 +9758,14 @@ var llmAsJudgeEval = () => {
9605
9758
  console.error("[EXULU] prompt is required for llm as judge eval but none is provided.");
9606
9759
  throw new Error("Prompt is required for llm as judge eval but none is provided.");
9607
9760
  }
9761
+ console.log("[EXULU] messages", messages);
9762
+ const lastTypes = messages[messages.length - 1]?.parts?.map((part) => ({
9763
+ type: part.type,
9764
+ text: part.type === "text" ? part.text?.slice(0, 100) : void 0
9765
+ }));
9608
9766
  const lastMessage = messages[messages.length - 1]?.parts?.filter((part) => part.type === "text").map((part) => part.text).join("\n");
9609
9767
  console.log("[EXULU] last message", lastMessage);
9768
+ console.log("[EXULU] last types", lastTypes);
9610
9769
  if (!lastMessage) {
9611
9770
  return 0;
9612
9771
  }
@@ -10618,9 +10777,7 @@ var ExuluApp = class {
10618
10777
  ...evals ?? []
10619
10778
  ] : [];
10620
10779
  this._contexts = {
10621
- ...contexts,
10622
- codeStandardsContext,
10623
- outputsContext
10780
+ ...contexts
10624
10781
  };
10625
10782
  this._agents = [
10626
10783
  claudeSonnet4Agent,
@@ -10677,8 +10834,8 @@ var ExuluApp = class {
10677
10834
  const queueSet = /* @__PURE__ */ new Set();
10678
10835
  if (redisServer.host?.length && redisServer.port?.length) {
10679
10836
  queues.register(global_queues.eval_runs, {
10680
- worker: 1,
10681
- queue: 1
10837
+ worker: 10,
10838
+ queue: 10
10682
10839
  }, 1);
10683
10840
  for (const queue of queues.list.values()) {
10684
10841
  const config2 = await queue.use();
@@ -12356,10 +12513,6 @@ var ExuluJobs = {
12356
12513
  validate: validateJob
12357
12514
  }
12358
12515
  };
12359
- var ExuluDefaultContexts = {
12360
- codeStandards: codeStandardsContext,
12361
- outputs: outputsContext
12362
- };
12363
12516
  var ExuluDefaultAgents = {
12364
12517
  anthropic: {
12365
12518
  opus4: claudeOpus4Agent,
@@ -12518,7 +12671,6 @@ var ExuluChunkers = {
12518
12671
  ExuluChunkers,
12519
12672
  ExuluContext,
12520
12673
  ExuluDefaultAgents,
12521
- ExuluDefaultContexts,
12522
12674
  ExuluEmbedder,
12523
12675
  ExuluEval,
12524
12676
  ExuluJobs,