@exulu/backend 1.54.0 → 1.55.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -101,10 +101,10 @@ async function redisClient() {
101
101
  }
102
102
 
103
103
  // src/exulu/app/index.ts
104
- var import_express6 = require("express");
104
+ var import_express7 = require("express");
105
105
 
106
106
  // src/exulu/routes.ts
107
- var import_express2 = require("express");
107
+ var import_express4 = require("express");
108
108
 
109
109
  // src/auth/get-token.ts
110
110
  var import_jose = require("jose");
@@ -548,7 +548,7 @@ var STATISTICS_TYPE_ENUM = {
548
548
  };
549
549
 
550
550
  // src/exulu/routes.ts
551
- var import_express3 = __toESM(require("express"), 1);
551
+ var import_express5 = __toESM(require("express"), 1);
552
552
  var import_server4 = require("@apollo/server");
553
553
  var import_cors = __toESM(require("cors"), 1);
554
554
  var import_reflect_metadata = require("reflect-metadata");
@@ -807,7 +807,7 @@ var JOB_STATUS_ENUM = {
807
807
  };
808
808
 
809
809
  // ee/agentic-retrieval/v3/index.ts
810
- var import_zod10 = require("zod");
810
+ var import_zod9 = require("zod");
811
811
  var import_bash_tool = require("bash-tool");
812
812
 
813
813
  // src/exulu/tool.ts
@@ -1041,10 +1041,25 @@ var createSessionItemsRetrievalTool = async ({
1041
1041
  return sessionItemsRetrievalTool;
1042
1042
  };
1043
1043
 
1044
+ // ee/agentic-retrieval/v3/session-tools-registry.ts
1045
+ var registry = /* @__PURE__ */ new Map();
1046
+ function registerSessionTools(sessionId, tools) {
1047
+ const existing = registry.get(sessionId) ?? /* @__PURE__ */ new Map();
1048
+ for (const [name, toolDef] of Object.entries(tools)) {
1049
+ existing.set(name, toolDef);
1050
+ }
1051
+ registry.set(sessionId, existing);
1052
+ }
1053
+ function getSessionTools(sessionId) {
1054
+ const toolMap = registry.get(sessionId);
1055
+ if (!toolMap || toolMap.size === 0) return {};
1056
+ return Object.fromEntries(toolMap.entries());
1057
+ }
1058
+
1044
1059
  // src/utils/sanitize-tool-name.ts
1045
1060
  function sanitizeToolName(name) {
1046
1061
  if (typeof name !== "string") return "";
1047
- let sanitized = name.replace(/[^a-zA-Z0-9_.\:-]+/g, "_");
1062
+ let sanitized = name.replace(/[^a-zA-Z0-9_\:-]+/g, "_");
1048
1063
  if (sanitized.length > 0 && !/^[a-zA-Z_]/.test(sanitized)) {
1049
1064
  sanitized = "_" + sanitized;
1050
1065
  }
@@ -1066,7 +1081,6 @@ var import_node_crypto = require("crypto");
1066
1081
 
1067
1082
  // src/templates/tools/memory-tool.ts
1068
1083
  var import_zod3 = require("zod");
1069
- var import_fs = require("fs");
1070
1084
  var createNewMemoryItemTool = (agent, context) => {
1071
1085
  const fields = {
1072
1086
  name: import_zod3.z.string().describe("The name of the item to create"),
@@ -1204,9 +1218,9 @@ var getMimeType = (type) => {
1204
1218
  return "";
1205
1219
  }
1206
1220
  };
1207
- var hydrateVariables = async (tool6) => {
1221
+ var hydrateVariables = async (tool5) => {
1208
1222
  const { db: db2 } = await postgresClient();
1209
- const promises2 = tool6.config.map(async (toolConfig) => {
1223
+ const promises2 = tool5.config.map(async (toolConfig) => {
1210
1224
  if (!toolConfig.variable) {
1211
1225
  return toolConfig;
1212
1226
  }
@@ -1237,9 +1251,9 @@ var hydrateVariables = async (tool6) => {
1237
1251
  return toolConfig;
1238
1252
  });
1239
1253
  await Promise.all(promises2);
1240
- return tool6;
1254
+ return tool5;
1241
1255
  };
1242
- var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExuluTools, configs, providerapikey, contexts, rerankers, user, exuluConfig, sessionID, req, project, items, model, agent) => {
1256
+ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExuluTools, configs, providerapikey, contexts, rerankers, user, exuluConfig, sessionID, req, project, sessionItems, model, agent) => {
1243
1257
  if (!currentTools) return {};
1244
1258
  if (!allExuluTools) {
1245
1259
  allExuluTools = [];
@@ -1274,13 +1288,13 @@ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExulu
1274
1288
  currentTools.push(createNewMemoryTool);
1275
1289
  }
1276
1290
  }
1277
- console.log("[EXULU] Convert tools array to object, session items", items);
1278
- if (items) {
1291
+ console.log("[EXULU] Convert tools array to object, session items", sessionItems);
1292
+ if (sessionItems) {
1279
1293
  const sessionItemsRetrievalTool = await createSessionItemsRetrievalTool({
1280
1294
  user,
1281
1295
  role: user?.role?.id,
1282
1296
  contexts,
1283
- items
1297
+ items: sessionItems
1284
1298
  });
1285
1299
  if (sessionItemsRetrievalTool) {
1286
1300
  currentTools.push(sessionItemsRetrievalTool);
@@ -1294,10 +1308,11 @@ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExulu
1294
1308
  rerankers: rerankers || [],
1295
1309
  user,
1296
1310
  role: user?.role?.id,
1297
- model
1311
+ model,
1312
+ preselectedItemIds: sessionItems
1298
1313
  });
1299
1314
  if (agenticSearchTool) {
1300
- const index = currentTools.findIndex((tool6) => tool6.id === "agentic_context_search");
1315
+ const index = currentTools.findIndex((tool5) => tool5.id === "agentic_context_search");
1301
1316
  if (index !== -1) {
1302
1317
  currentTools[index] = {
1303
1318
  ...currentTools[index],
@@ -1307,21 +1322,26 @@ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExulu
1307
1322
  }
1308
1323
  }
1309
1324
  } else {
1310
- const agenticSearchTool = currentTools.find((tool6) => tool6.id === "agentic_context_search");
1325
+ const agenticSearchTool = currentTools.find((tool5) => tool5.id === "agentic_context_search");
1311
1326
  if (agenticSearchTool) {
1312
1327
  currentTools.splice(currentTools.indexOf(agenticSearchTool), 1);
1313
1328
  }
1314
1329
  }
1315
- const sanitizedTools = currentTools ? currentTools.map((tool6) => ({
1316
- ...tool6,
1317
- name: sanitizeToolName(tool6.name)
1330
+ const sanitizedTools = currentTools ? currentTools.map((tool5) => ({
1331
+ ...tool5,
1332
+ name: sanitizeToolName(tool5.name)
1318
1333
  })) : [];
1319
1334
  console.log(
1320
1335
  "[EXULU] Sanitized tools",
1321
1336
  sanitizedTools.map((x) => x.name + " (" + x.id + ")")
1322
1337
  );
1323
1338
  console.log("[EXULU] Approved tools", approvedTools);
1339
+ const sessionDynamicTools = sessionID ? Object.entries(getSessionTools(sessionID)).reduce((acc, [name, t]) => {
1340
+ acc[name] = { ...t, needsApproval: false };
1341
+ return acc;
1342
+ }, {}) : {};
1324
1343
  return {
1344
+ ...sessionDynamicTools,
1325
1345
  ...sanitizedTools?.reduce((prev, cur) => {
1326
1346
  let toolVariableConfig = configs?.find((config) => config.id === cur.id);
1327
1347
  const userDefinedConfigDescription = toolVariableConfig?.config.find(
@@ -1523,7 +1543,7 @@ var ExuluTool = class {
1523
1543
  });
1524
1544
  }
1525
1545
  execute = async ({
1526
- agent: agentId,
1546
+ agent: agentId2,
1527
1547
  config,
1528
1548
  user,
1529
1549
  inputs,
@@ -1531,14 +1551,14 @@ var ExuluTool = class {
1531
1551
  items
1532
1552
  }) => {
1533
1553
  console.log("[EXULU] Calling tool execute directly", {
1534
- agentId,
1554
+ agentId: agentId2,
1535
1555
  config,
1536
1556
  user,
1537
1557
  inputs,
1538
1558
  project,
1539
1559
  items
1540
1560
  });
1541
- const agent = await exuluApp.get().agent(agentId);
1561
+ const agent = await exuluApp.get().agent(agentId2);
1542
1562
  if (!agent) {
1543
1563
  throw new Error("Agent not found.");
1544
1564
  }
@@ -1581,8 +1601,8 @@ var ExuluTool = class {
1581
1601
  void 0,
1582
1602
  agent
1583
1603
  );
1584
- const tool6 = tools[sanitizeName(this.name)] || tools[this.name] || tools[this.id];
1585
- if (!tool6?.execute) {
1604
+ const tool5 = tools[sanitizeName(this.name)] || tools[this.name] || tools[this.id];
1605
+ if (!tool5?.execute) {
1586
1606
  throw new Error("Tool " + sanitizeName(this.name) + " not found in " + JSON.stringify(tools));
1587
1607
  }
1588
1608
  console.log("[EXULU] Tool found", this.name);
@@ -1592,7 +1612,7 @@ var ExuluTool = class {
1592
1612
  toolCallId,
1593
1613
  messages: []
1594
1614
  });
1595
- const generator = tool6.execute(inputs, {
1615
+ const generator = tool5.execute(inputs, {
1596
1616
  toolCallId,
1597
1617
  messages: []
1598
1618
  });
@@ -1722,7 +1742,7 @@ var uploadFile = async (file, fileName, config, options = {}, user, customBucket
1722
1742
  if (error.name === "SignatureDoesNotMatch" || error.name === "InvalidAccessKeyId" || error.name === "AccessDenied") {
1723
1743
  if (attempt < maxRetries) {
1724
1744
  const backoffMs = Math.pow(2, attempt) * 1e3;
1725
- await new Promise((resolve4) => setTimeout(resolve4, backoffMs));
1745
+ await new Promise((resolve3) => setTimeout(resolve3, backoffMs));
1726
1746
  s3Client2 = void 0;
1727
1747
  getS3Client(config);
1728
1748
  continue;
@@ -1737,6 +1757,87 @@ var uploadFile = async (file, fileName, config, options = {}, user, customBucket
1737
1757
  }
1738
1758
  return addBucketPrefixToKey(key, customBucket || defaultBucket);
1739
1759
  };
1760
+ var listS3ObjectsByPrefix = async (prefix, config) => {
1761
+ if (!config.fileUploads) {
1762
+ throw new Error("File uploads are not configured");
1763
+ }
1764
+ const client2 = getS3Client(config);
1765
+ const bucket = config.fileUploads.s3Bucket;
1766
+ const fullPrefix = addGeneralPrefixToKey(prefix, config);
1767
+ const results = [];
1768
+ let continuationToken;
1769
+ do {
1770
+ const command = new import_client_s32.ListObjectsV2Command({
1771
+ Bucket: bucket,
1772
+ Prefix: fullPrefix,
1773
+ ...continuationToken && { ContinuationToken: continuationToken }
1774
+ });
1775
+ const response = await client2.send(command);
1776
+ for (const obj of response.Contents ?? []) {
1777
+ if (obj.Key && obj.Size !== void 0 && obj.LastModified) {
1778
+ results.push({
1779
+ key: obj.Key,
1780
+ size: obj.Size,
1781
+ lastModified: obj.LastModified
1782
+ });
1783
+ }
1784
+ }
1785
+ continuationToken = response.IsTruncated ? response.NextContinuationToken : void 0;
1786
+ } while (continuationToken);
1787
+ return results;
1788
+ };
1789
+ var copyS3Object = async (sourceKey, destKey, config) => {
1790
+ if (!config.fileUploads) {
1791
+ throw new Error("File uploads are not configured");
1792
+ }
1793
+ const client2 = getS3Client(config);
1794
+ const bucket = config.fileUploads.s3Bucket;
1795
+ const command = new import_client_s32.CopyObjectCommand({
1796
+ Bucket: bucket,
1797
+ CopySource: `${bucket}/${sourceKey}`,
1798
+ Key: destKey
1799
+ });
1800
+ await client2.send(command);
1801
+ };
1802
+ var getS3ObjectContent = async (key, config) => {
1803
+ if (!config.fileUploads) {
1804
+ throw new Error("File uploads are not configured");
1805
+ }
1806
+ const client2 = getS3Client(config);
1807
+ const bucket = config.fileUploads.s3Bucket;
1808
+ const command = new import_client_s32.GetObjectCommand({ Bucket: bucket, Key: key });
1809
+ const response = await client2.send(command);
1810
+ if (!response.Body) {
1811
+ throw new Error(`Empty body for S3 key: ${key}`);
1812
+ }
1813
+ return response.Body.transformToString("utf-8");
1814
+ };
1815
+ var deleteS3Object = async (key, config) => {
1816
+ if (!config.fileUploads) {
1817
+ throw new Error("File uploads are not configured");
1818
+ }
1819
+ const client2 = getS3Client(config);
1820
+ const bucket = config.fileUploads.s3Bucket;
1821
+ const command = new import_client_s32.DeleteObjectCommand({ Bucket: bucket, Key: key });
1822
+ await client2.send(command);
1823
+ };
1824
+ var getS3SignedUploadUrl = async (key, contentType, config) => {
1825
+ if (!config.fileUploads) {
1826
+ throw new Error("File uploads are not configured");
1827
+ }
1828
+ const client2 = getS3Client(config);
1829
+ const bucket = config.fileUploads.s3Bucket;
1830
+ const url = await (0, import_s3_request_presigner.getSignedUrl)(
1831
+ client2,
1832
+ new import_client_s32.PutObjectCommand({
1833
+ Bucket: bucket,
1834
+ Key: key,
1835
+ ContentType: contentType
1836
+ }),
1837
+ { expiresIn }
1838
+ );
1839
+ return url;
1840
+ };
1740
1841
  var createUppyRoutes = async (app, config) => {
1741
1842
  if (!config.fileUploads) {
1742
1843
  throw new Error("File uploads are not configured");
@@ -2995,6 +3096,53 @@ var agentSessionsSchema = {
2995
3096
  }
2996
3097
  ]
2997
3098
  };
3099
+ var skillsSchema = {
3100
+ type: "skills",
3101
+ name: {
3102
+ plural: "skills",
3103
+ singular: "skill"
3104
+ },
3105
+ RBAC: true,
3106
+ fields: [
3107
+ {
3108
+ name: "name",
3109
+ type: "text",
3110
+ index: true,
3111
+ unique: true
3112
+ },
3113
+ {
3114
+ name: "description",
3115
+ type: "text"
3116
+ },
3117
+ {
3118
+ name: "s3folder",
3119
+ type: "text"
3120
+ },
3121
+ {
3122
+ name: "tags",
3123
+ type: "json"
3124
+ },
3125
+ {
3126
+ name: "usage_count",
3127
+ type: "number",
3128
+ default: 0
3129
+ },
3130
+ {
3131
+ name: "favorite_count",
3132
+ type: "number",
3133
+ default: 0
3134
+ },
3135
+ {
3136
+ name: "history",
3137
+ type: "json"
3138
+ },
3139
+ {
3140
+ name: "current_version",
3141
+ type: "number",
3142
+ default: 1
3143
+ }
3144
+ ]
3145
+ };
2998
3146
  var variablesSchema = {
2999
3147
  type: "variables",
3000
3148
  name: {
@@ -3113,6 +3261,10 @@ var agentsSchema = {
3113
3261
  name: "tools",
3114
3262
  type: "json"
3115
3263
  },
3264
+ {
3265
+ name: "skills",
3266
+ type: "json"
3267
+ },
3116
3268
  {
3117
3269
  name: "animation_idle",
3118
3270
  type: "text"
@@ -3372,6 +3524,7 @@ var coreSchemas = {
3372
3524
  agentSessionsSchema: () => addCoreFields(agentSessionsSchema),
3373
3525
  projectsSchema: () => addCoreFields(projectsSchema),
3374
3526
  usersSchema: () => addCoreFields(usersSchema),
3527
+ skillsSchema: () => addCoreFields(skillsSchema),
3375
3528
  statisticsSchema: () => addCoreFields(statisticsSchema),
3376
3529
  variablesSchema: () => addCoreFields(variablesSchema),
3377
3530
  platformConfigurationsSchema: () => addCoreFields(platformConfigurationsSchema),
@@ -3993,9 +4146,6 @@ var mapType = (t, type, name, defaultValue, unique) => {
3993
4146
  throw new Error("Invalid field type for database: " + type);
3994
4147
  };
3995
4148
 
3996
- // src/exulu/context.ts
3997
- var import_zod5 = require("zod");
3998
-
3999
4149
  // ee/queues/decorator.ts
4000
4150
  var import_bullmq2 = require("bullmq");
4001
4151
  var import_uuid = require("uuid");
@@ -4712,69 +4862,6 @@ var ExuluContext2 = class {
4712
4862
  `);
4713
4863
  return;
4714
4864
  };
4715
- // Exports the context as a tool that can be used by an agent
4716
- tool = () => {
4717
- if (this.configuration.enableAsTool === false) {
4718
- return null;
4719
- }
4720
- return new ExuluTool({
4721
- id: this.id,
4722
- name: `${this.name}_context_search`,
4723
- type: "context",
4724
- category: "contexts",
4725
- needsApproval: true,
4726
- // todo make configurable
4727
- inputSchema: import_zod5.z.object({
4728
- query: import_zod5.z.string().describe("The original question that the user asked"),
4729
- keywords: import_zod5.z.array(import_zod5.z.string()).describe(
4730
- "The keywords that are relevant to the user's question, for example names of specific products, systems or parts, IDs, etc."
4731
- ),
4732
- method: import_zod5.z.enum(["keyword", "semantic", "hybrid"]).default("hybrid").describe(
4733
- "Search method: 'hybrid' (best for most queries - combines semantic understanding with exact term matching), 'keyword' (best for exact terms, technical names, IDs, or specific phrases), 'semantic' (best for conceptual queries where synonyms and paraphrasing matter)"
4734
- )
4735
- }),
4736
- config: [],
4737
- description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
4738
- execute: async ({ query, keywords, user, role, method }) => {
4739
- const { db: db2 } = await postgresClient();
4740
- const result = await vectorSearch({
4741
- page: 1,
4742
- limit: this.configuration.maxRetrievalResults ?? 10,
4743
- query,
4744
- keywords,
4745
- itemFilters: [],
4746
- chunkFilters: [],
4747
- user,
4748
- role,
4749
- method: method === "hybrid" ? "hybridSearch" : method === "keyword" ? "tsvector" : "cosineDistance",
4750
- context: this,
4751
- db: db2,
4752
- sort: void 0,
4753
- trigger: "agent"
4754
- });
4755
- await updateStatistic({
4756
- name: "count",
4757
- label: this.name,
4758
- type: STATISTICS_TYPE_ENUM.TOOL_CALL,
4759
- trigger: "tool",
4760
- count: 1,
4761
- user: user?.id,
4762
- role: user?.role?.id
4763
- });
4764
- return {
4765
- result: JSON.stringify(
4766
- result.chunks.map((chunk) => ({
4767
- ...chunk,
4768
- context: {
4769
- name: this.name,
4770
- id: this.id
4771
- }
4772
- }))
4773
- )
4774
- };
4775
- }
4776
- });
4777
- };
4778
4865
  };
4779
4866
 
4780
4867
  // ee/agentic-retrieval/v3/context-sampler.ts
@@ -4819,7 +4906,27 @@ var ContextSampler = class {
4819
4906
 
4820
4907
  // ee/agentic-retrieval/v3/classifier.ts
4821
4908
  var import_ai2 = require("ai");
4822
- var import_zod6 = require("zod");
4909
+ var import_zod5 = require("zod");
4910
+
4911
+ // src/utils/with-retry.ts
4912
+ async function withRetry(generateFn, maxRetries = 3) {
4913
+ let lastError;
4914
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
4915
+ try {
4916
+ return await generateFn();
4917
+ } catch (error) {
4918
+ lastError = error;
4919
+ console.error(`[EXULU] generateText attempt ${attempt} failed:`, error);
4920
+ if (attempt === maxRetries) {
4921
+ throw error;
4922
+ }
4923
+ await new Promise((resolve3) => setTimeout(resolve3, Math.pow(2, attempt) * 1e3));
4924
+ }
4925
+ }
4926
+ throw lastError;
4927
+ }
4928
+
4929
+ // ee/agentic-retrieval/v3/classifier.ts
4823
4930
  async function classifyQuery(query, contexts, samples, model) {
4824
4931
  const contextDescriptions = contexts.map((ctx) => {
4825
4932
  const sample = samples.find((s) => s.contextId === ctx.id);
@@ -4830,46 +4937,49 @@ async function classifyQuery(query, contexts, samples, model) {
4830
4937
  Description: ${ctx.description}
4831
4938
  Fields: ${fieldList}${exampleStr}`;
4832
4939
  }).join("\n\n");
4833
- const result = await (0, import_ai2.generateText)({
4834
- model,
4835
- temperature: 0,
4836
- output: import_ai2.Output.object({
4837
- schema: import_zod6.z.object({
4838
- queryType: import_zod6.z.enum(["aggregate", "list", "targeted", "exploratory"]).describe(
4839
- "aggregate: ONLY use when the user explicitly asks to COUNT how many documents/items/tickets exist in the knowledge base (e.g. 'how many documents about X?', 'total number of tickets'). NEVER use for: real-world statistics stored in a document, intent statements, how-to questions, error/fault descriptions, configuration questions, or any query that does not explicitly ask for a count of knowledge base entries. When in doubt, choose targeted. list: user wants to enumerate matching items/documents (show me all, list documents about). targeted: use for almost everything \u2014 specific fact, answer, configuration, how-to, error/fault, feature/behavior question. Also use for intent statements and short commands describing a desired state (phrases that state what the user wants to do or achieve, even without an explicit question word). Real-world statistics stored in documents also go here. When in doubt, choose targeted over aggregate or exploratory. exploratory: only for broad conceptual questions needing multi-source synthesis (what is the process for Z, explain how X works, general overview of topic Y)."
4840
- ),
4841
- language: import_zod6.z.string().describe("ISO 639-3 language code of the query (e.g. eng, deu, fra)"),
4842
- suggestedContextIds: import_zod6.z.array(import_zod6.z.string()).describe(
4843
- "IDs of knowledge bases most likely to contain the answer. Return empty array to search all contexts."
4844
- )
4845
- })
4846
- }),
4847
- toolChoice: "none",
4848
- system: `You are a query classifier for a multi-knowledge-base retrieval system.
4849
- Classify the query and identify which knowledge bases are most relevant.
4850
-
4851
- Available knowledge bases:
4852
- ${contextDescriptions}
4853
-
4854
- Guidelines for queryType:
4855
- - Use "aggregate" ONLY when the query contains explicit counting language (e.g., "how many", "count", "total number", "wie viele"). Short statements, commands, or phrases without a question word are NEVER aggregate \u2014 classify them as targeted.
4856
- - When in doubt between aggregate and targeted: always choose targeted.
4857
-
4858
- Guidelines for suggestedContextIds:
4859
- - Be conservative: only suggest contexts that are genuinely likely to contain the answer.
4860
- Aim for 2\u20133 focused suggestions rather than listing everything.
4861
- - Use each knowledge base's name and description (shown above) to judge relevance.
4862
- - Return an empty array only if you truly cannot determine which contexts are relevant.`,
4863
- prompt: `Query: ${query}`
4864
- });
4865
- return result.output;
4940
+ const result = await withRetry(async () => {
4941
+ const result2 = await (0, import_ai2.generateText)({
4942
+ model,
4943
+ temperature: 0,
4944
+ output: import_ai2.Output.object({
4945
+ schema: import_zod5.z.object({
4946
+ queryType: import_zod5.z.enum(["aggregate", "list", "targeted", "exploratory"]).describe(
4947
+ "aggregate: ONLY use when the user explicitly asks to COUNT how many documents/items/tickets exist in the knowledge base (e.g. 'how many documents about X?', 'total number of tickets'). NEVER use for: real-world statistics stored in a document, intent statements, how-to questions, error/fault descriptions, configuration questions, or any query that does not explicitly ask for a count of knowledge base entries. When in doubt, choose targeted. list: user wants to enumerate matching items/documents (show me all, list documents about). targeted: use for almost everything \u2014 specific fact, answer, configuration, how-to, error/fault, feature/behavior question. Also use for intent statements and short commands describing a desired state (phrases that state what the user wants to do or achieve, even without an explicit question word). Real-world statistics stored in documents also go here. When in doubt, choose targeted over aggregate or exploratory. exploratory: only for broad conceptual questions needing multi-source synthesis (what is the process for Z, explain how X works, general overview of topic Y)."
4948
+ ),
4949
+ language: import_zod5.z.string().describe("ISO 639-3 language code of the query (e.g. eng, deu, fra)"),
4950
+ suggestedContextIds: import_zod5.z.array(import_zod5.z.enum(contexts.map((c) => c.id))).describe(
4951
+ "IDs of knowledge bases most likely to contain the answer. Return empty array to search all contexts."
4952
+ )
4953
+ })
4954
+ }),
4955
+ toolChoice: "none",
4956
+ system: `You are a query classifier for a multi-knowledge-base retrieval system.
4957
+ Classify the query and identify which knowledge bases are most relevant.
4958
+
4959
+ Available knowledge bases:
4960
+ ${contextDescriptions}
4961
+
4962
+ Guidelines for queryType:
4963
+ - Use "aggregate" ONLY when the query contains explicit counting language (e.g., "how many", "count", "total number", "wie viele"). Short statements, commands, or phrases without a question word are NEVER aggregate \u2014 classify them as targeted.
4964
+ - When in doubt between aggregate and targeted: always choose targeted.
4965
+
4966
+ Guidelines for suggestedContextIds:
4967
+ - Be conservative: only suggest contexts that are genuinely likely to contain the answer.
4968
+ Aim for 2\u20133 focused suggestions rather than listing everything.
4969
+ - Use each knowledge base's name and description (shown above) to judge relevance.
4970
+ - Return an empty array only if you truly cannot determine which contexts are relevant.`,
4971
+ prompt: `Query: ${query}`
4972
+ });
4973
+ return result2.output;
4974
+ }, 3);
4975
+ return result;
4866
4976
  }
4867
4977
 
4868
4978
  // ee/agentic-retrieval/v3/tools.ts
4869
- var import_zod7 = require("zod");
4979
+ var import_zod6 = require("zod");
4870
4980
  var import_ai3 = require("ai");
4871
4981
  function buildContextEnum(contexts) {
4872
- return import_zod7.z.array(import_zod7.z.enum(contexts.map((c) => c.id))).describe(
4982
+ return import_zod6.z.array(import_zod6.z.enum(contexts.map((c) => c.id))).describe(
4873
4983
  contexts.map(
4874
4984
  (c) => `<knowledge_base id="${c.id}" name="${c.name}">${c.description}</knowledge_base>`
4875
4985
  ).join("\n")
@@ -4890,16 +5000,34 @@ function mapSearchMethod(method) {
4890
5000
  if (method === "keyword") return "tsvector";
4891
5001
  return "cosineDistance";
4892
5002
  }
5003
+ function parseGlobalItemIds(globalIds) {
5004
+ const map = /* @__PURE__ */ new Map();
5005
+ for (const gid of globalIds) {
5006
+ const slashIdx = gid.indexOf("/");
5007
+ if (slashIdx === -1) {
5008
+ if (gid) map.set(gid, null);
5009
+ continue;
5010
+ }
5011
+ const contextId = gid.slice(0, slashIdx);
5012
+ const itemId = gid.slice(slashIdx + 1);
5013
+ if (!contextId || !itemId) continue;
5014
+ if (map.get(contextId) === null) continue;
5015
+ const existing = map.get(contextId) ?? [];
5016
+ existing.push(itemId);
5017
+ map.set(contextId, existing);
5018
+ }
5019
+ return map;
5020
+ }
4893
5021
  function createRetrievalTools(params) {
4894
- const { contexts, user, role, updateVirtualFiles } = params;
5022
+ const { contexts, user, role, updateVirtualFiles, preselectedItemsByContext } = params;
4895
5023
  const ctxEnum = buildContextEnum(contexts);
4896
5024
  const count_items_or_chunks = (0, import_ai3.tool)({
4897
5025
  description: "Count items or chunks WITHOUT loading them into context. Use for 'how many', 'count', or 'total number of' queries.",
4898
- inputSchema: import_zod7.z.object({
5026
+ inputSchema: import_zod6.z.object({
4899
5027
  knowledge_base_ids: ctxEnum,
4900
- count_what: import_zod7.z.enum(["items", "chunks"]).describe("Whether to count items (documents) or chunks (pages/sections)"),
4901
- name_contains: import_zod7.z.string().optional().describe("Only count items whose name contains this text (case-insensitive)"),
4902
- content_query: import_zod7.z.string().optional().describe(
5028
+ count_what: import_zod6.z.enum(["items", "chunks"]).describe("Whether to count items (documents) or chunks (pages/sections)"),
5029
+ name_contains: import_zod6.z.string().optional().describe("Only count items whose name contains this text (case-insensitive)"),
5030
+ content_query: import_zod6.z.string().optional().describe(
4903
5031
  "Only count chunks matching this search query (uses hybrid search). Only used when count_what is 'chunks'."
4904
5032
  )
4905
5033
  }),
@@ -4908,6 +5036,10 @@ function createRetrievalTools(params) {
4908
5036
  const ctxList = resolveContexts(knowledge_base_ids, contexts);
4909
5037
  const counts = await Promise.all(
4910
5038
  ctxList.map(async (ctx) => {
5039
+ const contextItemIds = preselectedItemsByContext?.get(ctx.id);
5040
+ if (preselectedItemsByContext && contextItemIds === void 0) {
5041
+ return { context: ctx.id, context_name: ctx.name, count: 0 };
5042
+ }
4911
5043
  let count = 0;
4912
5044
  if (count_what === "items") {
4913
5045
  const tableName = getTableName(ctx.id);
@@ -4915,19 +5047,23 @@ function createRetrievalTools(params) {
4915
5047
  if (name_contains) {
4916
5048
  q = q.whereRaw("LOWER(name) LIKE ?", [`%${name_contains.toLowerCase()}%`]);
4917
5049
  }
5050
+ if (Array.isArray(contextItemIds)) {
5051
+ q = q.whereIn("id", contextItemIds);
5052
+ }
4918
5053
  const tableDefinition = convertContextToTableDefinition(ctx);
4919
5054
  q = applyAccessControl(tableDefinition, q, user, tableName);
4920
5055
  const result = await q.first();
4921
5056
  count = Number(result?.count ?? 0);
4922
5057
  } else {
4923
5058
  const chunksTable = getChunksTableName(ctx.id);
5059
+ const baseItemFilters = Array.isArray(contextItemIds) ? [{ id: { in: contextItemIds } }] : [];
4924
5060
  if (content_query) {
4925
5061
  const searchResults = await ctx.search({
4926
5062
  query: content_query,
4927
5063
  method: "hybridSearch",
4928
5064
  limit: 1e4,
4929
5065
  page: 1,
4930
- itemFilters: [],
5066
+ itemFilters: baseItemFilters,
4931
5067
  chunkFilters: [],
4932
5068
  sort: { field: "updatedAt", direction: "desc" },
4933
5069
  user,
@@ -4935,6 +5071,9 @@ function createRetrievalTools(params) {
4935
5071
  trigger: "tool"
4936
5072
  });
4937
5073
  count = searchResults.chunks.length;
5074
+ } else if (Array.isArray(contextItemIds)) {
5075
+ const result = await db2(chunksTable).count("id as count").whereIn("source", contextItemIds).first();
5076
+ count = Number(result?.count ?? 0);
4938
5077
  } else {
4939
5078
  const result = await db2(chunksTable).count("id as count").first();
4940
5079
  count = Number(result?.count ?? 0);
@@ -4950,11 +5089,13 @@ function createRetrievalTools(params) {
4950
5089
  }
4951
5090
  });
4952
5091
  const search_items_by_name = (0, import_ai3.tool)({
4953
- description: "Search for items by their name or external ID. Use only when the user is asking for documents BY TITLE, not by content topic.",
4954
- inputSchema: import_zod7.z.object({
5092
+ description: "Search for items by their name or external ID. Use when:\n\u2022 The user asks for a document BY TITLE or NAME\n\u2022 The user asks whether a specific named document EXISTS (e.g. 'do you have the X manual?', 'is there a document for Y?')\n\u2022 Any query that references a specific document, manual, or resource by its name rather than by topic\nDo NOT use for topic-based content queries (e.g. 'what are the parameters for X?', 'how do I configure Y?').",
5093
+ inputSchema: import_zod6.z.object({
4955
5094
  knowledge_base_ids: ctxEnum,
4956
- item_name: import_zod7.z.string().describe("The name or partial name to search for"),
4957
- limit: import_zod7.z.number().default(100).describe(
5095
+ item_name: import_zod6.z.string().describe(
5096
+ "The name or partial name to search for. Uses substring matching, so shorter and more specific terms work better than full phrases. Extract only the core identifying part \u2014 typically the product model, document title, or unique identifier. Do NOT include surrounding descriptors like type words ('manual', 'guide', 'document') or manufacturer names unless they are likely part of the actual document title."
5097
+ ),
5098
+ limit: import_zod6.z.number().default(100).describe(
4958
5099
  "Max items per knowledge base (max 400). Applies independently to each knowledge base."
4959
5100
  )
4960
5101
  }),
@@ -4962,9 +5103,12 @@ function createRetrievalTools(params) {
4962
5103
  const { db: db2 } = await postgresClient();
4963
5104
  const ctxList = resolveContexts(knowledge_base_ids, contexts);
4964
5105
  const safeLimit = Math.min(limit ?? 100, 400);
4965
- const itemFilters = item_name ? [{ name: { contains: item_name } }] : [];
4966
5106
  const results = await Promise.all(
4967
5107
  ctxList.map(async (ctx) => {
5108
+ const contextItemIds = preselectedItemsByContext?.get(ctx.id);
5109
+ if (preselectedItemsByContext && contextItemIds === void 0) return [];
5110
+ const itemFilters = item_name ? [{ name: { contains: item_name } }] : [];
5111
+ if (Array.isArray(contextItemIds)) itemFilters.push({ id: { in: contextItemIds } });
4968
5112
  const tableName = getTableName(ctx.id);
4969
5113
  const tableDefinition = convertContextToTableDefinition(ctx);
4970
5114
  let q = db2(`${tableName} as items`).select([
@@ -5000,30 +5144,35 @@ function createRetrievalTools(params) {
5000
5144
  }
5001
5145
  });
5002
5146
  const search_content = (0, import_ai3.tool)({
5003
- description: `Search across document content using hybrid, keyword, or semantic search.
5147
+ description: `Search ONE knowledge base for document content using hybrid, keyword, or semantic search.
5148
+ Always make a separate call for each knowledge base you want to search \u2014 never bundle multiple in one call.
5004
5149
 
5005
5150
  Use includeContent: false when you only need to know WHICH documents match (listing, overview, navigation).
5006
5151
  Use includeContent: true when you need the ACTUAL text to answer a question.
5007
5152
 
5008
5153
  For listing queries: always start with includeContent: false, then use dynamic tools to fetch specific pages.`,
5009
- inputSchema: import_zod7.z.object({
5010
- query: import_zod7.z.string().describe("Search query about the content you're looking for"),
5011
- knowledge_base_ids: ctxEnum,
5012
- keywords: import_zod7.z.array(import_zod7.z.string()).optional().describe("Keywords extracted from the query"),
5013
- searchMethod: import_zod7.z.enum(["hybrid", "keyword", "semantic"]).default("hybrid").describe(
5154
+ inputSchema: import_zod6.z.object({
5155
+ query: import_zod6.z.string().describe("Search query about the content you're looking for"),
5156
+ knowledge_base_id: import_zod6.z.enum(contexts.map((c) => c.id)).describe(
5157
+ contexts.map(
5158
+ (c) => `<knowledge_base id="${c.id}" name="${c.name}">${c.description}</knowledge_base>`
5159
+ ).join("\n")
5160
+ ),
5161
+ keywords: import_zod6.z.array(import_zod6.z.string()).optional().describe("Keywords extracted from the query"),
5162
+ searchMethod: import_zod6.z.enum(["hybrid", "keyword", "semantic"]).default("hybrid").describe(
5014
5163
  "hybrid: best default (semantic + keyword). keyword: exact terms, product codes, IDs. semantic: conceptual/synonyms."
5015
5164
  ),
5016
- includeContent: import_zod7.z.boolean().default(true).describe(
5165
+ includeContent: import_zod6.z.boolean().default(true).describe(
5017
5166
  "false: returns metadata only (document names, scores) \u2014 use for listing/navigation. true: returns full chunk text \u2014 use when you need content to answer a question."
5018
5167
  ),
5019
- item_ids: import_zod7.z.array(import_zod7.z.string()).optional().describe("Filter results to specific item IDs"),
5020
- item_names: import_zod7.z.array(import_zod7.z.string()).optional().describe("Filter results to items whose name contains one of these strings"),
5021
- item_external_ids: import_zod7.z.array(import_zod7.z.string()).optional().describe("Filter results to specific external IDs"),
5022
- limit: import_zod7.z.number().default(10).describe("Max chunks with content (max 10). Without content, up to 200 are returned.")
5168
+ item_ids: import_zod6.z.array(import_zod6.z.string()).optional().describe("Filter results to specific item IDs"),
5169
+ item_names: import_zod6.z.array(import_zod6.z.string()).optional().describe("Filter results to items whose name contains one of these strings"),
5170
+ item_external_ids: import_zod6.z.array(import_zod6.z.string()).optional().describe("Filter results to specific external IDs"),
5171
+ limit: import_zod6.z.number().default(20).describe("Max chunks with content (max 20). Without content, up to 200 are returned.")
5023
5172
  }),
5024
5173
  execute: async ({
5025
5174
  query,
5026
- knowledge_base_ids,
5175
+ knowledge_base_id,
5027
5176
  keywords,
5028
5177
  searchMethod,
5029
5178
  includeContent,
@@ -5032,67 +5181,81 @@ For listing queries: always start with includeContent: false, then use dynamic t
5032
5181
  item_external_ids,
5033
5182
  limit
5034
5183
  }) => {
5035
- const ctxList = resolveContexts(knowledge_base_ids, contexts);
5036
- const effectiveLimit = includeContent ? Math.min(limit ?? 10, 10) : Math.min((limit ?? 10) * 20, 400);
5037
- const results = await Promise.all(
5038
- ctxList.map(async (ctx) => {
5039
- const itemFilters = [];
5040
- if (item_ids) itemFilters.push({ id: { in: item_ids } });
5041
- if (item_names)
5042
- itemFilters.push({ name: { or: item_names.map((n) => ({ contains: n })) } });
5043
- if (item_external_ids) itemFilters.push({ external_id: { in: item_external_ids } });
5044
- const effectiveQuery = query || keywords?.join(" ") || "";
5045
- let method = mapSearchMethod(searchMethod ?? "hybrid");
5046
- if (method === "hybridSearch" || method === "cosineDistance") {
5047
- if (!ctx.embedder) {
5048
- console.error(`[EXULU] context "${ctx.id}" does not have an embedder, falling back to tsvector search`);
5049
- method = "tsvector";
5050
- }
5051
- }
5052
- try {
5053
- const { chunks } = await ctx.search({
5054
- query: effectiveQuery,
5055
- keywords,
5056
- method,
5057
- limit: effectiveLimit,
5058
- page: 1,
5059
- itemFilters,
5060
- chunkFilters: [],
5061
- sort: { field: "updatedAt", direction: "desc" },
5062
- user,
5063
- role,
5064
- trigger: "tool"
5065
- });
5066
- return chunks.map(
5067
- (chunk) => ({
5068
- item_name: chunk.item_name,
5069
- item_id: chunk.item_id,
5070
- context: chunk.context?.id ?? ctx.id,
5071
- chunk_id: chunk.chunk_id,
5072
- chunk_index: chunk.chunk_index,
5073
- chunk_content: includeContent ? chunk.chunk_content : void 0,
5074
- metadata: {
5075
- ...chunk.chunk_metadata,
5076
- cosine_distance: chunk.chunk_cosine_distance,
5077
- fts_rank: chunk.chunk_fts_rank,
5078
- hybrid_score: chunk.chunk_hybrid_score
5079
- }
5080
- })
5081
- );
5082
- } catch (err) {
5083
- console.error(`[EXULU] search_content failed for context "${ctx.id}":`, err);
5084
- return [];
5184
+ const [ctx] = resolveContexts([knowledge_base_id], contexts);
5185
+ const effectiveLimit = includeContent ? Math.min(limit ?? 20, 20) : Math.min((limit ?? 20) * 20, 400);
5186
+ const itemFilters = [];
5187
+ if (preselectedItemsByContext) {
5188
+ const contextItemIds = preselectedItemsByContext.get(knowledge_base_id);
5189
+ if (contextItemIds === void 0) {
5190
+ return JSON.stringify([]);
5191
+ }
5192
+ if (Array.isArray(contextItemIds)) {
5193
+ const intersection = item_ids?.length ? item_ids.filter((id) => contextItemIds.includes(id)) : contextItemIds;
5194
+ if (!intersection.length) {
5195
+ return JSON.stringify([]);
5085
5196
  }
5086
- })
5087
- );
5088
- return JSON.stringify(results.flat());
5197
+ itemFilters.push({ id: { in: intersection } });
5198
+ } else if (item_ids?.length) {
5199
+ itemFilters.push({ id: { in: item_ids } });
5200
+ }
5201
+ } else if (item_ids?.length) {
5202
+ itemFilters.push({ id: { in: item_ids } });
5203
+ }
5204
+ if (item_names)
5205
+ itemFilters.push({ name: { or: item_names.map((n) => ({ contains: n })) } });
5206
+ if (item_external_ids) itemFilters.push({ external_id: { in: item_external_ids } });
5207
+ const effectiveQuery = query || keywords?.join(" ") || "";
5208
+ let method = mapSearchMethod(searchMethod ?? "hybrid");
5209
+ if (method === "hybridSearch" || method === "cosineDistance") {
5210
+ if (!ctx.embedder) {
5211
+ console.error(`[EXULU] context "${ctx.id}" does not have an embedder, falling back to tsvector search`);
5212
+ method = "tsvector";
5213
+ }
5214
+ }
5215
+ try {
5216
+ const { chunks } = await ctx.search({
5217
+ query: effectiveQuery,
5218
+ keywords,
5219
+ method,
5220
+ limit: effectiveLimit,
5221
+ page: 1,
5222
+ itemFilters,
5223
+ chunkFilters: [],
5224
+ sort: { field: "updatedAt", direction: "desc" },
5225
+ user,
5226
+ role,
5227
+ trigger: "tool"
5228
+ });
5229
+ return JSON.stringify(
5230
+ chunks.map(
5231
+ (chunk) => ({
5232
+ item_name: chunk.item_name,
5233
+ item_id: chunk.item_id,
5234
+ context: chunk.context?.id ?? ctx.id,
5235
+ chunk_id: chunk.chunk_id,
5236
+ chunk_index: chunk.chunk_index,
5237
+ chunk_content: includeContent ? chunk.chunk_content : void 0,
5238
+ metadata: {
5239
+ ...chunk.chunk_metadata,
5240
+ cosine_distance: chunk.chunk_cosine_distance,
5241
+ fts_rank: chunk.chunk_fts_rank,
5242
+ hybrid_score: chunk.chunk_hybrid_score
5243
+ }
5244
+ })
5245
+ )
5246
+ );
5247
+ } catch (err) {
5248
+ console.error(`[EXULU] search_content failed for context "${ctx.id}":`, err);
5249
+ return JSON.stringify([]);
5250
+ }
5089
5251
  }
5090
5252
  });
5091
5253
  const save_search_results = (0, import_ai3.tool)({
5092
- description: `Execute a search and save ALL results to the virtual filesystem WITHOUT loading them into context.
5254
+ description: `Execute a search on ONE knowledge base and save ALL results to the virtual filesystem WITHOUT loading them into context.
5255
+ Always make a separate call for each knowledge base you want to search.
5093
5256
 
5094
5257
  Use this when you expect many results (>20) and need to filter iteratively:
5095
- 1. Call save_search_results to save up to 1000 results to /search_results.txt
5258
+ 1. Call save_search_results (once per knowledge base) to save up to 1000 results to /search_results_{knowledge_base_id}.txt
5096
5259
  2. Use bash grep/awk to identify relevant chunks by pattern
5097
5260
  3. Use dynamic get_content tools to load only the specific chunks you need
5098
5261
 
@@ -5107,40 +5270,49 @@ SCORE: ...
5107
5270
  ---CONTENT START---
5108
5271
  (content or placeholder)
5109
5272
  ---CONTENT END---`,
5110
- inputSchema: import_zod7.z.object({
5111
- knowledge_base_ids: ctxEnum,
5112
- query: import_zod7.z.string().describe("Search query"),
5113
- searchMethod: import_zod7.z.enum(["hybrid", "keyword", "semantic"]).default("hybrid"),
5114
- limit: import_zod7.z.number().max(1e3).default(100).describe("Max results to save (max 1000)"),
5115
- includeContent: import_zod7.z.boolean().default(true).describe(
5273
+ inputSchema: import_zod6.z.object({
5274
+ knowledge_base_id: import_zod6.z.enum(contexts.map((c) => c.id)).describe(
5275
+ contexts.map(
5276
+ (c) => `<knowledge_base id="${c.id}" name="${c.name}">${c.description}</knowledge_base>`
5277
+ ).join("\n")
5278
+ ),
5279
+ query: import_zod6.z.string().describe("Search query"),
5280
+ searchMethod: import_zod6.z.enum(["hybrid", "keyword", "semantic"]).default("hybrid"),
5281
+ limit: import_zod6.z.number().max(1e3).default(100).describe("Max results to save (max 1000)"),
5282
+ includeContent: import_zod6.z.boolean().default(true).describe(
5116
5283
  "Whether to include chunk text in the saved file. False saves tokens \u2014 use true only if you need to grep content."
5117
5284
  )
5118
5285
  }),
5119
- execute: async ({ query, knowledge_base_ids, searchMethod, limit, includeContent }) => {
5120
- const ctxList = resolveContexts(knowledge_base_ids, contexts);
5121
- const results = await Promise.all(
5122
- ctxList.map(async (ctx) => {
5123
- try {
5124
- const { chunks: chunks2 } = await ctx.search({
5125
- query,
5126
- method: mapSearchMethod(searchMethod ?? "hybrid"),
5127
- limit: Math.min(limit ?? 100, 1e3),
5128
- page: 1,
5129
- itemFilters: [],
5130
- chunkFilters: [],
5131
- sort: { field: "updatedAt", direction: "desc" },
5132
- user,
5133
- role,
5134
- trigger: "tool"
5135
- });
5136
- return chunks2;
5137
- } catch (err) {
5138
- console.error(`[EXULU] save_search_results failed for context "${ctx.id}":`, err);
5139
- return [];
5140
- }
5141
- })
5142
- );
5143
- const chunks = results.flat();
5286
+ execute: async ({ query, knowledge_base_id, searchMethod, limit, includeContent }) => {
5287
+ const [ctx] = resolveContexts([knowledge_base_id], contexts);
5288
+ const contextItemIds = preselectedItemsByContext?.get(knowledge_base_id);
5289
+ if (preselectedItemsByContext && contextItemIds === void 0) {
5290
+ return JSON.stringify({
5291
+ success: true,
5292
+ results_count: 0,
5293
+ message: `Context "${knowledge_base_id}" not in preselected scope \u2014 skipped.`
5294
+ });
5295
+ }
5296
+ const itemFilters = Array.isArray(contextItemIds) ? [{ id: { in: contextItemIds } }] : [];
5297
+ let chunks = [];
5298
+ try {
5299
+ const result = await ctx.search({
5300
+ query,
5301
+ method: mapSearchMethod(searchMethod ?? "hybrid"),
5302
+ limit: Math.min(limit ?? 100, 1e3),
5303
+ page: 1,
5304
+ itemFilters,
5305
+ chunkFilters: [],
5306
+ sort: { field: "updatedAt", direction: "desc" },
5307
+ user,
5308
+ role,
5309
+ trigger: "tool"
5310
+ });
5311
+ chunks = result.chunks;
5312
+ } catch (err) {
5313
+ console.error(`[EXULU] save_search_results failed for context "${ctx.id}":`, err);
5314
+ }
5315
+ const fileName = `search_results_${ctx.id}.txt`;
5144
5316
  const fileContent = chunks.map(
5145
5317
  (chunk, i) => `### RESULT ${i + 1} ###
5146
5318
  ITEM_NAME: ${chunk.item_name}
@@ -5155,14 +5327,14 @@ ${includeContent && chunk.chunk_content ? chunk.chunk_content : "[use includeCon
5155
5327
  `
5156
5328
  ).join("\n");
5157
5329
  await updateVirtualFiles([
5158
- { path: "search_results.txt", content: fileContent },
5330
+ { path: fileName, content: fileContent },
5159
5331
  {
5160
- path: "search_metadata.json",
5332
+ path: `search_metadata_${ctx.id}.json`,
5161
5333
  content: JSON.stringify({
5162
5334
  query,
5163
5335
  timestamp: (/* @__PURE__ */ new Date()).toISOString(),
5164
5336
  results_count: chunks.length,
5165
- contexts: ctxList.map((c) => c.id),
5337
+ context: ctx.id,
5166
5338
  method: searchMethod
5167
5339
  })
5168
5340
  }
@@ -5170,11 +5342,11 @@ ${includeContent && chunk.chunk_content ? chunk.chunk_content : "[use includeCon
5170
5342
  return JSON.stringify({
5171
5343
  success: true,
5172
5344
  results_count: chunks.length,
5173
- message: `Saved ${chunks.length} results to /search_results.txt`,
5345
+ message: `Saved ${chunks.length} results to /${fileName}`,
5174
5346
  grep_examples: [
5175
- "grep -i 'keyword' search_results.txt | head -20",
5176
- "grep 'ITEM_NAME:' search_results.txt",
5177
- "grep -B 5 'pattern' search_results.txt | grep 'CHUNK_ID:'"
5347
+ `grep -i 'keyword' ${fileName} | head -20`,
5348
+ `grep 'ITEM_NAME:' ${fileName}`,
5349
+ `grep -B 5 'pattern' ${fileName} | grep 'CHUNK_ID:'`
5178
5350
  ]
5179
5351
  });
5180
5352
  }
@@ -5196,9 +5368,10 @@ another component will do that based on what you retrieve.
5196
5368
  Always respond in the SAME LANGUAGE as the user's query.
5197
5369
  Always write search queries in the SAME LANGUAGE as the user's query \u2014 do NOT translate to English.
5198
5370
 
5199
- SEARCH APPROACH \u2014 go wide first, then deep:
5200
- 1. First step: search broadly across all sources the system instructions indicate \u2014 do NOT
5201
- pre-filter to a single context on step 1.
5371
+ SEARCH APPROACH \u2014 one knowledge base at a time, then go deep:
5372
+ 1. search_content and save_search_results accept ONE knowledge base per call. Make a separate
5373
+ call for each knowledge base you need to cover \u2014 never skip one. Search all relevant
5374
+ knowledge bases before concluding, even if the first one already returned good results.
5202
5375
  2. After finding a relevant document, use get_more_content_from_{item} dynamic tools to load
5203
5376
  additional pages/sections. The specific answer is often NOT in the first retrieved chunk \u2014
5204
5377
  always explore adjacent content before concluding.
@@ -5214,9 +5387,8 @@ var AGGREGATE_INSTRUCTIONS = `
5214
5387
  ${BASE_INSTRUCTIONS}
5215
5388
 
5216
5389
  STRATEGY: This is a COUNTING or AGGREGATION query.
5217
- - Use count_items_or_chunks exclusively
5390
+ - Use count_items_or_chunks exclusively \u2014 it accepts multiple knowledge bases in one call for efficiency
5218
5391
  - Do NOT use search_content \u2014 it loads unnecessary data
5219
- - Search ALL contexts in parallel in a single tool call
5220
5392
  - Return immediately after counting \u2014 one step is sufficient
5221
5393
  - If the count needs a content filter, use content_query parameter
5222
5394
  `.trim();
@@ -5249,9 +5421,23 @@ Search language:
5249
5421
  - Always write search queries in the SAME LANGUAGE as the user's query.
5250
5422
  - Do NOT translate the query to English \u2014 the documents are indexed in their original language.
5251
5423
 
5252
- Step 1 \u2014 wide hybrid search (includeContent: true, limit 10):
5253
- - Search broadly across all sources per the system instructions \u2014 do not limit to 1 context.
5254
- - This gives you the best results from every relevant source at once.
5424
+ Step 1 \u2014 match the opening move to what the query actually needs:
5425
+
5426
+ Query references a SPECIFIC NAMED DOCUMENT (product manual, titled report, named file):
5427
+ \u2192 ALWAYS start with search_items_by_name \u2014 searches document name/title directly
5428
+ \u2192 Only proceed to load content if the document is found
5429
+
5430
+ Query asks WHETHER a topic EXISTS or WHAT documents cover a topic (no specific title given):
5431
+ \u2192 search_content with includeContent: false
5432
+ \u2192 Returns matching document names without loading chunk text \u2014 efficient and precise
5433
+ \u2192 Load content with dynamic get_{item}_page_{n}_content tools only if needed in step 2
5434
+
5435
+ Query asks for CONTENT itself (procedures, parameters, explanations, how-to):
5436
+ \u2192 search_content with includeContent: true, limit 20, searchMethod: "hybrid"
5437
+ \u2192 Make one call per knowledge base \u2014 search each separately before concluding
5438
+
5439
+ Query provides an EXACT TERM (error code, product code, ID, parameter name):
5440
+ \u2192 search_content with searchMethod: "keyword"
5255
5441
 
5256
5442
  Step 2+ \u2014 depth and follow-up:
5257
5443
  - For any relevant document found with fewer than 5 chunks, use get_more_content_from_{item}
@@ -5261,19 +5447,9 @@ Step 2+ \u2014 depth and follow-up:
5261
5447
  - Try alternative phrasings if the first query doesn't surface the right answer.
5262
5448
 
5263
5449
  Product-specific filtering:
5264
- - When the query mentions a specific product (e.g., "FST-3", "ECO"), you MAY use
5265
- item_names: ["<product>"] on a follow-up search to narrow results \u2014 but only after an initial
5450
+ - When the query mentions a specific named entity (product, model, version), you MAY use
5451
+ item_names: ["<entity>"] on a follow-up search to narrow results \u2014 but only after an initial
5266
5452
  wide search. Never start with item_names filtering alone.
5267
-
5268
- Two-step approach \u2014 use includeContent: false first:
5269
- - Only when you expect many results (>20) and need to identify the right document first.
5270
- - Step 1: search_content with includeContent: false \u2192 see which documents/chunks match.
5271
- - Step 2: use dynamic get_{item}_page_{n}_content tools to load specific pages.
5272
-
5273
- Search method selection:
5274
- - hybrid (default): best for most queries
5275
- - keyword: exact product codes, document IDs, error codes
5276
- - semantic: conceptual questions, synonyms, paraphrasing
5277
5453
  `.trim();
5278
5454
  var EXPLORATORY_INSTRUCTIONS = `
5279
5455
  ${BASE_INSTRUCTIONS}
@@ -5281,13 +5457,13 @@ ${BASE_INSTRUCTIONS}
5281
5457
  STRATEGY: This is an EXPLORATORY query \u2014 general question requiring broad search.
5282
5458
 
5283
5459
  Recommended approach:
5284
- 1. Start with a wide hybrid search across all relevant contexts (includeContent: true, limit: 10)
5460
+ 1. Search each relevant knowledge base separately with hybrid search (includeContent: true, limit: 20) \u2014 one call per knowledge base
5285
5461
  2. If results are insufficient: try alternative search terms or different search method
5286
5462
  3. Use save_search_results + bash grep when you need to scan many results without context bloat
5287
5463
  4. Use dynamic get_more_content_from_{item} tools to read adjacent pages when a relevant item is found
5288
5464
 
5289
5465
  When to declare done:
5290
- - You have retrieved chunks that cover the key aspects of the query
5466
+ - You have retrieved chunks that cover the key aspects of the query from all relevant knowledge bases
5291
5467
  - OR you have tried 3+ different search strategies and found nothing relevant
5292
5468
 
5293
5469
  Do NOT use count_items_or_chunks for exploratory queries \u2014 the user wants content, not statistics.
@@ -5302,7 +5478,7 @@ var STRATEGIES = {
5302
5478
  },
5303
5479
  list: {
5304
5480
  queryType: "list",
5305
- stepBudget: 2,
5481
+ stepBudget: 3,
5306
5482
  retrieval_tools: ["count_items_or_chunks", "search_items_by_name", "search_content"],
5307
5483
  include_bash: false,
5308
5484
  instructions: LIST_INSTRUCTIONS
@@ -5316,7 +5492,7 @@ var STRATEGIES = {
5316
5492
  },
5317
5493
  exploratory: {
5318
5494
  queryType: "exploratory",
5319
- stepBudget: 4,
5495
+ stepBudget: 5,
5320
5496
  retrieval_tools: [
5321
5497
  "count_items_or_chunks",
5322
5498
  "search_items_by_name",
@@ -5330,28 +5506,10 @@ var STRATEGIES = {
5330
5506
 
5331
5507
  // ee/agentic-retrieval/v3/agent-loop.ts
5332
5508
  var import_ai5 = require("ai");
5333
- var import_zod9 = require("zod");
5334
-
5335
- // src/utils/with-retry.ts
5336
- async function withRetry(generateFn, maxRetries = 3) {
5337
- let lastError;
5338
- for (let attempt = 1; attempt <= maxRetries; attempt++) {
5339
- try {
5340
- return await generateFn();
5341
- } catch (error) {
5342
- lastError = error;
5343
- console.error(`[EXULU] generateText attempt ${attempt} failed:`, error);
5344
- if (attempt === maxRetries) {
5345
- throw error;
5346
- }
5347
- await new Promise((resolve4) => setTimeout(resolve4, Math.pow(2, attempt) * 1e3));
5348
- }
5349
- }
5350
- throw lastError;
5351
- }
5509
+ var import_zod8 = require("zod");
5352
5510
 
5353
5511
  // ee/agentic-retrieval/v3/dynamic-tools.ts
5354
- var import_zod8 = require("zod");
5512
+ var import_zod7 = require("zod");
5355
5513
  var import_ai4 = require("ai");
5356
5514
  async function createDynamicTools(chunks, hadExcludedContent) {
5357
5515
  const { db: db2 } = await postgresClient();
@@ -5370,9 +5528,9 @@ async function createDynamicTools(chunks, hadExcludedContent) {
5370
5528
  const capturedChunk = chunk;
5371
5529
  tools[browseToolName] = (0, import_ai4.tool)({
5372
5530
  description: `"${chunk.item_name}" has ${total} pages/chunks. Use this to read a range of pages from it.`,
5373
- inputSchema: import_zod8.z.object({
5374
- from_index: import_zod8.z.number().min(1).default(1).describe("Starting chunk index (1-based)"),
5375
- to_index: import_zod8.z.number().max(total).describe(`Ending chunk index (max ${total})`)
5531
+ inputSchema: import_zod7.z.object({
5532
+ from_index: import_zod7.z.number().min(1).default(1).describe("Starting chunk index (1-based)"),
5533
+ to_index: import_zod7.z.number().max(total).describe(`Ending chunk index (max ${total})`)
5376
5534
  }),
5377
5535
  execute: async ({ from_index, to_index }) => {
5378
5536
  const { db: db22 } = await postgresClient();
@@ -5401,8 +5559,8 @@ async function createDynamicTools(chunks, hadExcludedContent) {
5401
5559
  const capturedChunk = chunk;
5402
5560
  tools[pageToolName] = (0, import_ai4.tool)({
5403
5561
  description: `Load the full text of page ${chunk.chunk_index} from "${chunk.item_name}"`,
5404
- inputSchema: import_zod8.z.object({
5405
- reasoning: import_zod8.z.string().describe("Why you need this specific page's content")
5562
+ inputSchema: import_zod7.z.object({
5563
+ reasoning: import_zod7.z.string().describe("Why you need this specific page's content")
5406
5564
  }),
5407
5565
  execute: async () => {
5408
5566
  const { db: db22 } = await postgresClient();
@@ -5429,8 +5587,8 @@ async function createDynamicTools(chunks, hadExcludedContent) {
5429
5587
  var FINISH_TOOL_NAME = "finish_retrieval";
5430
5588
  var finishRetrievalTool = (0, import_ai5.tool)({
5431
5589
  description: "Call this tool when you have retrieved sufficient information and no further searches are needed. You MUST call this tool to signal that retrieval is complete \u2014 do not write a text conclusion.",
5432
- inputSchema: import_zod9.z.object({
5433
- reasoning: import_zod9.z.string().describe("One sentence explaining why retrieval is complete")
5590
+ inputSchema: import_zod8.z.object({
5591
+ reasoning: import_zod8.z.string().describe("One sentence explaining why retrieval is complete")
5434
5592
  }),
5435
5593
  execute: async ({ reasoning }) => JSON.stringify({ finished: true, reasoning })
5436
5594
  });
@@ -5463,7 +5621,7 @@ function extractChunksFromToolResults(toolResults) {
5463
5621
  return chunks;
5464
5622
  }
5465
5623
  async function* runAgentLoop(params) {
5466
- const { query, strategy, tools, model, reranker, contextGuidance, customInstructions, onStepComplete } = params;
5624
+ const { query, strategy, tools, model, reranker, contextGuidance, customInstructions, sessionId, onStepComplete, onTrajectoryStep } = params;
5467
5625
  const output = {
5468
5626
  steps: [],
5469
5627
  reasoning: [],
@@ -5519,6 +5677,13 @@ ${customInstructions}` : ""
5519
5677
  }
5520
5678
  messages.push(...result.response.messages);
5521
5679
  let stepChunks = extractChunksFromToolResults(result.toolResults);
5680
+ const seenChunkIds = /* @__PURE__ */ new Set();
5681
+ stepChunks = stepChunks.filter((c) => {
5682
+ if (!c.chunk_id) return true;
5683
+ if (seenChunkIds.has(c.chunk_id)) return false;
5684
+ seenChunkIds.add(c.chunk_id);
5685
+ return true;
5686
+ });
5522
5687
  const hadExcludedContent = result.toolCalls?.some(
5523
5688
  (tc) => tc.toolName === "search_content" && tc.input?.includeContent === false || tc.toolName === "search_items_by_name"
5524
5689
  );
@@ -5528,9 +5693,15 @@ ${customInstructions}` : ""
5528
5693
  }
5529
5694
  const newDynamic = await createDynamicTools(stepChunks, hadExcludedContent);
5530
5695
  Object.assign(dynamicTools, newDynamic);
5696
+ if (sessionId && Object.keys(newDynamic).length > 0) {
5697
+ registerSessionTools(sessionId, newDynamic);
5698
+ }
5531
5699
  forceDepthExploration = stepChunks.length > 0 && stepChunks.length < 5 && Object.keys(newDynamic).length > 0 && step < strategy.stepBudget - 2;
5532
5700
  for (const tc of result.toolCalls ?? []) {
5533
5701
  if (SEARCH_TOOL_NAMES.has(tc.toolName)) {
5702
+ if (tc.input?.knowledge_base_id) {
5703
+ searchedContextIds.add(tc.input.knowledge_base_id);
5704
+ }
5534
5705
  for (const id of tc.input?.knowledge_base_ids ?? []) {
5535
5706
  searchedContextIds.add(id);
5536
5707
  }
@@ -5565,9 +5736,30 @@ ${customInstructions}` : ""
5565
5736
  output: stepChunks
5566
5737
  })) ?? []
5567
5738
  });
5568
- output.chunks.push(...stepChunks);
5739
+ const existingChunkIds = new Set(output.chunks.map((c) => c.chunk_id).filter(Boolean));
5740
+ output.chunks.push(...stepChunks.filter((c) => !c.chunk_id || !existingChunkIds.has(c.chunk_id)));
5569
5741
  output.usage.push(result.usage);
5570
5742
  onStepComplete?.(stepRecord);
5743
+ if (onTrajectoryStep) {
5744
+ const toolResultMap = /* @__PURE__ */ new Map();
5745
+ for (const tr of result.toolResults ?? []) {
5746
+ toolResultMap.set(tr.toolCallId, tr.output ?? tr.result);
5747
+ }
5748
+ onTrajectoryStep({
5749
+ stepNumber: step + 1,
5750
+ systemPrompt: stepSystemPrompt,
5751
+ text: result.text ?? "",
5752
+ toolCalls: result.toolCalls?.map((tc) => ({
5753
+ name: tc.toolName,
5754
+ id: tc.toolCallId,
5755
+ input: tc.input,
5756
+ output: toolResultMap.get(tc.toolCallId)
5757
+ })) ?? [],
5758
+ chunks: stepChunks,
5759
+ dynamicToolsCreated: Object.keys(newDynamic),
5760
+ tokens: result.usage?.totalTokens ?? 0
5761
+ });
5762
+ }
5571
5763
  yield { ...output };
5572
5764
  const calledFinish = result.toolCalls?.some(
5573
5765
  (tc) => tc.toolName === FINISH_TOOL_NAME
@@ -5585,21 +5777,23 @@ ${customInstructions}` : ""
5585
5777
  }
5586
5778
 
5587
5779
  // ee/agentic-retrieval/v3/trajectory.ts
5588
- var fs2 = __toESM(require("fs/promises"), 1);
5780
+ var fs = __toESM(require("fs/promises"), 1);
5589
5781
  var path = __toESM(require("path"), 1);
5590
5782
  var trajectoryRegistry = {
5591
5783
  lastFile: void 0
5592
5784
  };
5593
5785
  var TrajectoryLogger = class {
5594
5786
  data;
5787
+ richSteps = [];
5595
5788
  startTime = Date.now();
5596
5789
  logDir;
5597
- constructor(query, classification, logDir = path.join(process.cwd(), "ee/agentic-retrieval/logs")) {
5790
+ constructor(query, classification, logDir = path.join(process.cwd(), "ee/agentic-retrieval/logs"), preselectedItemIds) {
5598
5791
  this.logDir = logDir;
5599
5792
  this.data = {
5600
5793
  timestamp: (/* @__PURE__ */ new Date()).toISOString(),
5601
5794
  query,
5602
5795
  classification,
5796
+ preselectedItemIds: preselectedItemIds?.length ? preselectedItemIds : void 0,
5603
5797
  steps: [],
5604
5798
  final: {
5605
5799
  total_chunks: 0,
@@ -5620,23 +5814,181 @@ var TrajectoryLogger = class {
5620
5814
  tokens: step.tokens
5621
5815
  });
5622
5816
  }
5623
- async finalize(output, success, error) {
5817
+ recordRichStep(data) {
5818
+ this.richSteps.push(data);
5819
+ }
5820
+ toMarkdown(durationMs, success, error) {
5821
+ const totalTokens = this.richSteps.reduce((sum, s) => sum + s.tokens, 0);
5822
+ const totalChunks = this.richSteps.reduce((sum, s) => sum + s.chunks.length, 0);
5823
+ const status = success ? "\u2713 Success" : `\u2717 Failed${error ? `: ${error.message}` : ""}`;
5824
+ const lines = [];
5825
+ lines.push(`# Agentic Retrieval \u2014 ${this.data.timestamp}`);
5826
+ lines.push("");
5827
+ lines.push(`**Query:** ${this.data.query} `);
5828
+ lines.push(
5829
+ `**Duration:** ${(durationMs / 1e3).toFixed(1)}s | **Tokens:** ${totalTokens} | **Status:** ${status}`
5830
+ );
5831
+ lines.push("");
5832
+ lines.push("## Classification");
5833
+ lines.push("");
5834
+ lines.push(`- **Type:** \`${this.data.classification.queryType}\``);
5835
+ lines.push(`- **Language:** \`${this.data.classification.language}\``);
5836
+ const suggested = this.data.classification.suggestedContextIds;
5837
+ lines.push(
5838
+ `- **Suggested contexts:** ${suggested.length > 0 ? suggested.map((id) => `\`${id}\``).join(", ") : "*(all)*"}`
5839
+ );
5840
+ if (this.data.preselectedItemIds?.length) {
5841
+ lines.push(
5842
+ `- **Preselected item IDs:** ${this.data.preselectedItemIds.map((id) => `\`${id}\``).join(", ")}`
5843
+ );
5844
+ }
5845
+ lines.push("");
5846
+ lines.push("---");
5847
+ lines.push("");
5848
+ const firstStep = this.richSteps[0];
5849
+ if (firstStep) {
5850
+ lines.push("## System Prompt");
5851
+ lines.push("");
5852
+ lines.push("<details>");
5853
+ lines.push("<summary>View system prompt</summary>");
5854
+ lines.push("");
5855
+ lines.push("```");
5856
+ lines.push(firstStep.systemPrompt);
5857
+ lines.push("```");
5858
+ lines.push("");
5859
+ lines.push("</details>");
5860
+ lines.push("");
5861
+ lines.push("---");
5862
+ lines.push("");
5863
+ }
5864
+ for (const step of this.richSteps) {
5865
+ const toolLabel = step.toolCalls.map((tc) => `\`${tc.name}\``).join(", ") || "*(no tool calls)*";
5866
+ lines.push(`## Step ${step.stepNumber} \u2014 ${toolLabel}`);
5867
+ lines.push("");
5868
+ const dynLabel = step.dynamicToolsCreated.length > 0 ? step.dynamicToolsCreated.map((t) => `\`${t}\``).join(", ") : "none";
5869
+ lines.push(
5870
+ `**Tokens:** ${step.tokens} | **Chunks retrieved:** ${step.chunks.length} | **Dynamic tools created:** ${dynLabel}`
5871
+ );
5872
+ lines.push("");
5873
+ if (step.text) {
5874
+ lines.push("### Reasoning");
5875
+ lines.push("");
5876
+ lines.push(step.text);
5877
+ lines.push("");
5878
+ }
5879
+ if (step.toolCalls.length > 0) {
5880
+ lines.push("### Tool Calls");
5881
+ lines.push("");
5882
+ for (const [i, tc] of step.toolCalls.entries()) {
5883
+ lines.push(`#### ${i + 1}. \`${tc.name}\``);
5884
+ lines.push("");
5885
+ lines.push("**Input:**");
5886
+ lines.push("```json");
5887
+ lines.push(JSON.stringify(tc.input, null, 2));
5888
+ lines.push("```");
5889
+ lines.push("");
5890
+ if (tc.output !== void 0) {
5891
+ let parsedOutput;
5892
+ try {
5893
+ parsedOutput = typeof tc.output === "string" ? JSON.parse(tc.output) : tc.output;
5894
+ } catch {
5895
+ parsedOutput = tc.output;
5896
+ }
5897
+ const outputStr = JSON.stringify(parsedOutput, null, 2);
5898
+ const truncated = outputStr.length > 2e3;
5899
+ lines.push("**Output:**");
5900
+ lines.push("```json");
5901
+ lines.push(truncated ? `${outputStr.slice(0, 2e3)}
5902
+ \u2026 (truncated)` : outputStr);
5903
+ lines.push("```");
5904
+ lines.push("");
5905
+ }
5906
+ }
5907
+ }
5908
+ if (step.chunks.length > 0) {
5909
+ lines.push("### Chunks Retrieved");
5910
+ lines.push("");
5911
+ lines.push("| # | Item | Context | Chunk | Score |");
5912
+ lines.push("|---|------|---------|-------|-------|");
5913
+ for (const [i, c] of step.chunks.entries()) {
5914
+ const score = c.metadata?.hybrid_score ?? c.metadata?.cosine_distance ?? c.metadata?.fts_rank ?? "\u2014";
5915
+ const scoreStr = typeof score === "number" ? score.toFixed(4) : String(score);
5916
+ lines.push(
5917
+ `| ${i + 1} | ${c.item_name ?? "\u2014"} | \`${c.context}\` | ${c.chunk_index ?? "\u2014"} | ${scoreStr} |`
5918
+ );
5919
+ }
5920
+ lines.push("");
5921
+ const withContent = step.chunks.filter((c) => c.chunk_content);
5922
+ if (withContent.length > 0) {
5923
+ lines.push("<details>");
5924
+ lines.push("<summary>View chunk content</summary>");
5925
+ lines.push("");
5926
+ for (const c of withContent) {
5927
+ lines.push(`**${c.item_name} (chunk ${c.chunk_index}):**`);
5928
+ lines.push("");
5929
+ const content = (c.chunk_content ?? "").trim();
5930
+ lines.push(`> ${content.split("\n").join("\n> ")}`);
5931
+ lines.push("");
5932
+ }
5933
+ lines.push("</details>");
5934
+ lines.push("");
5935
+ }
5936
+ }
5937
+ if (firstStep && step.stepNumber > 1 && step.systemPrompt !== firstStep.systemPrompt) {
5938
+ const addendum = step.systemPrompt.slice(firstStep.systemPrompt.length).trim();
5939
+ if (addendum) {
5940
+ lines.push("<details>");
5941
+ lines.push("<summary>System prompt addendum (this step only)</summary>");
5942
+ lines.push("");
5943
+ lines.push("```");
5944
+ lines.push(addendum);
5945
+ lines.push("```");
5946
+ lines.push("");
5947
+ lines.push("</details>");
5948
+ lines.push("");
5949
+ }
5950
+ }
5951
+ lines.push("---");
5952
+ lines.push("");
5953
+ }
5954
+ lines.push("## Summary");
5955
+ lines.push("");
5956
+ lines.push("| Metric | Value |");
5957
+ lines.push("|--------|-------|");
5958
+ lines.push(`| Steps | ${this.richSteps.length} |`);
5959
+ lines.push(`| Total chunks | ${totalChunks} |`);
5960
+ lines.push(`| Total tokens | ${totalTokens} |`);
5961
+ lines.push(`| Duration | ${(durationMs / 1e3).toFixed(1)}s |`);
5962
+ lines.push(`| Status | ${status} |`);
5963
+ if (error) {
5964
+ lines.push(`| Error | ${error.message} |`);
5965
+ }
5966
+ lines.push("");
5967
+ return lines.join("\n");
5968
+ }
5969
+ async finalize(output, success, error, writeFiles = false) {
5970
+ const durationMs = Date.now() - this.startTime;
5624
5971
  this.data.final = {
5625
5972
  total_chunks: output.chunks.length,
5626
5973
  total_steps: output.steps.length,
5627
5974
  total_tokens: output.totalTokens,
5628
- duration_ms: Date.now() - this.startTime,
5975
+ duration_ms: durationMs,
5629
5976
  success,
5630
5977
  error: error?.message
5631
5978
  };
5979
+ if (!writeFiles) return void 0;
5632
5980
  try {
5633
- await fs2.mkdir(this.logDir, { recursive: true });
5634
- const filename = `trajectory_${Date.now()}.json`;
5635
- const fullPath = path.join(this.logDir, filename);
5636
- await fs2.writeFile(fullPath, JSON.stringify(this.data, null, 2), "utf-8");
5637
- console.log(`[EXULU] v3 trajectory saved: ${filename}`);
5638
- trajectoryRegistry.lastFile = fullPath;
5639
- return fullPath;
5981
+ await fs.mkdir(this.logDir, { recursive: true });
5982
+ const ts = Date.now();
5983
+ const jsonPath = path.join(this.logDir, `trajectory_${ts}.json`);
5984
+ const mdPath = path.join(this.logDir, `trajectory_${ts}.md`);
5985
+ await Promise.all([
5986
+ fs.writeFile(jsonPath, JSON.stringify(this.data, null, 2), "utf-8"),
5987
+ fs.writeFile(mdPath, this.toMarkdown(durationMs, success, error), "utf-8")
5988
+ ]);
5989
+ console.log(`[EXULU] v3 trajectory saved: trajectory_${ts}.json + trajectory_${ts}.md`);
5990
+ trajectoryRegistry.lastFile = jsonPath;
5991
+ return jsonPath;
5640
5992
  } catch (e) {
5641
5993
  console.error("[EXULU] v3 failed to write trajectory:", e);
5642
5994
  return void 0;
@@ -5653,14 +6005,19 @@ async function* executeV3({
5653
6005
  model,
5654
6006
  user,
5655
6007
  role,
5656
- customInstructions
6008
+ customInstructions,
6009
+ logTrajectory,
6010
+ sessionId,
6011
+ preselectedItemIds
5657
6012
  }) {
6013
+ const preselectedByContext = preselectedItemIds?.length ? parseGlobalItemIds(preselectedItemIds) : void 0;
6014
+ const activeContexts = preselectedByContext?.size ? contexts.filter((c) => preselectedByContext.has(c.id)) : contexts;
5658
6015
  console.log("[EXULU] v3 \u2014 sampling contexts");
5659
- const samples = await sampler.getSamples(contexts, user, role);
6016
+ const samples = await sampler.getSamples(activeContexts, user, role);
5660
6017
  console.log("[EXULU] v3 \u2014 classifying query");
5661
6018
  let classification;
5662
6019
  try {
5663
- classification = await classifyQuery(query, contexts, samples, model);
6020
+ classification = await classifyQuery(query, activeContexts, samples, model);
5664
6021
  } catch (err) {
5665
6022
  console.warn("[EXULU] v3 \u2014 classification failed, falling back to exploratory:", err);
5666
6023
  classification = {
@@ -5672,15 +6029,18 @@ async function* executeV3({
5672
6029
  console.log("[EXULU] v3 \u2014 classified as:", classification);
5673
6030
  const strategy = STRATEGIES[classification.queryType];
5674
6031
  const suggestedIds = classification.suggestedContextIds;
5675
- const fallbackIds = contexts.filter((c) => !suggestedIds.includes(c.id)).map((c) => c.id);
5676
- const contextGuidance = suggestedIds.length > 0 ? `Suggested priority contexts: [${suggestedIds.join(", ")}]. Also available: [${fallbackIds.join(", ")}]. Custom instructions may require searching additional or all contexts \u2014 follow them.` : `All contexts available: [${contexts.map((c) => c.id).join(", ")}].`;
6032
+ const fallbackIds = activeContexts.filter((c) => !suggestedIds.includes(c.id)).map((c) => c.id);
6033
+ const contextBase = suggestedIds.length > 0 ? `Suggested priority contexts: [${suggestedIds.join(", ")}]. Also available: [${fallbackIds.join(", ")}]. Custom instructions may require searching additional or all contexts \u2014 follow them.` : `All contexts available: [${activeContexts.map((c) => c.id).join(", ")}].`;
6034
+ const preselectedNote = preselectedByContext?.size ? `
6035
+ SCOPE CONSTRAINT: Retrieval is scoped to preselected items/contexts. Per context: ${[...preselectedByContext.entries()].map(([ctx, ids]) => ids === null ? `${ctx} (full context)` : `${ctx} (${ids.length} item${ids.length === 1 ? "" : "s"})`).join(", ")}. All tools enforce this scope automatically. For full-context entries you may search freely; for item-restricted entries do NOT use search_items_by_name for discovery \u2014 go directly to search_content or save_search_results.` : "";
6036
+ const contextGuidance = contextBase + preselectedNote;
5677
6037
  const bashToolkit = await (0, import_bash_tool.createBashTool)({ files: {} });
5678
6038
  const retrievalTools = createRetrievalTools({
5679
- contexts,
5680
- // ALL contexts — agent decides which to search based on context guidance
6039
+ contexts: activeContexts,
5681
6040
  user,
5682
6041
  role,
5683
- updateVirtualFiles: (files) => bashToolkit.sandbox.writeFiles(files)
6042
+ updateVirtualFiles: (files) => bashToolkit.sandbox.writeFiles(files),
6043
+ preselectedItemsByContext: preselectedByContext
5684
6044
  });
5685
6045
  const activeTools = {};
5686
6046
  for (const name of strategy.retrieval_tools) {
@@ -5691,7 +6051,7 @@ async function* executeV3({
5691
6051
  if (strategy.include_bash) {
5692
6052
  Object.assign(activeTools, bashToolkit.tools);
5693
6053
  }
5694
- const trajectory = new TrajectoryLogger(query, classification);
6054
+ const trajectory = new TrajectoryLogger(query, classification, void 0, preselectedItemIds);
5695
6055
  let finalOutput;
5696
6056
  let executionError;
5697
6057
  try {
@@ -5704,7 +6064,9 @@ async function* executeV3({
5704
6064
  contextGuidance,
5705
6065
  customInstructions,
5706
6066
  classification,
5707
- onStepComplete: (step) => trajectory.recordStep(step)
6067
+ sessionId,
6068
+ onStepComplete: (step) => trajectory.recordStep(step),
6069
+ onTrajectoryStep: (data) => trajectory.recordRichStep(data)
5708
6070
  })) {
5709
6071
  finalOutput = output;
5710
6072
  yield output;
@@ -5715,7 +6077,7 @@ async function* executeV3({
5715
6077
  throw err;
5716
6078
  } finally {
5717
6079
  if (finalOutput) {
5718
- const trajectoryFile = await trajectory.finalize(finalOutput, !executionError, executionError);
6080
+ const trajectoryFile = await trajectory.finalize(finalOutput, !executionError, executionError, logTrajectory);
5719
6081
  if (trajectoryFile) {
5720
6082
  finalOutput.trajectoryFile = trajectoryFile;
5721
6083
  }
@@ -5728,7 +6090,8 @@ function createAgenticRetrievalToolV3({
5728
6090
  rerankers,
5729
6091
  user,
5730
6092
  role,
5731
- model
6093
+ model,
6094
+ preselectedItemIds
5732
6095
  }) {
5733
6096
  const license = checkLicense();
5734
6097
  if (!license["agentic-retrieval"]) {
@@ -5756,6 +6119,12 @@ function createAgenticRetrievalToolV3({
5756
6119
  type: "string",
5757
6120
  default: "none"
5758
6121
  },
6122
+ {
6123
+ name: "managed_context",
6124
+ description: "Makes sure the user defines which items from which contexts the agentic retrieval tool will search in",
6125
+ type: "boolean",
6126
+ default: false
6127
+ },
5759
6128
  {
5760
6129
  name: "reasoning_model",
5761
6130
  description: "By default the agentic retrieval tool uses the model from the agent calling the tool, but you can overwrite this here for the reasoning phase",
@@ -5768,6 +6137,18 @@ function createAgenticRetrievalToolV3({
5768
6137
  type: "string",
5769
6138
  default: ""
5770
6139
  },
6140
+ {
6141
+ name: "require_preselected_contexts",
6142
+ description: "Require the user to preselect contexts before executing the tool, meaning the user will be asked to select the contexts they want to search in",
6143
+ type: "boolean",
6144
+ default: false
6145
+ },
6146
+ {
6147
+ name: "log_trajectories",
6148
+ description: "Save a detailed markdown + JSON log of every retrieval execution to disk. Useful for debugging and evaluation.",
6149
+ type: "boolean",
6150
+ default: false
6151
+ },
5771
6152
  ...contexts.map((ctx) => ({
5772
6153
  name: ctx.id,
5773
6154
  description: `Enable search in "${ctx.name}". ${ctx.description}`,
@@ -5775,32 +6156,61 @@ function createAgenticRetrievalToolV3({
5775
6156
  default: true
5776
6157
  }))
5777
6158
  ],
5778
- inputSchema: import_zod10.z.object({
5779
- query: import_zod10.z.string().describe("The question or query to answer"),
5780
- userInstructions: import_zod10.z.string().optional().describe("Additional instructions from the user to guide retrieval")
6159
+ inputSchema: import_zod9.z.object({
6160
+ query: import_zod9.z.string().describe("The question or query to answer"),
6161
+ userInstructions: import_zod9.z.string().optional().describe("Additional instructions from the user to guide retrieval"),
6162
+ confirmedContextIds: import_zod9.z.array(import_zod9.z.string()).optional().describe(
6163
+ "Knowledge base IDs explicitly confirmed by the user to be used in the retrieval. When presen only searches these contexts. "
6164
+ )
5781
6165
  }),
5782
6166
  execute: async function* ({
5783
6167
  query,
5784
6168
  userInstructions,
5785
- toolVariablesConfig
6169
+ confirmedContextIds,
6170
+ toolVariablesConfig,
6171
+ sessionID
5786
6172
  }) {
5787
6173
  if (!model) {
5788
- throw new Error("Model is required for executing the agentic retrieval tool");
6174
+ yield { result: "Model is required for executing the agentic retrieval tool" };
6175
+ return;
5789
6176
  }
5790
6177
  let activeContexts = contexts;
5791
6178
  let configuredReranker;
5792
6179
  let configInstructions = "";
6180
+ let logTrajectory = false;
6181
+ let requiresPreselectedContexts = false;
6182
+ let managedContextEnabled = false;
5793
6183
  if (toolVariablesConfig) {
5794
6184
  configInstructions = toolVariablesConfig["instructions"] ?? "";
6185
+ logTrajectory = toolVariablesConfig["log_trajectories"] === true || toolVariablesConfig["log_trajectories"] === "true";
6186
+ managedContextEnabled = toolVariablesConfig["managed_context"] === true || toolVariablesConfig["managed_context"] === "true";
5795
6187
  activeContexts = contexts.filter(
5796
6188
  (ctx) => toolVariablesConfig[ctx.id] === true || toolVariablesConfig[ctx.id] === "true" || toolVariablesConfig[ctx.id] === 1
5797
6189
  );
5798
6190
  if (activeContexts.length === 0) activeContexts = contexts;
6191
+ requiresPreselectedContexts = toolVariablesConfig["require_preselected_contexts"] === true || toolVariablesConfig["require_preselected_contexts"] === "true";
5799
6192
  const rerankerId = toolVariablesConfig["reranker"];
5800
6193
  if (rerankerId && rerankerId !== "none") {
5801
6194
  configuredReranker = rerankers.find((r) => r.id === rerankerId);
5802
6195
  }
5803
6196
  }
6197
+ console.log("[EXULU] Managed context enabled:", managedContextEnabled);
6198
+ console.log("[EXULU] Preselected item IDs:", preselectedItemIds);
6199
+ if (managedContextEnabled && !preselectedItemIds?.length) {
6200
+ console.log("[EXULU] Managed context was enabled for the agentic retrieval tool. This means that the user must preselect items that the agentic retrieval tool will search in, please notify the user to preselect items before executing the tool.");
6201
+ yield { result: "Managed context was enabled for the agentic retrieval tool. This means that the user must preselect items that the agentic retrieval tool will search in, please notify the user to preselect items before executing the tool." };
6202
+ return;
6203
+ }
6204
+ if (requiresPreselectedContexts && !confirmedContextIds?.length && !preselectedItemIds?.length) {
6205
+ console.log("[EXULU] The user must choose between the available contexts before executing the tool. The available contexts are: " + activeContexts.map((c) => c.id).join(", ") + ". If the question_ask tool is available use that to ask the user which contexts they want to search in, otherwise just ask them in plain text.");
6206
+ yield { result: "The user must choose between the available contexts before executing the tool, the available contexts are: " + activeContexts.map((c) => c.id).join(", ") + ". If the question_ask tool is available use that to ask the user which contexts they want to search in, otherwise just ask them in plain text." };
6207
+ return;
6208
+ }
6209
+ if (confirmedContextIds?.length) {
6210
+ const confirmed = new Set(confirmedContextIds);
6211
+ const filtered = activeContexts.filter((c) => confirmed.has(c.id));
6212
+ if (filtered.length > 0) activeContexts = filtered;
6213
+ }
5804
6214
  const combinedInstructions = [
5805
6215
  configInstructions ? `Configuration instructions: ${configInstructions}` : "",
5806
6216
  adminInstructions ? `Admin instructions: ${adminInstructions}` : "",
@@ -5813,10 +6223,14 @@ function createAgenticRetrievalToolV3({
5813
6223
  model,
5814
6224
  user,
5815
6225
  role,
5816
- customInstructions: combinedInstructions || void 0
6226
+ customInstructions: combinedInstructions || void 0,
6227
+ logTrajectory,
6228
+ sessionId: sessionID,
6229
+ preselectedItemIds
5817
6230
  })) {
5818
6231
  yield { result: JSON.stringify(output) };
5819
6232
  }
6233
+ return;
5820
6234
  }
5821
6235
  });
5822
6236
  }
@@ -5971,9 +6385,9 @@ var addProviderFields = async (args, requestedFields, providers, result, tools,
5971
6385
  if (result.tools) {
5972
6386
  result.tools = await Promise.all(
5973
6387
  result.tools.map(
5974
- async (tool6) => {
6388
+ async (tool5) => {
5975
6389
  let hydrated;
5976
- if (tool6.id === "agentic_context_search") {
6390
+ if (tool5.id === "agentic_context_search") {
5977
6391
  const instance2 = createAgenticRetrievalToolV3({
5978
6392
  contexts: [],
5979
6393
  rerankers: [],
@@ -5989,23 +6403,23 @@ var addProviderFields = async (args, requestedFields, providers, result, tools,
5989
6403
  name: instance2.name,
5990
6404
  description: instance2.description,
5991
6405
  category: instance2.category,
5992
- config: tool6.config
6406
+ config: tool5.config
5993
6407
  };
5994
6408
  }
5995
- if (tool6.type === "agent") {
5996
- if (tool6.id === result.id) {
6409
+ if (tool5.type === "agent") {
6410
+ if (tool5.id === result.id) {
5997
6411
  return null;
5998
6412
  }
5999
- const instance2 = await exuluApp.get().agent(tool6.id);
6413
+ const instance2 = await exuluApp.get().agent(tool5.id);
6000
6414
  if (!instance2) {
6001
6415
  throw new Error(
6002
- "Trying to load a tool of type 'agent', but the associated agent with id " + tool6.id + " was not found in the database."
6416
+ "Trying to load a tool of type 'agent', but the associated agent with id " + tool5.id + " was not found in the database."
6003
6417
  );
6004
6418
  }
6005
6419
  const provider2 = providers.find((a) => a.id === instance2.provider);
6006
6420
  if (!provider2) {
6007
6421
  throw new Error(
6008
- "Trying to load a tool of type 'agent', but the associated agent with id " + tool6.id + " does not have a provider set for it."
6422
+ "Trying to load a tool of type 'agent', but the associated agent with id " + tool5.id + " does not have a provider set for it."
6009
6423
  );
6010
6424
  }
6011
6425
  const hasAccessToAgent = await checkRecordAccess(instance2, "read", user);
@@ -6014,13 +6428,13 @@ var addProviderFields = async (args, requestedFields, providers, result, tools,
6014
6428
  }
6015
6429
  hydrated = await provider2.tool(instance2.id, providers, contexts, rerankers);
6016
6430
  } else {
6017
- hydrated = tools.find((t) => t.id === tool6.id);
6431
+ hydrated = tools.find((t) => t.id === tool5.id);
6018
6432
  }
6019
6433
  const hydratedTool = {
6020
- ...tool6,
6434
+ ...tool5,
6021
6435
  name: hydrated?.name || "",
6022
6436
  description: hydrated?.description || "",
6023
- category: tool6?.category || "default"
6437
+ category: tool5?.category || "default"
6024
6438
  };
6025
6439
  console.log("[EXULU] hydratedTool", hydratedTool);
6026
6440
  return hydratedTool;
@@ -6038,7 +6452,7 @@ var addProviderFields = async (args, requestedFields, providers, result, tools,
6038
6452
  result.tools.unshift(projectTool);
6039
6453
  }
6040
6454
  }
6041
- result.tools = result.tools.filter((tool6) => tool6 !== null);
6455
+ result.tools = result.tools.filter((tool5) => tool5 !== null);
6042
6456
  } else {
6043
6457
  result.tools = [];
6044
6458
  }
@@ -7544,7 +7958,7 @@ var getEnabledTools = async (agent, allExuluTools, allContexts, allRerankers, di
7544
7958
  }
7545
7959
  console.log("[EXULU] available tools", enabledTools?.length);
7546
7960
  console.log("[EXULU] disabled tools", disabledTools?.length);
7547
- enabledTools = enabledTools.filter((tool6) => !disabledTools.includes(tool6.id));
7961
+ enabledTools = enabledTools.filter((tool5) => !disabledTools.includes(tool5.id));
7548
7962
  return enabledTools;
7549
7963
  };
7550
7964
 
@@ -7689,7 +8103,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
7689
8103
  );
7690
8104
  if (attempt < retries) {
7691
8105
  const backoffMs = 500 * Math.pow(2, attempt - 1);
7692
- await new Promise((resolve4) => setTimeout(resolve4, backoffMs));
8106
+ await new Promise((resolve3) => setTimeout(resolve3, backoffMs));
7693
8107
  }
7694
8108
  }
7695
8109
  }
@@ -7892,7 +8306,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
7892
8306
  } = await validateWorkflowPayload(data, providers);
7893
8307
  const retries2 = 3;
7894
8308
  let attempts = 0;
7895
- const promise = new Promise(async (resolve4, reject) => {
8309
+ const promise = new Promise(async (resolve3, reject) => {
7896
8310
  while (attempts < retries2) {
7897
8311
  try {
7898
8312
  const messages2 = await processUiMessagesFlow({
@@ -7907,7 +8321,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
7907
8321
  config,
7908
8322
  variables: data.inputs
7909
8323
  });
7910
- resolve4(messages2);
8324
+ resolve3(messages2);
7911
8325
  break;
7912
8326
  } catch (error) {
7913
8327
  console.error(
@@ -7918,7 +8332,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
7918
8332
  if (attempts >= retries2) {
7919
8333
  reject(new Error(error instanceof Error ? error.message : String(error)));
7920
8334
  }
7921
- await new Promise((resolve5) => setTimeout((resolve6) => resolve6(true), 2e3));
8335
+ await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
7922
8336
  }
7923
8337
  }
7924
8338
  });
@@ -7968,7 +8382,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
7968
8382
  } = await validateEvalPayload(data, providers);
7969
8383
  const retries2 = 3;
7970
8384
  let attempts = 0;
7971
- const promise = new Promise(async (resolve4, reject) => {
8385
+ const promise = new Promise(async (resolve3, reject) => {
7972
8386
  while (attempts < retries2) {
7973
8387
  try {
7974
8388
  const messages2 = await processUiMessagesFlow({
@@ -7982,7 +8396,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
7982
8396
  tools,
7983
8397
  config
7984
8398
  });
7985
- resolve4(messages2);
8399
+ resolve3(messages2);
7986
8400
  break;
7987
8401
  } catch (error) {
7988
8402
  console.error(
@@ -7993,7 +8407,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
7993
8407
  if (attempts >= retries2) {
7994
8408
  reject(new Error(error instanceof Error ? error.message : String(error)));
7995
8409
  }
7996
- await new Promise((resolve5) => setTimeout((resolve6) => resolve6(true), 2e3));
8410
+ await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
7997
8411
  }
7998
8412
  }
7999
8413
  });
@@ -8468,7 +8882,7 @@ var pollJobResult = async ({
8468
8882
  attempts++;
8469
8883
  const job = await import_bullmq3.Job.fromId(queue.queue, jobId);
8470
8884
  if (!job) {
8471
- await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
8885
+ await new Promise((resolve3) => setTimeout((resolve4) => resolve4(true), 2e3));
8472
8886
  continue;
8473
8887
  }
8474
8888
  const elapsedTime = Date.now() - startTime;
@@ -8498,7 +8912,7 @@ var pollJobResult = async ({
8498
8912
  console.log(`[EXULU] eval function ${job.id} result: ${result}`);
8499
8913
  break;
8500
8914
  }
8501
- await new Promise((resolve4) => setTimeout(() => resolve4(true), 2e3));
8915
+ await new Promise((resolve3) => setTimeout(() => resolve3(true), 2e3));
8502
8916
  }
8503
8917
  return result;
8504
8918
  };
@@ -8606,7 +9020,7 @@ var processUiMessagesFlow = async ({
8606
9020
  label: agent.name,
8607
9021
  trigger: "agent"
8608
9022
  };
8609
- messageHistory = await new Promise(async (resolve4, reject) => {
9023
+ messageHistory = await new Promise(async (resolve3, reject) => {
8610
9024
  const startTime = Date.now();
8611
9025
  try {
8612
9026
  const result = await provider.generateStream({
@@ -8614,7 +9028,7 @@ var processUiMessagesFlow = async ({
8614
9028
  rerankers,
8615
9029
  agent,
8616
9030
  user,
8617
- approvedTools: tools.map((tool6) => "tool-" + sanitizeToolName(tool6.name)),
9031
+ approvedTools: tools.map((tool5) => "tool-" + sanitizeToolName(tool5.name)),
8618
9032
  instructions: agent.instructions,
8619
9033
  session: void 0,
8620
9034
  previousMessages: messageHistory.messages,
@@ -8683,7 +9097,7 @@ var processUiMessagesFlow = async ({
8683
9097
  })
8684
9098
  ] : []
8685
9099
  ]);
8686
- resolve4({
9100
+ resolve3({
8687
9101
  messages,
8688
9102
  metadata: {
8689
9103
  tokens: {
@@ -9431,7 +9845,7 @@ type PageInfo {
9431
9845
  } = await validateWorkflowPayload(jobData, providers);
9432
9846
  const retries = 3;
9433
9847
  let attempts = 0;
9434
- const promise = new Promise(async (resolve4, reject) => {
9848
+ const promise = new Promise(async (resolve3, reject) => {
9435
9849
  while (attempts < retries) {
9436
9850
  try {
9437
9851
  const messages2 = await processUiMessagesFlow({
@@ -9446,7 +9860,7 @@ type PageInfo {
9446
9860
  config,
9447
9861
  variables: args.variables
9448
9862
  });
9449
- resolve4(messages2);
9863
+ resolve3(messages2);
9450
9864
  break;
9451
9865
  } catch (error) {
9452
9866
  console.error(
@@ -9460,7 +9874,7 @@ type PageInfo {
9460
9874
  if (attempts >= retries) {
9461
9875
  reject(error instanceof Error ? error : new Error(String(error)));
9462
9876
  }
9463
- await new Promise((resolve5) => setTimeout((resolve6) => resolve6(true), 2e3));
9877
+ await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
9464
9878
  }
9465
9879
  }
9466
9880
  });
@@ -9713,10 +10127,10 @@ type PageInfo {
9713
10127
  contexts.map(async (context2) => {
9714
10128
  let processor = null;
9715
10129
  if (context2.processor) {
9716
- processor = await new Promise(async (resolve4, reject) => {
10130
+ processor = await new Promise(async (resolve3, reject) => {
9717
10131
  const config2 = context2.processor?.config;
9718
10132
  const queue = await config2?.queue;
9719
- resolve4({
10133
+ resolve3({
9720
10134
  name: context2.processor.name,
9721
10135
  description: context2.processor.description,
9722
10136
  queue: queue?.queue?.name || void 0,
@@ -9797,10 +10211,10 @@ type PageInfo {
9797
10211
  }
9798
10212
  let processor = null;
9799
10213
  if (data.processor) {
9800
- processor = await new Promise(async (resolve4, reject) => {
10214
+ processor = await new Promise(async (resolve3, reject) => {
9801
10215
  const config2 = data.processor?.config;
9802
10216
  const queue = await config2?.queue;
9803
- resolve4({
10217
+ resolve3({
9804
10218
  name: data.processor.name,
9805
10219
  description: data.processor.description,
9806
10220
  queue: queue?.queue?.name || void 0,
@@ -9889,7 +10303,7 @@ type PageInfo {
9889
10303
  })
9890
10304
  );
9891
10305
  let agenticRetrievalTool = void 0;
9892
- const filtered = agentTools.filter((tool6) => tool6 !== null);
10306
+ const filtered = agentTools.filter((tool5) => tool5 !== null);
9893
10307
  let allTools = [...filtered, ...tools];
9894
10308
  if (contexts?.length) {
9895
10309
  agenticRetrievalTool = createAgenticRetrievalToolV3({
@@ -9907,21 +10321,21 @@ type PageInfo {
9907
10321
  if (search && search.trim()) {
9908
10322
  const searchTerm = search.toLowerCase().trim();
9909
10323
  allTools = allTools.filter(
9910
- (tool6) => tool6.name?.toLowerCase().includes(searchTerm) || tool6.description?.toLowerCase().includes(searchTerm)
10324
+ (tool5) => tool5.name?.toLowerCase().includes(searchTerm) || tool5.description?.toLowerCase().includes(searchTerm)
9911
10325
  );
9912
10326
  }
9913
10327
  if (category && category.trim()) {
9914
- allTools = allTools.filter((tool6) => tool6.category === category);
10328
+ allTools = allTools.filter((tool5) => tool5.category === category);
9915
10329
  }
9916
10330
  const total = allTools.length;
9917
10331
  const start = page * limit;
9918
10332
  const end = start + limit;
9919
10333
  const paginatedTools = allTools.slice(start, end);
9920
10334
  return {
9921
- items: paginatedTools.map((tool6) => {
10335
+ items: paginatedTools.map((tool5) => {
9922
10336
  const object = {};
9923
10337
  requestedFields.forEach((field) => {
9924
- object[field] = tool6[field];
10338
+ object[field] = tool5[field];
9925
10339
  });
9926
10340
  return object;
9927
10341
  }),
@@ -9931,7 +10345,7 @@ type PageInfo {
9931
10345
  };
9932
10346
  };
9933
10347
  resolvers.Query["toolCategories"] = async () => {
9934
- const array = tools.map((tool6) => tool6.category).filter((category) => category && typeof category === "string");
10348
+ const array = tools.map((tool5) => tool5.category).filter((category) => category && typeof category === "string");
9935
10349
  array.push("contexts");
9936
10350
  array.push("agents");
9937
10351
  return [...new Set(array)].sort();
@@ -10291,13 +10705,13 @@ type AgentWorldAgent {
10291
10705
  }
10292
10706
 
10293
10707
  // src/exulu/routes.ts
10294
- var import_express5 = require("@as-integrations/express5");
10708
+ var import_express52 = require("@as-integrations/express5");
10295
10709
  var import_utils5 = require("@apollo/utils.keyvaluecache");
10296
10710
  var import_body_parser = __toESM(require("body-parser"), 1);
10297
- var import_crypto_js6 = __toESM(require("crypto-js"), 1);
10711
+ var import_crypto_js7 = __toESM(require("crypto-js"), 1);
10298
10712
  var import_openai = __toESM(require("openai"), 1);
10299
- var import_fs3 = __toESM(require("fs"), 1);
10300
- var import_node_crypto4 = require("crypto");
10713
+ var import_fs2 = __toESM(require("fs"), 1);
10714
+ var import_node_crypto5 = require("crypto");
10301
10715
  var import_api2 = require("@opentelemetry/api");
10302
10716
  var import_sdk = __toESM(require("@anthropic-ai/sdk"), 1);
10303
10717
 
@@ -10327,11 +10741,11 @@ var CLAUDE_MESSAGES = {
10327
10741
  };
10328
10742
 
10329
10743
  // src/exulu/routes.ts
10330
- var import_ai9 = require("ai");
10744
+ var import_ai10 = require("ai");
10331
10745
  var import_cookie_parser = __toESM(require("cookie-parser"), 1);
10332
10746
 
10333
10747
  // src/exulu/provider.ts
10334
- var import_zod11 = require("zod");
10748
+ var import_zod10 = require("zod");
10335
10749
  var import_ai8 = require("ai");
10336
10750
 
10337
10751
  // src/utils/generate-slug.ts
@@ -10389,7 +10803,7 @@ async function clearSessionCurrentTask(session) {
10389
10803
  }
10390
10804
 
10391
10805
  // src/exulu/provider.ts
10392
- var import_fs2 = __toESM(require("fs"), 1);
10806
+ var import_fs = __toESM(require("fs"), 1);
10393
10807
  var ExuluProvider = class {
10394
10808
  // Must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or
10395
10809
  // underscores and be a max length of 80 characters and at least 5 characters long.
@@ -10471,9 +10885,9 @@ var ExuluProvider = class {
10471
10885
  name: `${agent.name}`,
10472
10886
  type: "agent",
10473
10887
  category: "agents",
10474
- inputSchema: import_zod11.z.object({
10475
- prompt: import_zod11.z.string().describe("The prompt (usually a question for the agent) to send to the agent."),
10476
- information: import_zod11.z.string().describe("A summary of relevant context / information from the current session")
10888
+ inputSchema: import_zod10.z.object({
10889
+ prompt: import_zod10.z.string().describe("The prompt (usually a question for the agent) to send to the agent."),
10890
+ information: import_zod10.z.string().describe("A summary of relevant context / information from the current session")
10477
10891
  }),
10478
10892
  description: `This tool calls an agent named: ${agent.name}. The agent does the following: ${agent.description}.`,
10479
10893
  config: [],
@@ -10676,12 +11090,12 @@ var ExuluProvider = class {
10676
11090
  system += "\n\n" + memoryContext;
10677
11091
  }
10678
11092
  const includesContextSearchTool = currentTools?.some(
10679
- (tool6) => tool6.name.toLowerCase().includes("context_search") || tool6.id.includes("context_search") || tool6.type === "context"
11093
+ (tool5) => tool5.name.toLowerCase().includes("context_search") || tool5.id.includes("context_search") || tool5.type === "context"
10680
11094
  );
10681
11095
  const includesWebSearchTool = currentTools?.some(
10682
- (tool6) => tool6.name.toLowerCase().includes("web_search") || tool6.id.includes("web_search") || tool6.type === "web_search"
11096
+ (tool5) => tool5.name.toLowerCase().includes("web_search") || tool5.id.includes("web_search") || tool5.type === "web_search"
10683
11097
  );
10684
- console.log("[EXULU] Current tools: " + currentTools?.map((tool6) => tool6.name).join("\n"));
11098
+ console.log("[EXULU] Current tools: " + currentTools?.map((tool5) => tool5.name).join("\n"));
10685
11099
  console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
10686
11100
  if (includesContextSearchTool) {
10687
11101
  system += `
@@ -11054,7 +11468,7 @@ ${extractedText}
11054
11468
  // todo make this configurable?
11055
11469
  page: 1
11056
11470
  });
11057
- import_fs2.default.writeFileSync("pre-fetched-relevant-information.json", JSON.stringify(result2, null, 2));
11471
+ import_fs.default.writeFileSync("pre-fetched-relevant-information.json", JSON.stringify(result2, null, 2));
11058
11472
  if (result2?.chunks?.length) {
11059
11473
  memoryContext = `
11060
11474
  <pre-fetched relevant information for this query>:
@@ -11079,12 +11493,12 @@ ${extractedText}
11079
11493
  system += "\n\n" + memoryContext;
11080
11494
  }
11081
11495
  const includesContextSearchTool = currentTools?.some(
11082
- (tool6) => tool6.name.toLowerCase().includes("context_search") || tool6.id.includes("context_search") || tool6.type === "context"
11496
+ (tool5) => tool5.name.toLowerCase().includes("context_search") || tool5.id.includes("context_search") || tool5.type === "context"
11083
11497
  );
11084
11498
  const includesWebSearchTool = currentTools?.some(
11085
- (tool6) => tool6.name.toLowerCase().includes("web_search") || tool6.id.includes("web_search") || tool6.type === "web_search"
11499
+ (tool5) => tool5.name.toLowerCase().includes("web_search") || tool5.id.includes("web_search") || tool5.type === "web_search"
11086
11500
  );
11087
- console.log("[EXULU] Current tools: " + currentTools?.map((tool6) => tool6.name).join("\n"));
11501
+ console.log("[EXULU] Current tools: " + currentTools?.map((tool5) => tool5.name).join("\n"));
11088
11502
  console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
11089
11503
  console.log("[EXULU] Includes web search tool: " + includesWebSearchTool);
11090
11504
  if (includesContextSearchTool) {
@@ -11129,7 +11543,7 @@ ${extractedText}
11129
11543
 
11130
11544
  When a tool execution is not approved by the user, do not retry it unless explicitly asked by the user. ' +
11131
11545
  'Inform the user that the action was not performed.`;
11132
- import_fs2.default.writeFileSync("system-prompt.txt", system);
11546
+ import_fs.default.writeFileSync("system-prompt.txt", system);
11133
11547
  const result = (0, import_ai8.streamText)({
11134
11548
  temperature: 0,
11135
11549
  // TODO Make this configurable
@@ -11274,13 +11688,458 @@ var providerRateLimiter = async (key, windowSeconds, limit, points) => {
11274
11688
  }
11275
11689
  };
11276
11690
 
11691
+ // src/exulu/openai-gateway.ts
11692
+ var import_express2 = require("express");
11693
+ var import_ai9 = require("ai");
11694
+ var import_node_crypto4 = require("crypto");
11695
+ var import_crypto_js6 = __toESM(require("crypto-js"), 1);
11696
+ var import_express3 = __toESM(require("express"), 1);
11697
+ function convertOpenAIMessagesToCoreMessages(messages) {
11698
+ const systemParts = [];
11699
+ const coreMessages = [];
11700
+ for (const msg of messages) {
11701
+ if (msg.role === "system") {
11702
+ systemParts.push(typeof msg.content === "string" ? msg.content : "");
11703
+ continue;
11704
+ }
11705
+ if (msg.role === "user") {
11706
+ if (typeof msg.content === "string") {
11707
+ coreMessages.push({ role: "user", content: msg.content });
11708
+ } else if (Array.isArray(msg.content)) {
11709
+ const parts = msg.content.flatMap((part) => {
11710
+ if (part.type === "text") return [{ type: "text", text: part.text }];
11711
+ if (part.type === "image_url") return [{ type: "image", image: part.image_url.url }];
11712
+ return [];
11713
+ });
11714
+ coreMessages.push({ role: "user", content: parts });
11715
+ }
11716
+ continue;
11717
+ }
11718
+ if (msg.role === "assistant") {
11719
+ if (msg.tool_calls && msg.tool_calls.length > 0) {
11720
+ const parts = [];
11721
+ if (typeof msg.content === "string" && msg.content) {
11722
+ parts.push({ type: "text", text: msg.content });
11723
+ }
11724
+ for (const tc of msg.tool_calls) {
11725
+ parts.push({
11726
+ type: "tool-call",
11727
+ toolCallId: tc.id,
11728
+ toolName: tc.function.name,
11729
+ args: JSON.parse(tc.function.arguments || "{}")
11730
+ });
11731
+ }
11732
+ coreMessages.push({ role: "assistant", content: parts });
11733
+ } else {
11734
+ coreMessages.push({
11735
+ role: "assistant",
11736
+ content: typeof msg.content === "string" ? msg.content : ""
11737
+ });
11738
+ }
11739
+ continue;
11740
+ }
11741
+ if (msg.role === "tool") {
11742
+ coreMessages.push({
11743
+ role: "tool",
11744
+ content: [
11745
+ {
11746
+ type: "tool-result",
11747
+ toolCallId: msg.tool_call_id ?? "",
11748
+ result: msg.content
11749
+ }
11750
+ ]
11751
+ });
11752
+ }
11753
+ }
11754
+ return { systemPrompt: systemParts.join("\n\n"), coreMessages };
11755
+ }
11756
+ async function writeStatistics(agent, project, user, inputTokens, outputTokens) {
11757
+ const label = agent.name;
11758
+ const trigger = "agent";
11759
+ const projectId = project?.id ? { project: project.id } : {};
11760
+ await Promise.all([
11761
+ updateStatistic({
11762
+ name: "count",
11763
+ label,
11764
+ type: STATISTICS_TYPE_ENUM.AGENT_RUN,
11765
+ trigger,
11766
+ count: 1,
11767
+ user: user.id,
11768
+ role: user.role?.id,
11769
+ ...projectId
11770
+ }),
11771
+ ...inputTokens ? [
11772
+ updateStatistic({
11773
+ name: "inputTokens",
11774
+ label,
11775
+ type: STATISTICS_TYPE_ENUM.AGENT_RUN,
11776
+ trigger,
11777
+ count: inputTokens,
11778
+ user: user.id,
11779
+ role: user.role?.id,
11780
+ ...projectId
11781
+ })
11782
+ ] : [],
11783
+ ...outputTokens ? [
11784
+ updateStatistic({
11785
+ name: "outputTokens",
11786
+ label,
11787
+ type: STATISTICS_TYPE_ENUM.AGENT_RUN,
11788
+ trigger,
11789
+ count: outputTokens,
11790
+ user: user.id,
11791
+ role: user.role?.id,
11792
+ ...projectId
11793
+ })
11794
+ ] : []
11795
+ ]);
11796
+ }
11797
+ var registerOpenAIGatewayRoutes = async (app, providers, tools, contexts, config, rerankers) => {
11798
+ const { agentsSchema: agentsSchema4, projectsSchema: projectsSchema4 } = coreSchemas.get();
11799
+ app.get(
11800
+ "/gateway/open-ai/v1/models",
11801
+ async (req, res) => {
11802
+ try {
11803
+ const authResult = await requestValidators.authenticate(req);
11804
+ if (!authResult.user?.id) {
11805
+ res.status(authResult.code || 401).json({ error: { message: authResult.message, type: "authentication_error" } });
11806
+ return;
11807
+ }
11808
+ const { db: db2 } = await postgresClient();
11809
+ let projectsQuery = db2("projects").select("id", "name");
11810
+ projectsQuery = applyAccessControl(projectsSchema4(), projectsQuery, authResult.user);
11811
+ const projects = await projectsQuery;
11812
+ let agentsQuery = db2("agents").select("id", "name");
11813
+ agentsQuery = applyAccessControl(agentsSchema4(), agentsQuery, authResult.user);
11814
+ const agents = await agentsQuery;
11815
+ const data = projects.flatMap(
11816
+ (p) => agents.map((a) => ({
11817
+ id: `${p.name}/${a.name}`,
11818
+ object: "model",
11819
+ created: 0,
11820
+ owned_by: "exulu"
11821
+ }))
11822
+ );
11823
+ res.json({ object: "list", data });
11824
+ } catch (error) {
11825
+ console.error("[OPENAI GATEWAY] /v1/models error:", error);
11826
+ res.status(500).json({ error: { message: error.message, type: "server_error" } });
11827
+ }
11828
+ }
11829
+ );
11830
+ app.get(
11831
+ "/gateway/open-ai/v1/models/:projectId/:agentId",
11832
+ async (req, res) => {
11833
+ try {
11834
+ const authResult = await requestValidators.authenticate(req);
11835
+ if (!authResult.user?.id) {
11836
+ res.status(authResult.code || 401).json({ error: { message: authResult.message, type: "authentication_error" } });
11837
+ return;
11838
+ }
11839
+ const { db: db2 } = await postgresClient();
11840
+ let projectQuery = db2("projects").select("id", "name");
11841
+ projectQuery = applyAccessControl(projectsSchema4(), projectQuery, authResult.user);
11842
+ projectQuery.where({ id: req.params.projectId });
11843
+ const project = await projectQuery.first();
11844
+ let agentQuery = db2("agents").select("id", "name");
11845
+ agentQuery = applyAccessControl(agentsSchema4(), agentQuery, authResult.user);
11846
+ agentQuery.where({ id: req.params.agentId });
11847
+ const agent = await agentQuery.first();
11848
+ if (!project || !agent) {
11849
+ res.status(404).json({ error: { message: "Model not found", type: "invalid_request_error" } });
11850
+ return;
11851
+ }
11852
+ res.json({
11853
+ id: `${project.name}/${agent.name}`,
11854
+ object: "model",
11855
+ created: 0,
11856
+ owned_by: "exulu"
11857
+ });
11858
+ } catch (error) {
11859
+ console.error("[OPENAI GATEWAY] /v1/models/:id error:", error);
11860
+ res.status(500).json({ error: { message: error.message, type: "server_error" } });
11861
+ }
11862
+ }
11863
+ );
11864
+ app.post(
11865
+ "/gateway/open-ai/v1/chat/completions",
11866
+ import_express3.default.json({ limit: REQUEST_SIZE_LIMIT }),
11867
+ async (req, res) => {
11868
+ try {
11869
+ const { db: db2 } = await postgresClient();
11870
+ const authResult = await requestValidators.authenticate(req);
11871
+ if (!authResult.user?.id) {
11872
+ res.status(authResult.code || 401).json({ error: { message: authResult.message, type: "authentication_error" } });
11873
+ return;
11874
+ }
11875
+ const user = authResult.user;
11876
+ const modelId = req.body.model;
11877
+ if (!modelId) {
11878
+ res.status(400).json({
11879
+ error: { message: "Missing required field: model", type: "invalid_request_error" }
11880
+ });
11881
+ return;
11882
+ }
11883
+ const separatorIndex = modelId.indexOf("/");
11884
+ if (separatorIndex === -1) {
11885
+ res.status(400).json({
11886
+ error: { message: "Invalid model format. Expected: 'projectname/agentname'", type: "invalid_request_error" }
11887
+ });
11888
+ return;
11889
+ }
11890
+ const projectName = modelId.substring(0, separatorIndex);
11891
+ const agentName = modelId.substring(separatorIndex + 1);
11892
+ let agentQuery = db2("agents").select("*");
11893
+ agentQuery = applyAccessControl(agentsSchema4(), agentQuery, user);
11894
+ agentQuery.where({ name: agentName });
11895
+ const agent = await agentQuery.first();
11896
+ if (!agent) {
11897
+ res.status(404).json({
11898
+ error: {
11899
+ message: `Agent '${agentName}' not found or you do not have access to it.`,
11900
+ type: "invalid_request_error"
11901
+ }
11902
+ });
11903
+ return;
11904
+ }
11905
+ let project = null;
11906
+ if (projectName) {
11907
+ let projectQuery = db2("projects").select("*");
11908
+ projectQuery = applyAccessControl(projectsSchema4(), projectQuery, user);
11909
+ projectQuery.where({ name: projectName });
11910
+ project = await projectQuery.first();
11911
+ }
11912
+ if (!process.env.NEXTAUTH_SECRET) {
11913
+ res.status(500).json({ error: { message: "Server configuration error", type: "server_error" } });
11914
+ return;
11915
+ }
11916
+ if (!agent.providerapikey) {
11917
+ res.status(400).json({
11918
+ error: { message: "Agent has no API key configured", type: "invalid_request_error" }
11919
+ });
11920
+ return;
11921
+ }
11922
+ const variable = await db2.from("variables").where({ name: agent.providerapikey }).first();
11923
+ if (!variable) {
11924
+ res.status(400).json({
11925
+ error: { message: "API key variable not found", type: "invalid_request_error" }
11926
+ });
11927
+ return;
11928
+ }
11929
+ if (!variable.encrypted) {
11930
+ res.status(400).json({
11931
+ error: { message: "API key variable must be encrypted", type: "invalid_request_error" }
11932
+ });
11933
+ return;
11934
+ }
11935
+ const bytes = import_crypto_js6.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
11936
+ const providerapikey = bytes.toString(import_crypto_js6.default.enc.Utf8);
11937
+ const provider = providers.find((p) => p.id === agent.provider);
11938
+ if (!provider?.config?.model?.create) {
11939
+ res.status(400).json({
11940
+ error: { message: "No provider configured for this agent", type: "invalid_request_error" }
11941
+ });
11942
+ return;
11943
+ }
11944
+ const languageModel = provider.config.model.create({ apiKey: providerapikey });
11945
+ const disabledTools = req.body.disabledTools ?? [];
11946
+ const enabledTools = await getEnabledTools(
11947
+ agent,
11948
+ tools,
11949
+ contexts ?? [],
11950
+ rerankers ?? [],
11951
+ disabledTools,
11952
+ providers,
11953
+ user
11954
+ );
11955
+ const convertedTools = await convertExuluToolsToAiSdkTools(
11956
+ enabledTools,
11957
+ [],
11958
+ tools,
11959
+ agent.tools,
11960
+ providerapikey,
11961
+ contexts,
11962
+ rerankers,
11963
+ user,
11964
+ config,
11965
+ void 0,
11966
+ req,
11967
+ project?.id,
11968
+ void 0,
11969
+ languageModel,
11970
+ agent
11971
+ );
11972
+ const openaiMessages = req.body.messages ?? [];
11973
+ const { systemPrompt: requestSystemPrompt, coreMessages } = convertOpenAIMessagesToCoreMessages(openaiMessages);
11974
+ const agentInstructions = agent.instructions ?? "";
11975
+ const systemParts = [
11976
+ agentInstructions ? `You are an agent named: ${agent.name}
11977
+ Here are your instructions: ${agentInstructions}` : `You are an agent named: ${agent.name}`,
11978
+ project ? `The project you are working on is: ${project.name}${project.description ? `
11979
+ ${project.description}` : ""}` : "",
11980
+ requestSystemPrompt
11981
+ ].filter(Boolean);
11982
+ const systemPrompt = systemParts.join("\n\n");
11983
+ const completionId = `chatcmpl-${(0, import_node_crypto4.randomUUID)()}`;
11984
+ const created = Math.floor(Date.now() / 1e3);
11985
+ const hasTools = Object.keys(convertedTools).length > 0;
11986
+ if (req.body.stream === true) {
11987
+ res.setHeader("Content-Type", "text/event-stream");
11988
+ res.setHeader("Cache-Control", "no-cache");
11989
+ res.setHeader("Connection", "keep-alive");
11990
+ const result = (0, import_ai9.streamText)({
11991
+ model: languageModel,
11992
+ system: systemPrompt || void 0,
11993
+ messages: coreMessages,
11994
+ tools: hasTools ? convertedTools : void 0,
11995
+ maxRetries: 2,
11996
+ stopWhen: [(0, import_ai9.stepCountIs)(5)],
11997
+ onError: (error) => {
11998
+ console.error("[OPENAI GATEWAY] stream error:", error);
11999
+ }
12000
+ });
12001
+ res.write(
12002
+ `data: ${JSON.stringify({
12003
+ id: completionId,
12004
+ object: "chat.completion.chunk",
12005
+ created,
12006
+ model: modelId,
12007
+ choices: [{ index: 0, delta: { role: "assistant", content: "" }, finish_reason: null }]
12008
+ })}
12009
+
12010
+ `
12011
+ );
12012
+ let inputTokens = 0;
12013
+ let outputTokens = 0;
12014
+ for await (const chunk of result.fullStream) {
12015
+ if (chunk.type === "text-delta") {
12016
+ res.write(
12017
+ `data: ${JSON.stringify({
12018
+ id: completionId,
12019
+ object: "chat.completion.chunk",
12020
+ created,
12021
+ model: modelId,
12022
+ choices: [{ index: 0, delta: { content: chunk.text }, finish_reason: null }]
12023
+ })}
12024
+
12025
+ `
12026
+ );
12027
+ } else if (chunk.type === "tool-input-start") {
12028
+ res.write(
12029
+ `data: ${JSON.stringify({
12030
+ id: completionId,
12031
+ object: "chat.completion.chunk",
12032
+ created,
12033
+ model: modelId,
12034
+ choices: [
12035
+ {
12036
+ index: 0,
12037
+ delta: {
12038
+ tool_calls: [
12039
+ {
12040
+ index: 0,
12041
+ id: chunk.id,
12042
+ type: "function",
12043
+ function: { name: chunk.toolName, arguments: "" }
12044
+ }
12045
+ ]
12046
+ },
12047
+ finish_reason: null
12048
+ }
12049
+ ]
12050
+ })}
12051
+
12052
+ `
12053
+ );
12054
+ } else if (chunk.type === "tool-input-delta") {
12055
+ res.write(
12056
+ `data: ${JSON.stringify({
12057
+ id: completionId,
12058
+ object: "chat.completion.chunk",
12059
+ created,
12060
+ model: modelId,
12061
+ choices: [
12062
+ {
12063
+ index: 0,
12064
+ delta: { tool_calls: [{ index: 0, function: { arguments: chunk.delta } }] },
12065
+ finish_reason: null
12066
+ }
12067
+ ]
12068
+ })}
12069
+
12070
+ `
12071
+ );
12072
+ } else if (chunk.type === "finish") {
12073
+ inputTokens = chunk.usage?.inputTokens ?? 0;
12074
+ outputTokens = chunk.usage?.outputTokens ?? 0;
12075
+ const finishReason = chunk.finishReason === "tool-calls" ? "tool_calls" : "stop";
12076
+ res.write(
12077
+ `data: ${JSON.stringify({
12078
+ id: completionId,
12079
+ object: "chat.completion.chunk",
12080
+ created,
12081
+ model: modelId,
12082
+ choices: [{ index: 0, delta: {}, finish_reason: finishReason }],
12083
+ usage: {
12084
+ prompt_tokens: inputTokens,
12085
+ completion_tokens: outputTokens,
12086
+ total_tokens: inputTokens + outputTokens
12087
+ }
12088
+ })}
12089
+
12090
+ `
12091
+ );
12092
+ }
12093
+ }
12094
+ res.write("data: [DONE]\n\n");
12095
+ res.end();
12096
+ await writeStatistics(agent, project, user, inputTokens, outputTokens);
12097
+ } else {
12098
+ const { text, usage } = await (0, import_ai9.generateText)({
12099
+ model: languageModel,
12100
+ system: systemPrompt || void 0,
12101
+ messages: coreMessages,
12102
+ tools: hasTools ? convertedTools : void 0,
12103
+ maxRetries: 2,
12104
+ stopWhen: [(0, import_ai9.stepCountIs)(5)]
12105
+ });
12106
+ res.json({
12107
+ id: completionId,
12108
+ object: "chat.completion",
12109
+ created,
12110
+ model: agentId,
12111
+ choices: [
12112
+ {
12113
+ index: 0,
12114
+ message: { role: "assistant", content: text },
12115
+ finish_reason: "stop"
12116
+ }
12117
+ ],
12118
+ usage: {
12119
+ prompt_tokens: usage.promptTokens,
12120
+ completion_tokens: usage.completionTokens,
12121
+ total_tokens: usage.totalTokens
12122
+ }
12123
+ });
12124
+ await writeStatistics(agent, project, user, usage.promptTokens, usage.completionTokens);
12125
+ }
12126
+ } catch (error) {
12127
+ console.error("[OPENAI GATEWAY] /v1/chat/completions error:", error);
12128
+ if (!res.headersSent) {
12129
+ res.status(500).json({ error: { message: error.message, type: "server_error" } });
12130
+ }
12131
+ }
12132
+ }
12133
+ );
12134
+ };
12135
+
11277
12136
  // src/exulu/routes.ts
11278
12137
  var import_zod_from_json_schema = require("zod-from-json-schema");
11279
12138
  var REQUEST_SIZE_LIMIT = "50mb";
11280
12139
  var getExuluVersionNumber = async () => {
11281
12140
  try {
11282
- const path5 = process.cwd();
11283
- const packageJson = import_fs3.default.readFileSync(path5 + "/package.json", "utf8");
12141
+ const path3 = process.cwd();
12142
+ const packageJson = import_fs2.default.readFileSync(path3 + "/package.json", "utf8");
11284
12143
  const packageData = JSON.parse(packageJson);
11285
12144
  const exuluVersion = packageData.dependencies["@exulu/backend"];
11286
12145
  console.log(`[EXULU] Installed exulu-backend version: ${exuluVersion}`);
@@ -11305,6 +12164,7 @@ var {
11305
12164
  agentMessagesSchema: agentMessagesSchema2,
11306
12165
  rolesSchema: rolesSchema2,
11307
12166
  usersSchema: usersSchema2,
12167
+ skillsSchema: skillsSchema2,
11308
12168
  variablesSchema: variablesSchema2,
11309
12169
  workflowTemplatesSchema: workflowTemplatesSchema2,
11310
12170
  rbacSchema: rbacSchema2,
@@ -11324,7 +12184,7 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
11324
12184
  if (tracer) {
11325
12185
  console.log("[EXULU] tracer configured", tracer);
11326
12186
  }
11327
- app.use(import_express3.default.json({ limit: REQUEST_SIZE_LIMIT }));
12187
+ app.use(import_express5.default.json({ limit: REQUEST_SIZE_LIMIT }));
11328
12188
  app.use((0, import_cors.default)(corsOptions));
11329
12189
  app.use(import_body_parser.default.urlencoded({ extended: true, limit: REQUEST_SIZE_LIMIT }));
11330
12190
  app.use(import_body_parser.default.json({ limit: REQUEST_SIZE_LIMIT }));
@@ -11349,6 +12209,7 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
11349
12209
  const schema = createSDL(
11350
12210
  [
11351
12211
  usersSchema2(),
12212
+ skillsSchema2(),
11352
12213
  rolesSchema2(),
11353
12214
  agentsSchema2(),
11354
12215
  feedbackSchema2(),
@@ -11384,8 +12245,8 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
11384
12245
  app.use(
11385
12246
  "/graphql",
11386
12247
  (0, import_cors.default)(corsOptions),
11387
- import_express3.default.json({ limit: REQUEST_SIZE_LIMIT }),
11388
- (0, import_express5.expressMiddleware)(server, {
12248
+ import_express5.default.json({ limit: REQUEST_SIZE_LIMIT }),
12249
+ (0, import_express52.expressMiddleware)(server, {
11389
12250
  context: async ({ req }) => {
11390
12251
  const authenticationResult = await requestValidators.authenticate(req);
11391
12252
  if (!authenticationResult.user?.id) {
@@ -11506,8 +12367,8 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
11506
12367
  return;
11507
12368
  }
11508
12369
  if (variable.encrypted) {
11509
- const bytes = import_crypto_js6.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
11510
- providerapikey = bytes.toString(import_crypto_js6.default.enc.Utf8);
12370
+ const bytes = import_crypto_js7.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
12371
+ providerapikey = bytes.toString(import_crypto_js7.default.enc.Utf8);
11511
12372
  }
11512
12373
  const openai = new import_openai.default({
11513
12374
  apiKey: providerapikey
@@ -11556,7 +12417,7 @@ Mood: friendly and intelligent.
11556
12417
  });
11557
12418
  return;
11558
12419
  }
11559
- const uuid = (0, import_node_crypto4.randomUUID)();
12420
+ const uuid = (0, import_node_crypto5.randomUUID)();
11560
12421
  const image_url = await uploadFile(Buffer.from(image_base64, "base64"), `${uuid}.png`, config, {
11561
12422
  contentType: "image/png"
11562
12423
  }, authenticationResult.user?.id, void 0, true);
@@ -11725,8 +12586,8 @@ Mood: friendly and intelligent.
11725
12586
  return;
11726
12587
  }
11727
12588
  if (variable.encrypted) {
11728
- const bytes = import_crypto_js6.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
11729
- providerapikey = bytes.toString(import_crypto_js6.default.enc.Utf8);
12589
+ const bytes = import_crypto_js7.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
12590
+ providerapikey = bytes.toString(import_crypto_js7.default.enc.Utf8);
11730
12591
  }
11731
12592
  }
11732
12593
  if (!!headers.stream) {
@@ -11794,7 +12655,7 @@ ${customInstructions}` : agent.instructions;
11794
12655
  }
11795
12656
  return JSON.stringify(error);
11796
12657
  },
11797
- generateMessageId: (0, import_ai9.createIdGenerator)({
12658
+ generateMessageId: (0, import_ai10.createIdGenerator)({
11798
12659
  prefix: "msg_",
11799
12660
  size: 16
11800
12661
  }),
@@ -11903,7 +12764,7 @@ ${customInstructions}` : agent.instructions;
11903
12764
  });
11904
12765
  app.use(
11905
12766
  "/gateway/anthropic/:agent/:project",
11906
- import_express3.default.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }),
12767
+ import_express5.default.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }),
11907
12768
  async (req, res) => {
11908
12769
  try {
11909
12770
  if (!req.body.tools) {
@@ -11987,8 +12848,8 @@ ${customInstructions}` : agent.instructions;
11987
12848
  return;
11988
12849
  }
11989
12850
  if (variable.encrypted) {
11990
- const bytes = import_crypto_js6.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
11991
- anthropicApiKey = bytes.toString(import_crypto_js6.default.enc.Utf8);
12851
+ const bytes = import_crypto_js7.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
12852
+ anthropicApiKey = bytes.toString(import_crypto_js7.default.enc.Utf8);
11992
12853
  }
11993
12854
  const headers = {
11994
12855
  "x-api-key": anthropicApiKey,
@@ -12067,14 +12928,14 @@ ${customInstructions}` : agent?.instructions;
12067
12928
  console.log("[EXULU] Using tool", toolName);
12068
12929
  const inputs = event.message?.input;
12069
12930
  const id = event.message?.id;
12070
- const tool6 = enabledTools.find(
12071
- (tool7) => tool7.id === toolName.replace("exulu_", "")
12931
+ const tool5 = enabledTools.find(
12932
+ (tool6) => tool6.id === toolName.replace("exulu_", "")
12072
12933
  );
12073
- if (!tool6 || !tool6.tool.execute) {
12934
+ if (!tool5 || !tool5.tool.execute) {
12074
12935
  console.error("[EXULU] Tool not found or not enabled.", toolName);
12075
12936
  continue;
12076
12937
  }
12077
- const toolResult = await tool6.tool.execute(inputs, {
12938
+ const toolResult = await tool5.tool.execute(inputs, {
12078
12939
  toolCallId: id,
12079
12940
  messages: [
12080
12941
  {
@@ -12166,33 +13027,437 @@ data: ${JSON.stringify(event)}
12166
13027
  }
12167
13028
  }
12168
13029
  );
12169
- app.use(import_express3.default.static("public"));
12170
- return app;
12171
- };
12172
- var createCustomAnthropicStreamingMessage = (message) => {
12173
- const responseData = {
12174
- type: "message",
12175
- content: [
12176
- {
12177
- type: "text",
12178
- text: message
13030
+ function buildFileTree(files, stripPrefix) {
13031
+ const root = { name: "/", path: "/", key: "", type: "folder", children: [] };
13032
+ for (const file of files) {
13033
+ const relativePath = file.key.startsWith(stripPrefix) ? file.key.slice(stripPrefix.length) : file.key;
13034
+ const parts = relativePath.split("/").filter(Boolean);
13035
+ let current = root;
13036
+ for (let i = 0; i < parts.length; i++) {
13037
+ const part = parts[i];
13038
+ const isFile = i === parts.length - 1;
13039
+ const existingChild = current.children?.find((c) => c.name === part);
13040
+ if (existingChild) {
13041
+ current = existingChild;
13042
+ } else {
13043
+ const nodePath = "/" + parts.slice(0, i + 1).join("/");
13044
+ const node = isFile ? {
13045
+ name: part,
13046
+ path: nodePath,
13047
+ key: file.key,
13048
+ type: "file",
13049
+ size: file.size,
13050
+ lastModified: file.lastModified
13051
+ } : { name: part, path: nodePath, key: "", type: "folder", children: [] };
13052
+ current.children = current.children ?? [];
13053
+ current.children.push(node);
13054
+ current = node;
13055
+ }
12179
13056
  }
12180
- ]
12181
- };
12182
- const jsonString = JSON.stringify(responseData);
12183
- const arrayBuffer = new TextEncoder().encode(jsonString).buffer;
12184
- return arrayBuffer;
12185
- };
12186
-
12187
- // src/mcp/index.ts
12188
- var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
12189
- var import_node_crypto5 = require("crypto");
12190
- var import_streamableHttp = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
12191
- var import_types2 = require("@modelcontextprotocol/sdk/types.js");
12192
- var import_express4 = require("express");
12193
- var import_api3 = require("@opentelemetry/api");
12194
- var import_crypto_js7 = __toESM(require("crypto-js"), 1);
12195
- var import_zod12 = require("zod");
13057
+ }
13058
+ return root;
13059
+ }
13060
+ app.post("/skills/:skillId/init", async (req, res) => {
13061
+ const authResult = await requestValidators.authenticate(req);
13062
+ if (!authResult.user?.id) {
13063
+ res.status(authResult.code ?? 401).json({ detail: authResult.message });
13064
+ return;
13065
+ }
13066
+ const { skillId } = req.params;
13067
+ const { name = "Skill", description = "" } = req.body;
13068
+ const skillMdContent = [
13069
+ `# ${name}`,
13070
+ "",
13071
+ description || "Describe what this skill does and when to use it.",
13072
+ "",
13073
+ "## Overview",
13074
+ "",
13075
+ "...",
13076
+ "",
13077
+ "## Usage",
13078
+ "",
13079
+ "..."
13080
+ ].join("\n");
13081
+ const s3Key = `skills/${skillId}/v1/SKILL.md`;
13082
+ try {
13083
+ await uploadFile(Buffer.from(skillMdContent, "utf-8"), s3Key, config, { contentType: "text/markdown" }, void 0, void 0, true);
13084
+ } catch (err) {
13085
+ console.error("[SKILLS] Failed to create SKILL.md in S3", err);
13086
+ res.status(500).json({ detail: "Failed to initialise skill folder in S3." });
13087
+ return;
13088
+ }
13089
+ const { db: db2 } = await postgresClient();
13090
+ await db2("skills").where({ id: skillId }).update({
13091
+ s3folder: `skills/${skillId}`,
13092
+ current_version: 1,
13093
+ history: JSON.stringify([
13094
+ { version: 1, created_at: (/* @__PURE__ */ new Date()).toISOString(), label: "Initial" }
13095
+ ])
13096
+ });
13097
+ res.json({ version: 1, skillMdKey: s3Key });
13098
+ });
13099
+ app.get("/skills/:skillId/files", async (req, res) => {
13100
+ const authResult = await requestValidators.authenticate(req);
13101
+ if (!authResult.user?.id) {
13102
+ res.status(authResult.code ?? 401).json({ detail: authResult.message });
13103
+ return;
13104
+ }
13105
+ const { skillId } = req.params;
13106
+ const { db: db2 } = await postgresClient();
13107
+ const skill = await db2("skills").where({ id: skillId }).first();
13108
+ if (!skill) {
13109
+ res.status(404).json({ detail: "Skill not found." });
13110
+ return;
13111
+ }
13112
+ const version = req.query.version ? Number(req.query.version) : skill.current_version ?? 1;
13113
+ const prefix = `skills/${skillId}/v${version}/`;
13114
+ const files = await listS3ObjectsByPrefix(prefix, config);
13115
+ const tree = buildFileTree(files, config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` + prefix : prefix);
13116
+ res.json({ version, tree, fileCount: files.length });
13117
+ });
13118
+ app.post("/skills/:skillId/sign", async (req, res) => {
13119
+ const authResult = await requestValidators.authenticate(req);
13120
+ if (!authResult.user?.id) {
13121
+ res.status(authResult.code ?? 401).json({ detail: authResult.message });
13122
+ return;
13123
+ }
13124
+ const { skillId } = req.params;
13125
+ const { filePath, contentType = "application/octet-stream" } = req.body;
13126
+ if (!filePath || typeof filePath !== "string") {
13127
+ res.status(400).json({ detail: "Missing filePath in request body." });
13128
+ return;
13129
+ }
13130
+ const { db: db2 } = await postgresClient();
13131
+ const skill = await db2("skills").where({ id: skillId }).first();
13132
+ if (!skill) {
13133
+ res.status(404).json({ detail: "Skill not found." });
13134
+ return;
13135
+ }
13136
+ const version = skill.current_version ?? 1;
13137
+ const safePath = filePath.replace(/^\/+/, "").replace(/\.\.\//g, "");
13138
+ const s3Key = `skills/${skillId}/v${version}/${safePath}`;
13139
+ const fullKey = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/${s3Key}` : s3Key;
13140
+ const url = await getS3SignedUploadUrl(fullKey, contentType, config);
13141
+ res.json({ key: s3Key, url, method: "PUT" });
13142
+ });
13143
+ app.get("/skills/:skillId/file", async (req, res) => {
13144
+ const authResult = await requestValidators.authenticate(req);
13145
+ if (!authResult.user?.id) {
13146
+ res.status(authResult.code ?? 401).json({ detail: authResult.message });
13147
+ return;
13148
+ }
13149
+ const { skillId } = req.params;
13150
+ const { key } = req.query;
13151
+ if (!key || typeof key !== "string") {
13152
+ res.status(400).json({ detail: "Missing key query parameter." });
13153
+ return;
13154
+ }
13155
+ if (!key.startsWith(`skills/${skillId}/`)) {
13156
+ res.status(403).json({ detail: "Key does not belong to this skill." });
13157
+ return;
13158
+ }
13159
+ const { db: db2 } = await postgresClient();
13160
+ const skill = await db2("skills").where({ id: skillId }).first();
13161
+ if (!skill) {
13162
+ res.status(404).json({ detail: "Skill not found." });
13163
+ return;
13164
+ }
13165
+ const fullKey = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/${key}` : key;
13166
+ const TEXT_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".txt", ".py", ".js", ".ts", ".json", ".yaml", ".yml", ".sh", ".env", ".toml", ".xml", ".html", ".css"]);
13167
+ const ext = key.slice(key.lastIndexOf(".")).toLowerCase();
13168
+ const isText = TEXT_EXTENSIONS.has(ext);
13169
+ const MAX_INLINE_BYTES = 200 * 1024;
13170
+ let content;
13171
+ if (isText) {
13172
+ try {
13173
+ const raw = await getS3ObjectContent(fullKey, config);
13174
+ if (raw.length <= MAX_INLINE_BYTES) {
13175
+ content = raw;
13176
+ }
13177
+ } catch {
13178
+ }
13179
+ }
13180
+ const bucket = config.fileUploads?.s3Bucket ?? "";
13181
+ const url = await getPresignedUrl(bucket, fullKey, config);
13182
+ res.json({ url, content, key });
13183
+ });
13184
+ app.delete("/skills/:skillId/file", async (req, res) => {
13185
+ const authResult = await requestValidators.authenticate(req);
13186
+ if (!authResult.user?.id) {
13187
+ res.status(authResult.code ?? 401).json({ detail: authResult.message });
13188
+ return;
13189
+ }
13190
+ const { skillId } = req.params;
13191
+ const { key, prefix } = req.query;
13192
+ if (!key && !prefix) {
13193
+ res.status(400).json({ detail: "Provide either key or prefix query parameter." });
13194
+ return;
13195
+ }
13196
+ const guard = (s) => !s.startsWith(`skills/${skillId}/`);
13197
+ if (key && typeof key === "string") {
13198
+ if (guard(key)) {
13199
+ res.status(403).json({ detail: "Key does not belong to this skill." });
13200
+ return;
13201
+ }
13202
+ }
13203
+ if (prefix && typeof prefix === "string") {
13204
+ if (guard(prefix)) {
13205
+ res.status(403).json({ detail: "Prefix does not belong to this skill." });
13206
+ return;
13207
+ }
13208
+ }
13209
+ const { db: db2 } = await postgresClient();
13210
+ const skill = await db2("skills").where({ id: skillId }).first();
13211
+ if (!skill) {
13212
+ res.status(404).json({ detail: "Skill not found." });
13213
+ return;
13214
+ }
13215
+ const s3Prefix = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
13216
+ if (key && typeof key === "string") {
13217
+ const fullKey = s3Prefix + key;
13218
+ await deleteS3Object(fullKey, config);
13219
+ res.json({ deleted: 1 });
13220
+ return;
13221
+ }
13222
+ if (prefix && typeof prefix === "string") {
13223
+ const files = await listS3ObjectsByPrefix(prefix, config);
13224
+ await Promise.all(files.map((f) => deleteS3Object(f.key, config)));
13225
+ res.json({ deleted: files.length });
13226
+ return;
13227
+ }
13228
+ });
13229
+ app.post("/skills/:skillId/version", async (req, res) => {
13230
+ const authResult = await requestValidators.authenticate(req);
13231
+ if (!authResult.user?.id) {
13232
+ res.status(authResult.code ?? 401).json({ detail: authResult.message });
13233
+ return;
13234
+ }
13235
+ const { skillId } = req.params;
13236
+ const { label } = req.body;
13237
+ const { db: db2 } = await postgresClient();
13238
+ const skill = await db2("skills").where({ id: skillId }).first();
13239
+ if (!skill) {
13240
+ res.status(404).json({ detail: "Skill not found." });
13241
+ return;
13242
+ }
13243
+ const currentVersion = skill.current_version ?? 1;
13244
+ const newVersion = currentVersion + 1;
13245
+ const currentPrefix = `skills/${skillId}/v${currentVersion}/`;
13246
+ const newPrefix = `skills/${skillId}/v${newVersion}/`;
13247
+ const files = await listS3ObjectsByPrefix(currentPrefix, config);
13248
+ if (files.length === 0) {
13249
+ res.status(400).json({ detail: "No files found in current version to snapshot." });
13250
+ return;
13251
+ }
13252
+ const s3GeneralPrefix = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
13253
+ for (const file of files) {
13254
+ const destKey = file.key.replace(s3GeneralPrefix + currentPrefix, s3GeneralPrefix + newPrefix);
13255
+ await copyS3Object(file.key, destKey, config);
13256
+ }
13257
+ const existingHistory = Array.isArray(skill.history) ? skill.history : [];
13258
+ const newHistory = [
13259
+ ...existingHistory,
13260
+ { version: newVersion, created_at: (/* @__PURE__ */ new Date()).toISOString(), label: label ?? `v${newVersion}` }
13261
+ ];
13262
+ await db2("skills").where({ id: skillId }).update({
13263
+ current_version: newVersion,
13264
+ history: JSON.stringify(newHistory)
13265
+ });
13266
+ res.json({ newVersion, fileCount: files.length });
13267
+ });
13268
+ app.post("/skills/:skillId/rename", async (req, res) => {
13269
+ const authResult = await requestValidators.authenticate(req);
13270
+ if (!authResult.user?.id) {
13271
+ res.status(authResult.code ?? 401).json({ detail: authResult.message });
13272
+ return;
13273
+ }
13274
+ const { skillId } = req.params;
13275
+ const { sourceKey, destPath } = req.body;
13276
+ if (!sourceKey || !destPath) {
13277
+ res.status(400).json({ detail: "sourceKey and destPath are required." });
13278
+ return;
13279
+ }
13280
+ if (!sourceKey.startsWith(`skills/${skillId}/`)) {
13281
+ res.status(403).json({ detail: "sourceKey does not belong to this skill." });
13282
+ return;
13283
+ }
13284
+ const { db: db2 } = await postgresClient();
13285
+ const skill = await db2("skills").where({ id: skillId }).first();
13286
+ if (!skill) {
13287
+ res.status(404).json({ detail: "Skill not found." });
13288
+ return;
13289
+ }
13290
+ const version = skill.current_version ?? 1;
13291
+ const safeDest = destPath.replace(/^\/+/, "").replace(/\.\.\//g, "");
13292
+ const destKey = `skills/${skillId}/v${version}/${safeDest}`;
13293
+ const s3Prefix = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
13294
+ const fullSourceKey = s3Prefix + sourceKey;
13295
+ const fullDestKey = s3Prefix + destKey;
13296
+ await copyS3Object(fullSourceKey, fullDestKey, config);
13297
+ await deleteS3Object(fullSourceKey, config);
13298
+ res.json({ newKey: destKey });
13299
+ });
13300
+ app.get("/skills/:skillId/diff", async (req, res) => {
13301
+ const authResult = await requestValidators.authenticate(req);
13302
+ if (!authResult.user?.id) {
13303
+ res.status(authResult.code ?? 401).json({ detail: authResult.message });
13304
+ return;
13305
+ }
13306
+ const { skillId } = req.params;
13307
+ const fromVersion = Number(req.query.fromVersion);
13308
+ const toVersion = Number(req.query.toVersion);
13309
+ if (!fromVersion || !toVersion) {
13310
+ res.status(400).json({ detail: "fromVersion and toVersion query params are required." });
13311
+ return;
13312
+ }
13313
+ const { db: db2 } = await postgresClient();
13314
+ const skill = await db2("skills").where({ id: skillId }).first();
13315
+ if (!skill) {
13316
+ res.status(404).json({ detail: "Skill not found." });
13317
+ return;
13318
+ }
13319
+ const s3Prefix = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
13320
+ const fromPrefix = `skills/${skillId}/v${fromVersion}/`;
13321
+ const toPrefix = `skills/${skillId}/v${toVersion}/`;
13322
+ const [fromFiles, toFiles] = await Promise.all([
13323
+ listS3ObjectsByPrefix(fromPrefix, config),
13324
+ listS3ObjectsByPrefix(toPrefix, config)
13325
+ ]);
13326
+ const relativise = (files, prefix) => {
13327
+ const full = s3Prefix + prefix;
13328
+ return new Map(files.map((f) => [f.key.replace(full, ""), f]));
13329
+ };
13330
+ const fromMap = relativise(fromFiles, fromPrefix);
13331
+ const toMap = relativise(toFiles, toPrefix);
13332
+ const allPaths = /* @__PURE__ */ new Set([...fromMap.keys(), ...toMap.keys()]);
13333
+ const TEXT_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".txt", ".py", ".js", ".ts", ".json", ".yaml", ".yml", ".sh", ".toml"]);
13334
+ const MAX_DIFF_BYTES = 500 * 1024;
13335
+ const fileDiffs = await Promise.all(
13336
+ [...allPaths].map(async (path3) => {
13337
+ const inFrom = fromMap.has(path3);
13338
+ const inTo = toMap.has(path3);
13339
+ const status = !inFrom ? "added" : !inTo ? "removed" : "modified";
13340
+ if (status === "modified") {
13341
+ const ext = path3.slice(path3.lastIndexOf(".")).toLowerCase();
13342
+ if (!TEXT_EXTENSIONS.has(ext)) {
13343
+ return { path: path3, status };
13344
+ }
13345
+ try {
13346
+ const [fromContent, toContent] = await Promise.all([
13347
+ getS3ObjectContent(s3Prefix + fromPrefix + path3, config),
13348
+ getS3ObjectContent(s3Prefix + toPrefix + path3, config)
13349
+ ]);
13350
+ if (fromContent === toContent) {
13351
+ return { path: path3, status: "unchanged" };
13352
+ }
13353
+ if (fromContent.length + toContent.length > MAX_DIFF_BYTES) {
13354
+ return { path: path3, status };
13355
+ }
13356
+ const fromLines = fromContent.split("\n");
13357
+ const toLines = toContent.split("\n");
13358
+ const diff = buildUnifiedDiff(fromLines, toLines, `v${fromVersion}/${path3}`, `v${toVersion}/${path3}`);
13359
+ return { path: path3, status, diff };
13360
+ } catch {
13361
+ return { path: path3, status };
13362
+ }
13363
+ }
13364
+ return { path: path3, status };
13365
+ })
13366
+ );
13367
+ res.json({
13368
+ fromVersion,
13369
+ toVersion,
13370
+ files: fileDiffs.filter((f) => f.status !== "unchanged")
13371
+ });
13372
+ });
13373
+ app.use(import_express5.default.static("public"));
13374
+ await registerOpenAIGatewayRoutes(app, providers, tools, contexts, config, rerankers);
13375
+ return app;
13376
+ };
13377
+ function buildUnifiedDiff(fromLines, toLines, fromLabel, toLabel) {
13378
+ function lcs(a, b) {
13379
+ const m = a.length;
13380
+ const n = b.length;
13381
+ const dp = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0));
13382
+ for (let i = 1; i <= m; i++) {
13383
+ for (let j = 1; j <= n; j++) {
13384
+ dp[i][j] = a[i - 1] === b[j - 1] ? (dp[i - 1][j - 1] ?? 0) + 1 : Math.max(dp[i - 1][j] ?? 0, dp[i][j - 1] ?? 0);
13385
+ }
13386
+ }
13387
+ return dp;
13388
+ }
13389
+ function diff(a, b) {
13390
+ const table = lcs(a, b);
13391
+ const result = [];
13392
+ let i = a.length;
13393
+ let j = b.length;
13394
+ while (i > 0 || j > 0) {
13395
+ if (i > 0 && j > 0 && a[i - 1] === b[j - 1]) {
13396
+ result.unshift({ op: "=", line: a[i - 1] });
13397
+ i--;
13398
+ j--;
13399
+ } else if (j > 0 && (i === 0 || (table[i][j - 1] ?? 0) >= (table[i - 1][j] ?? 0))) {
13400
+ result.unshift({ op: "+", line: b[j - 1] });
13401
+ j--;
13402
+ } else {
13403
+ result.unshift({ op: "-", line: a[i - 1] });
13404
+ i--;
13405
+ }
13406
+ }
13407
+ return result;
13408
+ }
13409
+ const CONTEXT = 3;
13410
+ const hunks = diff(fromLines, toLines);
13411
+ const lines = [`--- ${fromLabel}`, `+++ ${toLabel}`];
13412
+ let hunkStart = -1;
13413
+ for (let idx = 0; idx < hunks.length; idx++) {
13414
+ const h = hunks[idx];
13415
+ if (h.op !== "=") {
13416
+ if (hunkStart < 0) {
13417
+ hunkStart = Math.max(0, idx - CONTEXT);
13418
+ }
13419
+ } else if (hunkStart >= 0 && idx - hunkStart > CONTEXT * 2) {
13420
+ const slice = hunks.slice(hunkStart, Math.min(idx, hunkStart + (idx - hunkStart)));
13421
+ lines.push(`@@ -${hunkStart + 1} +${hunkStart + 1} @@`);
13422
+ for (const s of slice) {
13423
+ lines.push((s.op === "=" ? " " : s.op) + s.line);
13424
+ }
13425
+ hunkStart = -1;
13426
+ }
13427
+ }
13428
+ if (hunkStart >= 0) {
13429
+ const slice = hunks.slice(hunkStart, Math.min(hunks.length, hunkStart + hunks.length));
13430
+ lines.push(`@@ -${hunkStart + 1} +${hunkStart + 1} @@`);
13431
+ for (const s of slice) {
13432
+ lines.push((s.op === "=" ? " " : s.op) + s.line);
13433
+ }
13434
+ }
13435
+ return lines.join("\n");
13436
+ }
13437
+ var createCustomAnthropicStreamingMessage = (message) => {
13438
+ const responseData = {
13439
+ type: "message",
13440
+ content: [
13441
+ {
13442
+ type: "text",
13443
+ text: message
13444
+ }
13445
+ ]
13446
+ };
13447
+ const jsonString = JSON.stringify(responseData);
13448
+ const arrayBuffer = new TextEncoder().encode(jsonString).buffer;
13449
+ return arrayBuffer;
13450
+ };
13451
+
13452
+ // src/mcp/index.ts
13453
+ var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
13454
+ var import_node_crypto6 = require("crypto");
13455
+ var import_streamableHttp = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
13456
+ var import_types2 = require("@modelcontextprotocol/sdk/types.js");
13457
+ var import_express6 = require("express");
13458
+ var import_api3 = require("@opentelemetry/api");
13459
+ var import_crypto_js8 = __toESM(require("crypto-js"), 1);
13460
+ var import_zod11 = require("zod");
12196
13461
  var SESSION_ID_HEADER = "mcp-session-id";
12197
13462
  var ExuluMCP = class {
12198
13463
  server = {};
@@ -12257,34 +13522,34 @@ var ExuluMCP = class {
12257
13522
  );
12258
13523
  }
12259
13524
  if (variable.encrypted) {
12260
- const bytes = import_crypto_js7.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
12261
- providerapikey = bytes.toString(import_crypto_js7.default.enc.Utf8);
13525
+ const bytes = import_crypto_js8.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
13526
+ providerapikey = bytes.toString(import_crypto_js8.default.enc.Utf8);
12262
13527
  }
12263
13528
  }
12264
13529
  console.log(
12265
13530
  "[EXULU] Enabled tools",
12266
13531
  enabledTools?.map((x) => x.name + " (" + x.id + ")")
12267
13532
  );
12268
- for (const tool6 of enabledTools || []) {
12269
- if (server.tools[tool6.id]) {
13533
+ for (const tool5 of enabledTools || []) {
13534
+ if (server.tools[tool5.id]) {
12270
13535
  continue;
12271
13536
  }
12272
13537
  server.mcp.registerTool(
12273
- sanitizeToolName(tool6.name + "_agent_" + tool6.id),
13538
+ sanitizeToolName(tool5.name + "_agent_" + tool5.id),
12274
13539
  {
12275
- title: tool6.name + " agent",
12276
- description: tool6.description,
13540
+ title: tool5.name + " agent",
13541
+ description: tool5.description,
12277
13542
  inputSchema: {
12278
- inputs: tool6.inputSchema || import_zod12.z.object({})
13543
+ inputs: tool5.inputSchema || import_zod11.z.object({})
12279
13544
  }
12280
13545
  },
12281
13546
  async ({ inputs }, args) => {
12282
- console.log("[EXULU] MCP tool name", tool6.name);
13547
+ console.log("[EXULU] MCP tool name", tool5.name);
12283
13548
  console.log("[EXULU] MCP tool inputs", inputs);
12284
13549
  console.log("[EXULU] MCP tool args", args);
12285
13550
  const configValues = agent.tools;
12286
13551
  const tools = await convertExuluToolsToAiSdkTools(
12287
- [tool6],
13552
+ [tool5],
12288
13553
  [],
12289
13554
  allTools,
12290
13555
  configValues,
@@ -12297,13 +13562,13 @@ var ExuluMCP = class {
12297
13562
  void 0,
12298
13563
  void 0
12299
13564
  );
12300
- const convertedTool = tools[sanitizeToolName(tool6.name)];
13565
+ const convertedTool = tools[sanitizeToolName(tool5.name)];
12301
13566
  if (!convertedTool?.execute) {
12302
13567
  console.error("[EXULU] Tool not found in converted tools array.", tools);
12303
13568
  throw new Error("Tool not found in converted tools array.");
12304
13569
  }
12305
13570
  const iterator = await convertedTool.execute(inputs, {
12306
- toolCallId: tool6.id + "_" + (0, import_node_crypto5.randomUUID)(),
13571
+ toolCallId: tool5.id + "_" + (0, import_node_crypto6.randomUUID)(),
12307
13572
  messages: []
12308
13573
  });
12309
13574
  let result;
@@ -12317,7 +13582,7 @@ var ExuluMCP = class {
12317
13582
  };
12318
13583
  }
12319
13584
  );
12320
- server.tools[tool6.id] = tool6.name;
13585
+ server.tools[tool5.id] = tool5.name;
12321
13586
  }
12322
13587
  const getListOfPromptTemplatesName = "getListOfPromptTemplates";
12323
13588
  if (!server.tools[getListOfPromptTemplatesName]) {
@@ -12327,7 +13592,7 @@ var ExuluMCP = class {
12327
13592
  title: "Get List of Prompt Templates",
12328
13593
  description: "Retrieves a list of prompt templates available for this agent. Returns the name, description, and ID of each template.",
12329
13594
  inputSchema: {
12330
- inputs: import_zod12.z.object({})
13595
+ inputs: import_zod11.z.object({})
12331
13596
  }
12332
13597
  },
12333
13598
  async ({ inputs }, args) => {
@@ -12373,8 +13638,8 @@ var ExuluMCP = class {
12373
13638
  title: "Get Prompt Template Details",
12374
13639
  description: "Retrieves the full details of a specific prompt template by ID, including the actual template content with variables.",
12375
13640
  inputSchema: {
12376
- inputs: import_zod12.z.object({
12377
- id: import_zod12.z.string().describe("The ID of the prompt template to retrieve")
13641
+ inputs: import_zod11.z.object({
13642
+ id: import_zod11.z.string().describe("The ID of the prompt template to retrieve")
12378
13643
  })
12379
13644
  }
12380
13645
  },
@@ -12437,20 +13702,20 @@ var ExuluMCP = class {
12437
13702
  return server.mcp;
12438
13703
  };
12439
13704
  create = async ({
12440
- express: express3,
13705
+ express: express4,
12441
13706
  allTools,
12442
13707
  allProviders,
12443
13708
  allContexts,
12444
13709
  allRerankers,
12445
13710
  config
12446
13711
  }) => {
12447
- if (!express3) {
13712
+ if (!express4) {
12448
13713
  throw new Error("Express not initialized.");
12449
13714
  }
12450
13715
  if (!this.server) {
12451
13716
  throw new Error("MCP server not initialized.");
12452
13717
  }
12453
- express3.post("/mcp/:agent", async (req, res) => {
13718
+ express4.post("/mcp/:agent", async (req, res) => {
12454
13719
  console.log("[EXULU] MCP request received.", req.params.agent);
12455
13720
  if (!req.params.agent) {
12456
13721
  res.status(400).json({
@@ -12497,7 +13762,7 @@ var ExuluMCP = class {
12497
13762
  transport = this.transports[sessionId];
12498
13763
  } else if (!sessionId && (0, import_types2.isInitializeRequest)(req.body)) {
12499
13764
  transport = new import_streamableHttp.StreamableHTTPServerTransport({
12500
- sessionIdGenerator: () => (0, import_node_crypto5.randomUUID)(),
13765
+ sessionIdGenerator: () => (0, import_node_crypto6.randomUUID)(),
12501
13766
  onsessioninitialized: (sessionId2) => {
12502
13767
  this.transports[sessionId2] = transport;
12503
13768
  }
@@ -12531,15 +13796,15 @@ var ExuluMCP = class {
12531
13796
  const transport = this.transports[sessionId];
12532
13797
  await transport.handleRequest(req, res);
12533
13798
  };
12534
- express3.get("/mcp/:agent", handleSessionRequest);
12535
- express3.delete("/mcp/:agent", handleSessionRequest);
13799
+ express4.get("/mcp/:agent", handleSessionRequest);
13800
+ express4.delete("/mcp/:agent", handleSessionRequest);
12536
13801
  console.log("[EXULU] MCP server created.");
12537
- return express3;
13802
+ return express4;
12538
13803
  };
12539
13804
  };
12540
13805
 
12541
13806
  // src/exulu/app/index.ts
12542
- var import_express7 = __toESM(require("express"), 1);
13807
+ var import_express8 = __toESM(require("express"), 1);
12543
13808
 
12544
13809
  // src/templates/providers/anthropic/claude.ts
12545
13810
  var import_anthropic = require("@ai-sdk/anthropic");
@@ -13245,7 +14510,7 @@ var import_winston2 = __toESM(require("winston"), 1);
13245
14510
  var import_util = __toESM(require("util"), 1);
13246
14511
 
13247
14512
  // src/exulu/evals.ts
13248
- var import_ai10 = require("ai");
14513
+ var import_ai11 = require("ai");
13249
14514
  var ExuluEval = class {
13250
14515
  id;
13251
14516
  name;
@@ -13282,7 +14547,7 @@ var ExuluEval = class {
13282
14547
  };
13283
14548
 
13284
14549
  // src/templates/evals/index.ts
13285
- var import_zod13 = require("zod");
14550
+ var import_zod12 = require("zod");
13286
14551
  var llmAsJudgeEval = () => {
13287
14552
  if (process.env.REDIS_HOST?.length && process.env.REDIS_PORT?.length) {
13288
14553
  return new ExuluEval({
@@ -13327,8 +14592,8 @@ var llmAsJudgeEval = () => {
13327
14592
  contexts: [],
13328
14593
  rerankers: [],
13329
14594
  prompt,
13330
- outputSchema: import_zod13.z.object({
13331
- score: import_zod13.z.number().min(0).max(100).describe("The score between 0 and 100.")
14595
+ outputSchema: import_zod12.z.object({
14596
+ score: import_zod12.z.number().min(0).max(100).describe("The score between 0 and 100.")
13332
14597
  }),
13333
14598
  providerapikey
13334
14599
  });
@@ -13556,12 +14821,12 @@ Usage:
13556
14821
  - If no todos exist yet, an empty list will be returned`;
13557
14822
 
13558
14823
  // src/templates/tools/todo/todo.ts
13559
- var import_zod14 = __toESM(require("zod"), 1);
13560
- var TodoSchema = import_zod14.default.object({
13561
- content: import_zod14.default.string().describe("Brief description of the task"),
13562
- status: import_zod14.default.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
13563
- priority: import_zod14.default.string().describe("Priority level of the task: high, medium, low"),
13564
- id: import_zod14.default.string().describe("Unique identifier for the todo item")
14824
+ var import_zod13 = __toESM(require("zod"), 1);
14825
+ var TodoSchema = import_zod13.default.object({
14826
+ content: import_zod13.default.string().describe("Brief description of the task"),
14827
+ status: import_zod13.default.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
14828
+ priority: import_zod13.default.string().describe("Priority level of the task: high, medium, low"),
14829
+ id: import_zod13.default.string().describe("Unique identifier for the todo item")
13565
14830
  });
13566
14831
  var TodoWriteTool = new ExuluTool({
13567
14832
  id: "todo_write",
@@ -13577,8 +14842,8 @@ var TodoWriteTool = new ExuluTool({
13577
14842
  default: todowrite_default
13578
14843
  }
13579
14844
  ],
13580
- inputSchema: import_zod14.default.object({
13581
- todos: import_zod14.default.array(TodoSchema).describe("The updated todo list")
14845
+ inputSchema: import_zod13.default.object({
14846
+ todos: import_zod13.default.array(TodoSchema).describe("The updated todo list")
13582
14847
  }),
13583
14848
  execute: async (inputs) => {
13584
14849
  const { sessionID, todos, user } = inputs;
@@ -13613,7 +14878,7 @@ var TodoReadTool = new ExuluTool({
13613
14878
  id: "todo_read",
13614
14879
  name: "Todo Read",
13615
14880
  description: "Use this tool to read your todo list",
13616
- inputSchema: import_zod14.default.object({}),
14881
+ inputSchema: import_zod13.default.object({}),
13617
14882
  type: "function",
13618
14883
  category: "todo",
13619
14884
  config: [
@@ -13739,18 +15004,18 @@ After asking a question, use the Question Read tool to check if the user has ans
13739
15004
  var questionread_default = 'Use this tool to read questions you\'ve asked and check if they\'ve been answered by the user. This tool helps you track the status of questions and retrieve the user\'s selected answers.\n\n## When to Use This Tool\n\nUse this tool proactively in these situations:\n- After asking a question to check if the user has responded\n- To retrieve the user\'s answer before proceeding with implementation\n- To review all questions and answers in the current session\n- When you need to reference a previous answer\n\n## How It Works\n\n- This tool takes no parameters (leave the input blank or empty)\n- Returns an array of all questions in the session\n- Each question includes:\n - `id`: Unique identifier for the question\n - `question`: The question text\n - `answerOptions`: Array of answer options with their IDs and text\n - `status`: Either "pending" (not answered) or "answered"\n - `selectedAnswerId`: The ID of the chosen answer (only present if answered)\n\n## Usage Pattern\n\nTypically you\'ll:\n1. Use Question Ask to pose a question\n2. Wait for the user to respond\n3. Use Question Read to check the answer\n4. Find the selected answer by matching the `selectedAnswerId` with an option in `answerOptions`\n5. Proceed with implementation based on the user\'s choice\n\n## Example Response\n\n```json\n[\n {\n "id": "question123",\n "question": "Which authentication method would you like to implement?",\n "answerOptions": [\n { "id": "ans1", "text": "JWT tokens" },\n { "id": "ans2", "text": "OAuth 2.0" },\n { "id": "ans3", "text": "Session-based auth" },\n { "id": "ans4", "text": "None of the above..." }\n ],\n "status": "answered",\n "selectedAnswerId": "ans1"\n }\n]\n```\n\nIn this example, the user selected "JWT tokens" (id: ans1).\n\n## Important Notes\n\n- If no questions exist in the session, an empty array will be returned\n- Questions remain in the session even after being answered for reference\n- Use the `selectedAnswerId` to find which answer option the user chose by matching it against the `id` field in `answerOptions`\n';
13740
15005
 
13741
15006
  // src/templates/tools/question/question.ts
13742
- var import_zod15 = __toESM(require("zod"), 1);
13743
- var import_node_crypto6 = require("crypto");
13744
- var AnswerOptionSchema = import_zod15.default.object({
13745
- id: import_zod15.default.string().describe("Unique identifier for the answer option"),
13746
- text: import_zod15.default.string().describe("The text of the answer option")
15007
+ var import_zod14 = __toESM(require("zod"), 1);
15008
+ var import_node_crypto7 = require("crypto");
15009
+ var AnswerOptionSchema = import_zod14.default.object({
15010
+ id: import_zod14.default.string().describe("Unique identifier for the answer option"),
15011
+ text: import_zod14.default.string().describe("The text of the answer option")
13747
15012
  });
13748
- var _QuestionSchema = import_zod15.default.object({
13749
- id: import_zod15.default.string().describe("Unique identifier for the question"),
13750
- question: import_zod15.default.string().describe("The question to ask the user"),
13751
- answerOptions: import_zod15.default.array(AnswerOptionSchema).describe("Array of possible answer options"),
13752
- selectedAnswerId: import_zod15.default.string().optional().describe("The ID of the answer option selected by the user"),
13753
- status: import_zod15.default.enum(["pending", "answered"]).describe("Status of the question: pending or answered")
15013
+ var _QuestionSchema = import_zod14.default.object({
15014
+ id: import_zod14.default.string().describe("Unique identifier for the question"),
15015
+ question: import_zod14.default.string().describe("The question to ask the user"),
15016
+ answerOptions: import_zod14.default.array(AnswerOptionSchema).describe("Array of possible answer options"),
15017
+ selectedAnswerId: import_zod14.default.string().optional().describe("The ID of the answer option selected by the user"),
15018
+ status: import_zod14.default.enum(["pending", "answered"]).describe("Status of the question: pending or answered")
13754
15019
  });
13755
15020
  var QuestionAskTool = new ExuluTool({
13756
15021
  id: "question_ask",
@@ -13758,6 +15023,7 @@ var QuestionAskTool = new ExuluTool({
13758
15023
  description: "Use this tool to ask a question to the user with multiple choice answers",
13759
15024
  type: "function",
13760
15025
  category: "question",
15026
+ needsApproval: false,
13761
15027
  config: [
13762
15028
  {
13763
15029
  name: "description",
@@ -13766,9 +15032,9 @@ var QuestionAskTool = new ExuluTool({
13766
15032
  default: questionask_default
13767
15033
  }
13768
15034
  ],
13769
- inputSchema: import_zod15.default.object({
13770
- question: import_zod15.default.string().describe("The question to ask the user"),
13771
- answerOptions: import_zod15.default.array(import_zod15.default.string()).describe("Array of possible answer options (strings)")
15035
+ inputSchema: import_zod14.default.object({
15036
+ question: import_zod14.default.string().describe("The question to ask the user"),
15037
+ answerOptions: import_zod14.default.array(import_zod14.default.string()).describe("Array of possible answer options (strings)")
13772
15038
  }),
13773
15039
  execute: async (inputs) => {
13774
15040
  const { sessionID, question, answerOptions, user } = inputs;
@@ -13793,15 +15059,15 @@ var QuestionAskTool = new ExuluTool({
13793
15059
  throw new Error("You don't have access to this session " + session.id + ".");
13794
15060
  }
13795
15061
  const answerOptionsWithIds = answerOptions.map((text) => ({
13796
- id: (0, import_node_crypto6.randomUUID)(),
15062
+ id: (0, import_node_crypto7.randomUUID)(),
13797
15063
  text
13798
15064
  }));
13799
15065
  answerOptionsWithIds.push({
13800
- id: (0, import_node_crypto6.randomUUID)(),
15066
+ id: (0, import_node_crypto7.randomUUID)(),
13801
15067
  text: "None of the above..."
13802
15068
  });
13803
15069
  const newQuestion = {
13804
- id: (0, import_node_crypto6.randomUUID)(),
15070
+ id: (0, import_node_crypto7.randomUUID)(),
13805
15071
  question,
13806
15072
  answerOptions: answerOptionsWithIds,
13807
15073
  status: "pending"
@@ -13827,8 +15093,9 @@ var QuestionAskTool = new ExuluTool({
13827
15093
  var QuestionReadTool = new ExuluTool({
13828
15094
  id: "question_read",
13829
15095
  name: "Question Read",
15096
+ needsApproval: false,
13830
15097
  description: "Use this tool to read questions and their answers",
13831
- inputSchema: import_zod15.default.object({}),
15098
+ inputSchema: import_zod14.default.object({}),
13832
15099
  type: "function",
13833
15100
  category: "question",
13834
15101
  config: [
@@ -13868,15 +15135,15 @@ async function getQuestions(sessionID) {
13868
15135
  var questionTools = [QuestionAskTool, QuestionReadTool];
13869
15136
 
13870
15137
  // src/templates/tools/perplexity.ts
13871
- var import_zod16 = __toESM(require("zod"), 1);
15138
+ var import_zod15 = __toESM(require("zod"), 1);
13872
15139
  var import_perplexity_ai = __toESM(require("@perplexity-ai/perplexity_ai"), 1);
13873
15140
  var internetSearchTool = new ExuluTool({
13874
15141
  id: "internet_search",
13875
- name: "Perplexity Live Internet Search",
15142
+ name: "Internet Search",
13876
15143
  description: "Search the internet for information.",
13877
- inputSchema: import_zod16.default.object({
13878
- query: import_zod16.default.string().describe("The query to the tool."),
13879
- search_recency_filter: import_zod16.default.enum(["day", "week", "month", "year"]).optional().describe("The recency filter for the search, can be day, week, month or year.")
15144
+ inputSchema: import_zod15.default.object({
15145
+ query: import_zod15.default.string().describe("The query to the tool."),
15146
+ search_recency_filter: import_zod15.default.enum(["day", "week", "month", "year"]).optional().describe("The recency filter for the search, can be day, week, month or year.")
13880
15147
  }),
13881
15148
  category: "internet_search",
13882
15149
  type: "web_search",
@@ -13954,7 +15221,7 @@ var internetSearchTool = new ExuluTool({
13954
15221
  } catch (error) {
13955
15222
  if (error instanceof import_perplexity_ai.default.RateLimitError && attempt < maxRetries - 1) {
13956
15223
  const delay = Math.pow(2, attempt) * 1e3 + Math.random() * 1e3;
13957
- await new Promise((resolve4) => setTimeout(resolve4, delay));
15224
+ await new Promise((resolve3) => setTimeout(resolve3, delay));
13958
15225
  continue;
13959
15226
  }
13960
15227
  throw error;
@@ -13967,6 +15234,101 @@ var internetSearchTool = new ExuluTool({
13967
15234
  });
13968
15235
  var perplexityTools = [internetSearchTool];
13969
15236
 
15237
+ // src/templates/tools/email.ts
15238
+ var nodemailer = __toESM(require("nodemailer"), 1);
15239
+ var import_zod16 = require("zod");
15240
+ var transporter = null;
15241
+ function getTransporter(config) {
15242
+ if (!transporter) {
15243
+ transporter = nodemailer.createTransport(config);
15244
+ }
15245
+ return transporter;
15246
+ }
15247
+ async function sendEmail(recipient, subject, html, text, config) {
15248
+ const transport = getTransporter(config);
15249
+ html = html.trim();
15250
+ text = text.trim();
15251
+ await transport.sendMail({
15252
+ from: config.from,
15253
+ to: recipient,
15254
+ subject,
15255
+ text,
15256
+ html
15257
+ });
15258
+ }
15259
+ var emailTool = new ExuluTool({
15260
+ id: "email",
15261
+ name: "Email",
15262
+ description: "Send an email.",
15263
+ inputSchema: import_zod16.z.object({
15264
+ recipient: import_zod16.z.string().describe("The recipient of the email."),
15265
+ subject: import_zod16.z.string().describe("The subject of the email."),
15266
+ html: import_zod16.z.string().describe("The HTML body of the email."),
15267
+ text: import_zod16.z.string().describe("The text body of the email.")
15268
+ }),
15269
+ type: "function",
15270
+ config: [{
15271
+ name: "smtp_host",
15272
+ description: "The SMTP host to send the email from.",
15273
+ type: "variable",
15274
+ default: void 0
15275
+ }, {
15276
+ name: "smtp_port",
15277
+ description: "The SMTP port to send the email from.",
15278
+ type: "variable",
15279
+ default: void 0
15280
+ }, {
15281
+ name: "smtp_user",
15282
+ description: "The SMTP user to send the email from.",
15283
+ type: "variable",
15284
+ default: void 0
15285
+ }, {
15286
+ name: "smtp_password",
15287
+ description: "The SMTP password to send the email from.",
15288
+ type: "variable",
15289
+ default: void 0
15290
+ }, {
15291
+ name: "smtp_from",
15292
+ description: "The SMTP from address to send the email from.",
15293
+ type: "variable",
15294
+ default: void 0
15295
+ }, {
15296
+ name: "allowed_recipient_domains",
15297
+ description: "A comma seperated list of allowed recipient domains to send emails to.",
15298
+ type: "string",
15299
+ default: void 0
15300
+ }],
15301
+ execute: async ({ recipient, subject, html, text, toolVariablesConfig }) => {
15302
+ const EMAIL_CONFIG = {
15303
+ host: toolVariablesConfig.smtp_host || process.env.SMTP_HOST || "",
15304
+ port: parseInt(toolVariablesConfig.smtp_port || process.env.SMTP_PORT || "587", 10),
15305
+ secure: toolVariablesConfig.smtp_secure === "true" || process.env.SMTP_SECURE === "true",
15306
+ // true for 465, false for other ports
15307
+ auth: {
15308
+ user: toolVariablesConfig.smtp_user || process.env.SMTP_USER || "",
15309
+ pass: toolVariablesConfig.smtp_password || process.env.SMTP_PASSWORD || ""
15310
+ },
15311
+ from: toolVariablesConfig.smtp_from || process.env.SMTP_FROM || "",
15312
+ // Allow self-signed certificates if SMTP_REJECT_UNAUTHORIZED is set to 'false'
15313
+ tls: {
15314
+ rejectUnauthorized: false
15315
+ }
15316
+ };
15317
+ if (toolVariablesConfig.allowed_recipient_domains) {
15318
+ const allowedRecipientDomains = toolVariablesConfig.allowed_recipient_domains.split(",");
15319
+ if (!allowedRecipientDomains.some((domain) => recipient.endsWith(`@${domain}`))) {
15320
+ return {
15321
+ result: "Recipient domain not allowed to send emails to."
15322
+ };
15323
+ }
15324
+ }
15325
+ await sendEmail(recipient, subject, html, text, EMAIL_CONFIG);
15326
+ return {
15327
+ result: "Email sent successfully to " + recipient + " with subject " + subject + "."
15328
+ };
15329
+ }
15330
+ });
15331
+
13970
15332
  // src/validators/postgres-name.ts
13971
15333
  var isValidPostgresName = (id) => {
13972
15334
  const regex = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
@@ -14071,8 +15433,7 @@ var ExuluApp2 = class {
14071
15433
  ...todoTools,
14072
15434
  ...questionTools,
14073
15435
  ...perplexityTools,
14074
- // Add contexts as tools
14075
- ...Object.values(contexts || {}).map((context) => context.tool()).filter(Boolean)
15436
+ emailTool
14076
15437
  // Because agents are stored in the database, we add those as tools
14077
15438
  // at request time, not during ExuluApp initialization. We add them
14078
15439
  // in the grahql tools resolver.
@@ -14089,9 +15450,9 @@ var ExuluApp2 = class {
14089
15450
  id: provider.id ?? "",
14090
15451
  type: "agent"
14091
15452
  })),
14092
- ...this._tools.map((tool6) => ({
14093
- name: tool6.name ?? "",
14094
- id: tool6.id ?? "",
15453
+ ...this._tools.map((tool5) => ({
15454
+ name: tool5.name ?? "",
15455
+ id: tool5.id ?? "",
14095
15456
  type: "tool"
14096
15457
  })),
14097
15458
  ...this._rerankers.map((reranker) => ({
@@ -14137,7 +15498,7 @@ var ExuluApp2 = class {
14137
15498
  express = {
14138
15499
  init: async () => {
14139
15500
  if (!this._expressApp) {
14140
- this._expressApp = (0, import_express7.default)();
15501
+ this._expressApp = (0, import_express8.default)();
14141
15502
  await this.server.express.init();
14142
15503
  console.log("[EXULU] Express app initialized.");
14143
15504
  }
@@ -15353,7 +16714,7 @@ var RecursiveChunker = class _RecursiveChunker extends BaseChunker {
15353
16714
  };
15354
16715
 
15355
16716
  // src/exulu/embedder.ts
15356
- var import_crypto_js8 = __toESM(require("crypto-js"), 1);
16717
+ var import_crypto_js9 = __toESM(require("crypto-js"), 1);
15357
16718
  var ExuluEmbedder = class {
15358
16719
  id;
15359
16720
  name;
@@ -15424,8 +16785,8 @@ var ExuluEmbedder = class {
15424
16785
  );
15425
16786
  }
15426
16787
  try {
15427
- const bytes = import_crypto_js8.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
15428
- const decrypted = bytes.toString(import_crypto_js8.default.enc.Utf8);
16788
+ const bytes = import_crypto_js9.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
16789
+ const decrypted = bytes.toString(import_crypto_js9.default.enc.Utf8);
15429
16790
  if (!decrypted) {
15430
16791
  throw new Error("Decryption returned empty string - invalid key or corrupted data");
15431
16792
  }
@@ -15913,6 +17274,7 @@ var {
15913
17274
  agentMessagesSchema: agentMessagesSchema3,
15914
17275
  rolesSchema: rolesSchema3,
15915
17276
  usersSchema: usersSchema3,
17277
+ skillsSchema: skillsSchema3,
15916
17278
  statisticsSchema: statisticsSchema3,
15917
17279
  variablesSchema: variablesSchema3,
15918
17280
  workflowTemplatesSchema: workflowTemplatesSchema3,
@@ -15963,6 +17325,7 @@ var up = async function(knex) {
15963
17325
  agentsSchema3(),
15964
17326
  feedbackSchema3(),
15965
17327
  variablesSchema3(),
17328
+ skillsSchema3(),
15966
17329
  workflowTemplatesSchema3()
15967
17330
  ];
15968
17331
  const createTable = async (schema) => {
@@ -16165,7 +17528,7 @@ var create = ({
16165
17528
  };
16166
17529
 
16167
17530
  // src/index.ts
16168
- var import_crypto_js9 = __toESM(require("crypto-js"), 1);
17531
+ var import_crypto_js10 = __toESM(require("crypto-js"), 1);
16169
17532
 
16170
17533
  // ee/chunking/markdown.ts
16171
17534
  var extractPageTag = (text) => {
@@ -16649,7 +18012,7 @@ var MarkdownChunker = class {
16649
18012
  var import_child_process = require("child_process");
16650
18013
  var import_util2 = require("util");
16651
18014
  var import_path = require("path");
16652
- var import_fs4 = require("fs");
18015
+ var import_fs3 = require("fs");
16653
18016
  var import_url = require("url");
16654
18017
  var execAsync = (0, import_util2.promisify)(import_child_process.exec);
16655
18018
  function getPackageRoot() {
@@ -16659,9 +18022,9 @@ function getPackageRoot() {
16659
18022
  const maxAttempts = 10;
16660
18023
  while (attempts < maxAttempts) {
16661
18024
  const packageJsonPath = (0, import_path.join)(currentDir, "package.json");
16662
- if ((0, import_fs4.existsSync)(packageJsonPath)) {
18025
+ if ((0, import_fs3.existsSync)(packageJsonPath)) {
16663
18026
  try {
16664
- const packageJson = JSON.parse((0, import_fs4.readFileSync)(packageJsonPath, "utf-8"));
18027
+ const packageJson = JSON.parse((0, import_fs3.readFileSync)(packageJsonPath, "utf-8"));
16665
18028
  if (packageJson.name === "@exulu/backend") {
16666
18029
  return currentDir;
16667
18030
  }
@@ -16688,7 +18051,7 @@ function isPythonEnvironmentSetup(packageRoot) {
16688
18051
  const root = packageRoot ?? getPackageRoot();
16689
18052
  const venvPath = getVenvPath(root);
16690
18053
  const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
16691
- return (0, import_fs4.existsSync)(venvPath) && (0, import_fs4.existsSync)(pythonPath);
18054
+ return (0, import_fs3.existsSync)(venvPath) && (0, import_fs3.existsSync)(pythonPath);
16692
18055
  }
16693
18056
  async function setupPythonEnvironment(options = {}) {
16694
18057
  const {
@@ -16709,7 +18072,7 @@ async function setupPythonEnvironment(options = {}) {
16709
18072
  };
16710
18073
  }
16711
18074
  const setupScriptPath = getSetupScriptPath(packageRoot);
16712
- if (!(0, import_fs4.existsSync)(setupScriptPath)) {
18075
+ if (!(0, import_fs3.existsSync)(setupScriptPath)) {
16713
18076
  return {
16714
18077
  success: false,
16715
18078
  message: `Setup script not found at: ${setupScriptPath}`,
@@ -16791,13 +18154,13 @@ async function validatePythonEnvironment(packageRoot, checkPackages = true) {
16791
18154
  const root = packageRoot ?? getPackageRoot();
16792
18155
  const venvPath = getVenvPath(root);
16793
18156
  const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
16794
- if (!(0, import_fs4.existsSync)(venvPath)) {
18157
+ if (!(0, import_fs3.existsSync)(venvPath)) {
16795
18158
  return {
16796
18159
  valid: false,
16797
18160
  message: getPythonSetupInstructions()
16798
18161
  };
16799
18162
  }
16800
- if (!(0, import_fs4.existsSync)(pythonPath)) {
18163
+ if (!(0, import_fs3.existsSync)(pythonPath)) {
16801
18164
  return {
16802
18165
  valid: false,
16803
18166
  message: "Python virtual environment is corrupted. Please run:\n await setupPythonEnvironment({ force: true })"
@@ -16849,9 +18212,9 @@ Or manually run the setup script:
16849
18212
  }
16850
18213
 
16851
18214
  // ee/python/documents/processing/doc_processor.ts
16852
- var fs5 = __toESM(require("fs"), 1);
18215
+ var fs4 = __toESM(require("fs"), 1);
16853
18216
  var path2 = __toESM(require("path"), 1);
16854
- var import_ai11 = require("ai");
18217
+ var import_ai12 = require("ai");
16855
18218
  var import_zod17 = require("zod");
16856
18219
  var import_p_limit = __toESM(require("p-limit"), 1);
16857
18220
  var import_crypto = require("crypto");
@@ -16864,7 +18227,7 @@ var import_officeparser2 = require("officeparser");
16864
18227
  var import_child_process2 = require("child_process");
16865
18228
  var import_util3 = require("util");
16866
18229
  var import_path2 = require("path");
16867
- var import_fs5 = require("fs");
18230
+ var import_fs4 = require("fs");
16868
18231
  var import_url2 = require("url");
16869
18232
  var execAsync2 = (0, import_util3.promisify)(import_child_process2.exec);
16870
18233
  function getPackageRoot2() {
@@ -16874,9 +18237,9 @@ function getPackageRoot2() {
16874
18237
  const maxAttempts = 10;
16875
18238
  while (attempts < maxAttempts) {
16876
18239
  const packageJsonPath = (0, import_path2.join)(currentDir, "package.json");
16877
- if ((0, import_fs5.existsSync)(packageJsonPath)) {
18240
+ if ((0, import_fs4.existsSync)(packageJsonPath)) {
16878
18241
  try {
16879
- const packageJson = JSON.parse((0, import_fs5.readFileSync)(packageJsonPath, "utf-8"));
18242
+ const packageJson = JSON.parse((0, import_fs4.readFileSync)(packageJsonPath, "utf-8"));
16880
18243
  if (packageJson.name === "@exulu/backend") {
16881
18244
  return currentDir;
16882
18245
  }
@@ -16938,7 +18301,7 @@ async function executePythonScript(config) {
16938
18301
  await validatePythonEnvironmentForExecution(packageRoot);
16939
18302
  }
16940
18303
  const resolvedScriptPath = (0, import_path2.resolve)(packageRoot, scriptPath);
16941
- if (!(0, import_fs5.existsSync)(resolvedScriptPath)) {
18304
+ if (!(0, import_fs4.existsSync)(resolvedScriptPath)) {
16942
18305
  throw new PythonExecutionError(
16943
18306
  `Python script not found: ${resolvedScriptPath}`,
16944
18307
  "",
@@ -17031,6 +18394,55 @@ async function processWord(file) {
17031
18394
  markdown: content
17032
18395
  };
17033
18396
  }
18397
+ async function processImage(buffer, paths, config, verbose = false) {
18398
+ try {
18399
+ await fs4.promises.mkdir(paths.images, { recursive: true });
18400
+ const imagePath = path2.join(paths.images, "1.png");
18401
+ await fs4.promises.writeFile(imagePath, buffer);
18402
+ console.log(`[EXULU] Image saved to: ${imagePath}`);
18403
+ let json = [{
18404
+ page: 1,
18405
+ content: "",
18406
+ // Empty initially, will be populated by VLM if enabled
18407
+ image: imagePath,
18408
+ headings: []
18409
+ }];
18410
+ if (config?.vlm?.model) {
18411
+ console.log("[EXULU] Extracting content from image using VLM...");
18412
+ json = await validateWithVLM(
18413
+ json,
18414
+ config.vlm.model,
18415
+ verbose,
18416
+ config.vlm.concurrency
18417
+ );
18418
+ await fs4.promises.writeFile(
18419
+ paths.json,
18420
+ JSON.stringify(json, null, 2),
18421
+ "utf-8"
18422
+ );
18423
+ console.log("[EXULU] VLM content extraction complete");
18424
+ const correctedCount = json.filter((p) => p.vlm_corrected_text).length;
18425
+ console.log(`[EXULU] Content extracted: ${correctedCount > 0 ? "Yes" : "No"}`);
18426
+ } else {
18427
+ console.log("[EXULU] No VLM configured, image saved without content extraction");
18428
+ console.log("[EXULU] Note: Enable VLM in config to extract text/content from images");
18429
+ await fs4.promises.writeFile(
18430
+ paths.json,
18431
+ JSON.stringify(json, null, 2),
18432
+ "utf-8"
18433
+ );
18434
+ }
18435
+ const markdown = json.map((p) => p.vlm_corrected_text ?? p.content).join("\n\n\n<!-- END_OF_PAGE -->\n\n\n");
18436
+ await fs4.promises.writeFile(paths.markdown, markdown, "utf-8");
18437
+ return {
18438
+ markdown,
18439
+ json
18440
+ };
18441
+ } catch (error) {
18442
+ console.error("[EXULU] Error processing image:", error);
18443
+ throw error;
18444
+ }
18445
+ }
17034
18446
  function normalizeMarkdownContent(content) {
17035
18447
  const lines = content.split("\n");
17036
18448
  const normalizedLines = [];
@@ -17074,7 +18486,7 @@ function reconstructHeadings(correctedText, headingsHierarchy) {
17074
18486
  return result;
17075
18487
  }
17076
18488
  async function validatePageWithVLM(page, imagePath, model) {
17077
- const imageBuffer = await fs5.promises.readFile(imagePath);
18489
+ const imageBuffer = await fs4.promises.readFile(imagePath);
17078
18490
  const imageBase64 = imageBuffer.toString("base64");
17079
18491
  const mimeType = "image/png";
17080
18492
  const prompt = `You are a document validation assistant. Your task is to analyze a page image and correct the output of an OCR/parsing pipeline. The content may include tables, technical diagrams, schematics, and structured text.
@@ -17152,9 +18564,9 @@ If the page contains a flow-chart, schematic, technical drawing or control board
17152
18564
 
17153
18565
  ### 7. Only populate \`corrected_text\` when \`needs_correction\` is true. If the OCR output is accurate, return \`needs_correction: false\` and \`corrected_content: null\`.
17154
18566
  `;
17155
- const result = await (0, import_ai11.generateText)({
18567
+ const result = await (0, import_ai12.generateText)({
17156
18568
  model,
17157
- output: import_ai11.Output.object({
18569
+ output: import_ai12.Output.object({
17158
18570
  schema: import_zod17.z.object({
17159
18571
  needs_correction: import_zod17.z.boolean(),
17160
18572
  corrected_text: import_zod17.z.string().nullable(),
@@ -17187,6 +18599,7 @@ If the page contains a flow-chart, schematic, technical drawing or control board
17187
18599
  current_page_table: parsedOutput.current_page_table || void 0,
17188
18600
  reasoning: parsedOutput.reasoning
17189
18601
  };
18602
+ console.log(`[EXULU] VLM validation result: ${JSON.stringify(validation)}`);
17190
18603
  return validation;
17191
18604
  }
17192
18605
  function reconstructTableHeaders(document2, validationResults, verbose = false) {
@@ -17242,26 +18655,29 @@ async function validateWithVLM(document2, model, verbose = false, concurrency =
17242
18655
  let correctedCount = 0;
17243
18656
  const validationTasks = document2.map(
17244
18657
  (page) => limit(async () => {
17245
- await new Promise((resolve4) => setImmediate(resolve4));
18658
+ await new Promise((resolve3) => setImmediate(resolve3));
17246
18659
  const imagePath = page.image;
17247
- if (!page.content) {
17248
- console.warn(`[EXULU] Page ${page.page}: No content found, skipping validation`);
17249
- return;
17250
- }
17251
18660
  if (!imagePath) {
17252
18661
  console.warn(`[EXULU] Page ${page.page}: No image found, skipping validation`);
17253
18662
  return;
17254
18663
  }
17255
- const hasImage = page.content.match(/\.(jpeg|jpg|png|gif|webp)/i);
17256
- const hasTable = (page.content.match(/\|/g)?.length || 0) > 1;
17257
- if (!hasImage && !hasTable) {
18664
+ if (page.content) {
18665
+ const hasImage = page.content.match(/\.(jpeg|jpg|png|gif|webp)/i);
18666
+ const hasTable = (page.content.match(/\|/g)?.length || 0) > 1;
18667
+ if (!hasImage && !hasTable) {
18668
+ if (verbose) {
18669
+ console.log(`[EXULU] Page ${page.page}: No image or table found, SKIPPING VLM validation`);
18670
+ }
18671
+ return;
18672
+ }
18673
+ } else {
17258
18674
  if (verbose) {
17259
- console.log(`[EXULU] Page ${page.page}: No image or table found, SKIPPING VLM validation`);
18675
+ console.log(`[EXULU] Page ${page.page}: Standalone image, proceeding with VLM content extraction`);
17260
18676
  }
17261
- return;
17262
18677
  }
17263
18678
  let validation;
17264
18679
  try {
18680
+ console.log(`[EXULU] Validating page ${page.page} with VLM`);
17265
18681
  validation = await withRetry(async () => {
17266
18682
  return await validatePageWithVLM(page, imagePath, model);
17267
18683
  }, 3);
@@ -17344,6 +18760,13 @@ async function processDocument(filePath, fileType, buffer, tempDir, config, verb
17344
18760
  case "doc":
17345
18761
  result = await processWord(buffer);
17346
18762
  break;
18763
+ case "jpg":
18764
+ case "jpeg":
18765
+ case "png":
18766
+ case "gif":
18767
+ case "webp":
18768
+ result = await processImage(buffer, paths, config, verbose);
18769
+ break;
17347
18770
  // Todo other file types with docx and officeparser
17348
18771
  default:
17349
18772
  throw new Error(`[EXULU] Unsupported file type: ${fileType}`);
@@ -17406,7 +18829,7 @@ ${setupResult.output || ""}`);
17406
18829
  if (!result.success) {
17407
18830
  throw new Error(`Document processing failed: ${result.stderr}`);
17408
18831
  }
17409
- const jsonContent = await fs5.promises.readFile(paths.json, "utf-8");
18832
+ const jsonContent = await fs4.promises.readFile(paths.json, "utf-8");
17410
18833
  json = JSON.parse(jsonContent);
17411
18834
  } else if (config?.processor.name === "officeparser") {
17412
18835
  const text = await (0, import_officeparser2.parseOfficeAsync)(buffer, {
@@ -17423,7 +18846,7 @@ ${setupResult.output || ""}`);
17423
18846
  if (!MISTRAL_API_KEY) {
17424
18847
  throw new Error('[EXULU] MISTRAL_API_KEY is not set, please set it in the environment variable via process.env or via an Exulu variable named "MISTRAL_API_KEY".');
17425
18848
  }
17426
- await new Promise((resolve4) => setTimeout(resolve4, Math.floor(Math.random() * 4e3) + 1e3));
18849
+ await new Promise((resolve3) => setTimeout(resolve3, Math.floor(Math.random() * 4e3) + 1e3));
17427
18850
  const base64Pdf = buffer.toString("base64");
17428
18851
  const client2 = new import_mistralai.Mistral({ apiKey: MISTRAL_API_KEY });
17429
18852
  const ocrResponse = await withRetry(async () => {
@@ -17439,9 +18862,9 @@ ${setupResult.output || ""}`);
17439
18862
  }, 10);
17440
18863
  const parser = new import_liteparse.LiteParse();
17441
18864
  const screenshots = await parser.screenshot(paths.source, void 0);
17442
- await fs5.promises.mkdir(paths.images, { recursive: true });
18865
+ await fs4.promises.mkdir(paths.images, { recursive: true });
17443
18866
  for (const screenshot of screenshots) {
17444
- await fs5.promises.writeFile(
18867
+ await fs4.promises.writeFile(
17445
18868
  path2.join(
17446
18869
  paths.images,
17447
18870
  `${screenshot.pageNum}.png`
@@ -17456,15 +18879,15 @@ ${setupResult.output || ""}`);
17456
18879
  image: screenshots.find((s) => s.pageNum === page.index + 1)?.imagePath,
17457
18880
  headings: []
17458
18881
  }));
17459
- fs5.writeFileSync(paths.json, JSON.stringify(json, null, 2));
18882
+ fs4.writeFileSync(paths.json, JSON.stringify(json, null, 2));
17460
18883
  } else if (config?.processor.name === "liteparse") {
17461
18884
  const parser = new import_liteparse.LiteParse();
17462
18885
  const result = await parser.parse(paths.source);
17463
18886
  const screenshots = await parser.screenshot(paths.source, void 0);
17464
18887
  console.log(`[EXULU] Liteparse screenshots: ${JSON.stringify(screenshots)}`);
17465
- await fs5.promises.mkdir(paths.images, { recursive: true });
18888
+ await fs4.promises.mkdir(paths.images, { recursive: true });
17466
18889
  for (const screenshot of screenshots) {
17467
- await fs5.promises.writeFile(path2.join(paths.images, `${screenshot.pageNum}.png`), screenshot.imageBuffer);
18890
+ await fs4.promises.writeFile(path2.join(paths.images, `${screenshot.pageNum}.png`), screenshot.imageBuffer);
17468
18891
  screenshot.imagePath = path2.join(paths.images, `${screenshot.pageNum}.png`);
17469
18892
  }
17470
18893
  json = result.pages.map((page) => ({
@@ -17472,7 +18895,7 @@ ${setupResult.output || ""}`);
17472
18895
  content: page.text,
17473
18896
  image: screenshots.find((s) => s.pageNum === page.pageNum)?.imagePath
17474
18897
  }));
17475
- fs5.writeFileSync(paths.json, JSON.stringify(json, null, 2));
18898
+ fs4.writeFileSync(paths.json, JSON.stringify(json, null, 2));
17476
18899
  }
17477
18900
  console.log(`[EXULU]
17478
18901
  \u2713 Document processing completed successfully`);
@@ -17503,13 +18926,13 @@ ${setupResult.output || ""}`);
17503
18926
  console.log(`[EXULU] Corrected: ${page.vlm_corrected_text.substring(0, 150)}...`);
17504
18927
  });
17505
18928
  }
17506
- await fs5.promises.writeFile(
18929
+ await fs4.promises.writeFile(
17507
18930
  paths.json,
17508
18931
  JSON.stringify(json, null, 2),
17509
18932
  "utf-8"
17510
18933
  );
17511
18934
  }
17512
- const markdownStream = fs5.createWriteStream(paths.markdown, { encoding: "utf-8" });
18935
+ const markdownStream = fs4.createWriteStream(paths.markdown, { encoding: "utf-8" });
17513
18936
  for (let i = 0; i < json.length; i++) {
17514
18937
  const p = json[i];
17515
18938
  if (!p) continue;
@@ -17519,13 +18942,13 @@ ${setupResult.output || ""}`);
17519
18942
  markdownStream.write("\n\n\n<!-- END_OF_PAGE -->\n\n\n");
17520
18943
  }
17521
18944
  }
17522
- await new Promise((resolve4, reject) => {
17523
- markdownStream.end(() => resolve4());
18945
+ await new Promise((resolve3, reject) => {
18946
+ markdownStream.end(() => resolve3());
17524
18947
  markdownStream.on("error", reject);
17525
18948
  });
17526
18949
  console.log(`[EXULU] Validated output saved to: ${paths.json}`);
17527
18950
  console.log(`[EXULU] Validated markdown saved to: ${paths.markdown}`);
17528
- const markdown = await fs5.promises.readFile(paths.markdown, "utf-8");
18951
+ const markdown = await fs4.promises.readFile(paths.markdown, "utf-8");
17529
18952
  const processedJson = json.map((e) => {
17530
18953
  const finalContent = e.vlm_corrected_text ?? e.content;
17531
18954
  return {
@@ -17556,7 +18979,7 @@ var loadFile = async (file, name, tempDir) => {
17556
18979
  let buffer;
17557
18980
  if (Buffer.isBuffer(file)) {
17558
18981
  filePath = path2.join(tempDir, `${UUID}.${fileType}`);
17559
- await fs5.promises.writeFile(filePath, file);
18982
+ await fs4.promises.writeFile(filePath, file);
17560
18983
  buffer = file;
17561
18984
  } else {
17562
18985
  filePath = filePath.trim();
@@ -17564,11 +18987,11 @@ var loadFile = async (file, name, tempDir) => {
17564
18987
  const response = await fetch(filePath);
17565
18988
  const array = await response.arrayBuffer();
17566
18989
  const tempFilePath = path2.join(tempDir, `${UUID}.${fileType}`);
17567
- await fs5.promises.writeFile(tempFilePath, Buffer.from(array));
18990
+ await fs4.promises.writeFile(tempFilePath, Buffer.from(array));
17568
18991
  buffer = Buffer.from(array);
17569
18992
  filePath = tempFilePath;
17570
18993
  } else {
17571
- buffer = await fs5.promises.readFile(file);
18994
+ buffer = await fs4.promises.readFile(file);
17572
18995
  }
17573
18996
  }
17574
18997
  return { filePath, fileType, buffer };
@@ -17586,9 +19009,9 @@ async function documentProcessor({
17586
19009
  const tempDir = path2.join(process.cwd(), "temp", uuid);
17587
19010
  const localFilesAndFoldersToDelete = [tempDir];
17588
19011
  console.log(`[EXULU] Temporary directory for processing document ${name}: ${tempDir}`);
17589
- await fs5.promises.mkdir(tempDir, { recursive: true });
19012
+ await fs4.promises.mkdir(tempDir, { recursive: true });
17590
19013
  const timestamp = (/* @__PURE__ */ new Date()).toISOString();
17591
- await fs5.promises.writeFile(path2.join(tempDir, "created_at.txt"), timestamp);
19014
+ await fs4.promises.writeFile(path2.join(tempDir, "created_at.txt"), timestamp);
17592
19015
  try {
17593
19016
  const {
17594
19017
  filePath,
@@ -17598,7 +19021,7 @@ async function documentProcessor({
17598
19021
  let supportedTypes = [];
17599
19022
  switch (config?.processor.name) {
17600
19023
  case "docling":
17601
- supportedTypes = ["pdf", "docx", "doc", "txt", "md"];
19024
+ supportedTypes = ["pdf", "docx", "doc", "txt", "md", "jpg", "jpeg", "png", "gif", "webp"];
17602
19025
  break;
17603
19026
  case "officeparser":
17604
19027
  supportedTypes = [];
@@ -17607,7 +19030,7 @@ async function documentProcessor({
17607
19030
  supportedTypes = ["pdf", "doc", "docx", "docm", "odt", "rtf", "ppt", "pptx", "pptm", "odp", "xls", "xlsx", "xlsm", "ods", "csv", "tsv"];
17608
19031
  break;
17609
19032
  case "mistral":
17610
- supportedTypes = ["pdf", "docx", "doc", "txt", "md"];
19033
+ supportedTypes = ["pdf", "docx", "doc", "txt", "md", "jpg", "jpeg", "png", "gif", "webp"];
17611
19034
  break;
17612
19035
  }
17613
19036
  if (!supportedTypes.includes(fileType)) {
@@ -17629,7 +19052,7 @@ async function documentProcessor({
17629
19052
  if (config?.debugging?.deleteTempFiles !== false) {
17630
19053
  for (const file2 of localFilesAndFoldersToDelete) {
17631
19054
  try {
17632
- await fs5.promises.rm(file2, { recursive: true });
19055
+ await fs4.promises.rm(file2, { recursive: true });
17633
19056
  console.log(`[EXULU] Deleted file or folder: ${file2}`);
17634
19057
  } catch (error) {
17635
19058
  console.error(`[EXULU] Error deleting file or folder: ${file2}`, error);
@@ -17640,634 +19063,6 @@ async function documentProcessor({
17640
19063
  }
17641
19064
  }
17642
19065
 
17643
- // ee/agentic-retrieval/v4/index.ts
17644
- var os = __toESM(require("os"), 1);
17645
- var path4 = __toESM(require("path"), 1);
17646
- var fs7 = __toESM(require("fs/promises"), 1);
17647
- var import_zod19 = require("zod");
17648
- var import_crypto2 = require("crypto");
17649
-
17650
- // ee/agentic-retrieval/v4/tools.ts
17651
- var fs6 = __toESM(require("fs/promises"), 1);
17652
- var path3 = __toESM(require("path"), 1);
17653
- var import_child_process3 = require("child_process");
17654
- var import_util4 = require("util");
17655
- var import_zod18 = require("zod");
17656
- var import_ai12 = require("ai");
17657
-
17658
- // ee/agentic-retrieval/v4/embed-preprocessor.ts
17659
- async function preprocessEmbedCalls(sql, contexts, user, role) {
17660
- const EMBED_RE = /embed\('((?:[^'\\]|\\.)*)'\s*(?:,\s*'((?:[^'\\]|\\.)*)')?\)/gi;
17661
- const matches = [];
17662
- let m;
17663
- while ((m = EMBED_RE.exec(sql)) !== null) {
17664
- matches.push({
17665
- fullMatch: m[0],
17666
- text: m[1],
17667
- contextId: m[2] || void 0,
17668
- index: m.index
17669
- });
17670
- }
17671
- if (matches.length === 0) return sql;
17672
- const substitutions = await Promise.all(
17673
- matches.map(async ({ text, contextId }) => {
17674
- const context = contextId ? contexts.find((c) => c.id === contextId) : contexts.find((c) => c.embedder != null);
17675
- if (!context?.embedder) {
17676
- throw new Error(
17677
- `No embedder available${contextId ? ` for context "${contextId}"` : ""}. Available contexts with embedders: [${contexts.filter((c) => c.embedder).map((c) => c.id).join(", ")}]`
17678
- );
17679
- }
17680
- const result2 = await context.embedder.generateFromQuery(
17681
- context.id,
17682
- text,
17683
- void 0,
17684
- user?.id,
17685
- role
17686
- );
17687
- const vector = result2?.chunks?.[0]?.vector;
17688
- if (!vector?.length) {
17689
- throw new Error(`Embedder returned no vector for text: "${text}"`);
17690
- }
17691
- return `ARRAY[${vector.join(",")}]::vector`;
17692
- })
17693
- );
17694
- let result = sql;
17695
- for (let i = matches.length - 1; i >= 0; i--) {
17696
- const { fullMatch, index } = matches[i];
17697
- result = result.slice(0, index) + substitutions[i] + result.slice(index + fullMatch.length);
17698
- }
17699
- return result;
17700
- }
17701
-
17702
- // ee/agentic-retrieval/v4/tools.ts
17703
- var execAsync3 = (0, import_util4.promisify)(import_child_process3.exec);
17704
- var MAX_INLINE_CHARS = 2e4;
17705
- var MAX_GREP_OUTPUT_CHARS = 5e3;
17706
- var WRITE_PATTERN = /^\s*(INSERT|UPDATE|DELETE|DROP|CREATE|ALTER|TRUNCATE|GRANT|REVOKE|VACUUM|ANALYZE|EXPLAIN\s+ANALYZE)\b/i;
17707
- function assertReadOnly(sql) {
17708
- if (WRITE_PATTERN.test(sql)) {
17709
- throw new Error(
17710
- "Only SELECT queries are allowed. Write operations (INSERT, UPDATE, DELETE, DROP, etc.) are not permitted."
17711
- );
17712
- }
17713
- }
17714
- function rowToChunkResult(row) {
17715
- const chunkId = row.chunk_id ?? row.id;
17716
- const chunkContent = row.chunk_content ?? row.content;
17717
- const itemId = row.item_id ?? row.source;
17718
- const context = row.context ?? row.context_id;
17719
- const itemName = row.item_name ?? row.name;
17720
- if (!chunkId || !chunkContent && !itemId) return null;
17721
- return {
17722
- item_name: itemName ?? "",
17723
- item_id: itemId ?? "",
17724
- context: context ?? "",
17725
- chunk_id: chunkId,
17726
- chunk_index: row.chunk_index ?? void 0,
17727
- chunk_content: chunkContent ?? void 0,
17728
- metadata: row.metadata ?? row.chunk_metadata ?? void 0
17729
- };
17730
- }
17731
- function createTools(params) {
17732
- const { contexts, user, role, sessionDir } = params;
17733
- let queryCount = 0;
17734
- const execute_query = (0, import_ai12.tool)({
17735
- description: `Execute a read-only PostgreSQL SELECT query against the knowledge base.
17736
-
17737
- Use this to search, filter, aggregate, and explore content. The database contains items
17738
- and chunks tables for each knowledge base (see schema in the system prompt).
17739
-
17740
- Use embed('your text') anywhere in the query to generate a semantic search vector:
17741
- embedding <=> embed('machine learning') AS distance
17742
-
17743
- If the result exceeds ${(MAX_INLINE_CHARS / 1e3).toFixed(0)}k characters it is saved to a file.
17744
- Use the grep tool to iteratively search the file for relevant information.`,
17745
- inputSchema: import_zod18.z.object({
17746
- sql: import_zod18.z.string().describe("A read-only SELECT (or WITH ... SELECT) PostgreSQL query")
17747
- }),
17748
- execute: async ({ sql }) => {
17749
- assertReadOnly(sql);
17750
- let processedSql;
17751
- try {
17752
- processedSql = await preprocessEmbedCalls(sql, contexts, user, role);
17753
- } catch (err) {
17754
- return JSON.stringify({ error: `embed() preprocessing failed: ${err.message}` });
17755
- }
17756
- let rows;
17757
- try {
17758
- const { db: db2 } = await postgresClient();
17759
- const result = await db2.raw(processedSql);
17760
- rows = result.rows ?? [];
17761
- } catch (err) {
17762
- return JSON.stringify({ error: `Query failed: ${err.message}` });
17763
- }
17764
- const json = JSON.stringify(rows, null, 2);
17765
- if (json.length <= MAX_INLINE_CHARS) {
17766
- return json;
17767
- }
17768
- await fs6.mkdir(sessionDir, { recursive: true });
17769
- const filename = `query_${++queryCount}.json`;
17770
- const filePath = path3.join(sessionDir, filename);
17771
- await fs6.writeFile(filePath, json, "utf-8");
17772
- return JSON.stringify({
17773
- stored: true,
17774
- file: filePath,
17775
- row_count: rows.length,
17776
- message: `Results too large to display (${rows.length} rows, ${(json.length / 1e3).toFixed(1)}k chars). Stored at ${filePath}. Use the grep tool to search for relevant information.`,
17777
- grep_hint: `grep -i "keyword" ${filePath}`
17778
- });
17779
- }
17780
- });
17781
- const grep = (0, import_ai12.tool)({
17782
- description: `Search a stored query result file using grep.
17783
-
17784
- Use this after execute_query returns a file path because results were too large.
17785
- Iteratively narrow down the results with multiple grep calls.`,
17786
- inputSchema: import_zod18.z.object({
17787
- pattern: import_zod18.z.string().describe("Regular expression or literal string to search for"),
17788
- file: import_zod18.z.string().describe("Absolute path to the file returned by execute_query"),
17789
- context_lines: import_zod18.z.number().int().min(0).max(10).default(2).describe("Number of lines of context to show around each match (default 2)"),
17790
- case_insensitive: import_zod18.z.boolean().default(true).describe("Case-insensitive matching (default true)")
17791
- }),
17792
- execute: async ({ pattern, file, context_lines, case_insensitive }) => {
17793
- const resolvedFile = path3.resolve(file);
17794
- const resolvedSession = path3.resolve(sessionDir);
17795
- if (!resolvedFile.startsWith(resolvedSession)) {
17796
- return JSON.stringify({
17797
- error: `Access denied. Only files within the session directory (${sessionDir}) can be searched.`
17798
- });
17799
- }
17800
- try {
17801
- await fs6.access(resolvedFile);
17802
- } catch {
17803
- return JSON.stringify({ error: `File not found: ${file}` });
17804
- }
17805
- const flags = [
17806
- "-n",
17807
- context_lines > 0 ? `-C${context_lines}` : "",
17808
- case_insensitive ? "-i" : ""
17809
- ].filter(Boolean).join(" ");
17810
- const escapedPattern = pattern.replace(/'/g, `'\\''`);
17811
- const cmd = `grep ${flags} '${escapedPattern}' '${resolvedFile}'`;
17812
- let output;
17813
- try {
17814
- const { stdout } = await execAsync3(cmd, { maxBuffer: 10 * 1024 * 1024 });
17815
- output = stdout;
17816
- } catch (err) {
17817
- if (err.code === 1) {
17818
- return JSON.stringify({ matches: 0, output: "No matches found." });
17819
- }
17820
- return JSON.stringify({ error: `grep failed: ${err.message}` });
17821
- }
17822
- if (output.length > MAX_GREP_OUTPUT_CHARS) {
17823
- output = output.slice(0, MAX_GREP_OUTPUT_CHARS) + `
17824
- ... (output truncated at ${MAX_GREP_OUTPUT_CHARS} chars \u2014 refine your pattern to narrow results)`;
17825
- }
17826
- const lineCount = output.split("\n").filter(Boolean).length;
17827
- return JSON.stringify({ matches: lineCount, output });
17828
- }
17829
- });
17830
- return { execute_query, grep };
17831
- }
17832
- function harvestChunks(toolResults) {
17833
- const chunks = [];
17834
- for (const result of toolResults ?? []) {
17835
- const rawOutput = result.output ?? result.result;
17836
- let parsed;
17837
- try {
17838
- parsed = typeof rawOutput === "string" ? JSON.parse(rawOutput) : rawOutput;
17839
- } catch {
17840
- continue;
17841
- }
17842
- if (Array.isArray(parsed)) {
17843
- for (const row of parsed) {
17844
- if (row && typeof row === "object") {
17845
- const chunk = rowToChunkResult(row);
17846
- if (chunk) chunks.push(chunk);
17847
- }
17848
- }
17849
- }
17850
- }
17851
- return chunks;
17852
- }
17853
-
17854
- // ee/agentic-retrieval/v4/system-prompt.ts
17855
- function buildSystemPrompt(contexts, customInstructions) {
17856
- const schemaBlock = buildSchemaBlock(contexts);
17857
- const hasEmbedder = contexts.some((c) => c.embedder != null);
17858
- return `You are a knowledge base retrieval agent. Your job is to find all information relevant to the user's query.
17859
-
17860
- ## Approach: Observe \u2192 Infer \u2192 Act
17861
-
17862
- Work iteratively:
17863
- 1. **Observe** \u2014 examine what data you have and what the query asks for
17864
- 2. **Infer** \u2014 decide what SQL query will best surface relevant information
17865
- 3. **Act** \u2014 execute the query and study the results
17866
- 4. Repeat until you have found sufficient information, then write your final answer.
17867
-
17868
- Do NOT guess or hallucinate. If results are empty, try alternative queries (different keywords,
17869
- broader filters, semantic search). Exhaust the available search strategies before concluding
17870
- that no relevant data exists.
17871
-
17872
- ---
17873
-
17874
- ## Database Schema
17875
-
17876
- ${schemaBlock}
17877
-
17878
- ---
17879
-
17880
- ## Query Patterns
17881
-
17882
- ### Keyword / Full-Text Search
17883
- \`\`\`sql
17884
- SELECT
17885
- c.id AS chunk_id,
17886
- c.chunk_index,
17887
- c.content AS chunk_content,
17888
- c.metadata,
17889
- c.source AS item_id,
17890
- i.name AS item_name,
17891
- '<context_id>' AS context
17892
- FROM <context_id>_chunks c
17893
- JOIN <context_id>_items i ON c.source = i.id
17894
- WHERE c.fts @@ plainto_tsquery('english', 'your search terms')
17895
- AND (i.archived IS FALSE OR i.archived IS NULL)
17896
- ORDER BY ts_rank(c.fts, plainto_tsquery('english', 'your search terms')) DESC
17897
- LIMIT 20;
17898
- \`\`\`
17899
-
17900
- For German text use \`'german'\` instead of \`'english'\`.
17901
- For multi-language, use \`websearch_to_tsquery\` or UNION both languages.
17902
- ${hasEmbedder ? `
17903
- ### Semantic Search (use embed() helper)
17904
- \`\`\`sql
17905
- SELECT
17906
- c.id AS chunk_id,
17907
- c.chunk_index,
17908
- c.content AS chunk_content,
17909
- c.metadata,
17910
- c.source AS item_id,
17911
- i.name AS item_name,
17912
- '<context_id>' AS context,
17913
- c.embedding <=> embed('your concept here') AS distance
17914
- FROM <context_id>_chunks c
17915
- JOIN <context_id>_items i ON c.source = i.id
17916
- WHERE (i.archived IS FALSE OR i.archived IS NULL)
17917
- ORDER BY distance ASC
17918
- LIMIT 20;
17919
- \`\`\`
17920
-
17921
- ### Hybrid Search (keyword + semantic combined via RRF)
17922
- \`\`\`sql
17923
- WITH fts AS (
17924
- SELECT id, ROW_NUMBER() OVER (ORDER BY ts_rank(fts, q) DESC) AS rank
17925
- FROM <context_id>_chunks, plainto_tsquery('english', 'your query') q
17926
- WHERE fts @@ q
17927
- LIMIT 500
17928
- ),
17929
- sem AS (
17930
- SELECT id, ROW_NUMBER() OVER (ORDER BY embedding <=> embed('your query') ASC) AS rank
17931
- FROM <context_id>_chunks
17932
- LIMIT 500
17933
- ),
17934
- rrf AS (
17935
- SELECT
17936
- COALESCE(fts.id, sem.id) AS id,
17937
- (COALESCE(1.0 / (50 + fts.rank), 0) * 2 + COALESCE(1.0 / (50 + sem.rank), 0)) AS score
17938
- FROM fts FULL OUTER JOIN sem ON fts.id = sem.id
17939
- )
17940
- SELECT
17941
- c.id AS chunk_id,
17942
- c.chunk_index,
17943
- c.content AS chunk_content,
17944
- c.metadata,
17945
- c.source AS item_id,
17946
- i.name AS item_name,
17947
- '<context_id>' AS context,
17948
- rrf.score
17949
- FROM rrf
17950
- JOIN <context_id>_chunks c ON c.id = rrf.id
17951
- JOIN <context_id>_items i ON c.source = i.id
17952
- WHERE (i.archived IS FALSE OR i.archived IS NULL)
17953
- ORDER BY rrf.score DESC
17954
- LIMIT 20;
17955
- \`\`\`
17956
- ` : `
17957
- Note: No embedder is configured for these contexts. Use keyword/full-text search only.
17958
- `}
17959
- ### Browse all chunks of a specific document (in order)
17960
- \`\`\`sql
17961
- SELECT
17962
- c.id AS chunk_id,
17963
- c.chunk_index,
17964
- c.content AS chunk_content,
17965
- c.metadata,
17966
- c.source AS item_id,
17967
- i.name AS item_name,
17968
- '<context_id>' AS context
17969
- FROM <context_id>_chunks c
17970
- JOIN <context_id>_items i ON c.source = i.id
17971
- WHERE c.source = '<item_id>'
17972
- ORDER BY c.chunk_index;
17973
- \`\`\`
17974
-
17975
- ### Count / aggregate
17976
- \`\`\`sql
17977
- SELECT COUNT(*) FROM <context_id>_items WHERE archived IS FALSE;
17978
- SELECT COUNT(*) FROM <context_id>_chunks;
17979
- \`\`\`
17980
-
17981
- ### Explore item names (when query is about a specific document)
17982
- \`\`\`sql
17983
- SELECT id, name, external_id, "createdAt"
17984
- FROM <context_id>_items
17985
- WHERE (archived IS FALSE OR archived IS NULL)
17986
- AND LOWER(name) LIKE '%keyword%'
17987
- LIMIT 50;
17988
- \`\`\`
17989
-
17990
- ### Filter by custom metadata on chunks
17991
- \`\`\`sql
17992
- SELECT chunk_id, chunk_content, item_name, context
17993
- FROM ...
17994
- WHERE c.metadata->>'page' = '5'
17995
- OR c.metadata @> '{"category": "finance"}'
17996
- \`\`\`
17997
-
17998
- ---
17999
-
18000
- ## Column Alias Convention
18001
-
18002
- **Always use these aliases** in queries that return chunks so results are collected correctly:
18003
-
18004
- | Alias | Source column |
18005
- |----------------|-------------------------|
18006
- | \`chunk_id\` | \`c.id\` |
18007
- | \`chunk_index\` | \`c.chunk_index\` |
18008
- | \`chunk_content\`| \`c.content\` |
18009
- | \`item_id\` | \`c.source\` |
18010
- | \`item_name\` | \`i.name\` |
18011
- | \`context\` | literal context id string |
18012
- | \`metadata\` | \`c.metadata\` |
18013
-
18014
- ---
18015
-
18016
- ## Handling Large Results
18017
-
18018
- When execute_query returns a file path (results > 20k chars):
18019
- 1. Use \`grep\` with a specific pattern to find relevant sections
18020
- 2. Multiple grep calls are fine \u2014 narrow down iteratively
18021
- 3. Once you know specific \`item_id\` or \`chunk_id\` values, run a targeted SELECT to get full content
18022
-
18023
- ---
18024
-
18025
- ## Search Strategy
18026
-
18027
- - **Start broad**: use keyword or hybrid search with your main terms, LIMIT 30\u201350
18028
- - **Go deeper**: if results are sparse, try alternative phrasings, synonyms, or semantic search
18029
- - **Drill into documents**: once you find a relevant item, fetch its chunks in order to get full context
18030
- - **Cross-context**: search multiple contexts when the query could span knowledge bases
18031
- - **Aggregate last**: use COUNT queries only for "how many" questions
18032
-
18033
- ---
18034
- ${customInstructions ? `## Additional Instructions
18035
-
18036
- ${customInstructions}
18037
-
18038
- ---
18039
- ` : ""}
18040
- When you have gathered sufficient information, write a clear answer. Do not call any more tools once you have what you need.`;
18041
- }
18042
- function buildSchemaBlock(contexts) {
18043
- return contexts.map((ctx) => {
18044
- const itemsTable = getTableName(ctx.id);
18045
- const chunksTable = getChunksTableName(ctx.id);
18046
- const customFields = ctx.fields.length > 0 ? ctx.fields.map((f) => ` ${f.name} (${f.type})`).join("\n") : " (no custom fields)";
18047
- const embedderNote = ctx.embedder ? `Embedder: ${ctx.embedder.name} \u2014 semantic search and embed() are available` : "No embedder \u2014 use keyword search only";
18048
- return `### Context: "${ctx.name}" (id: \`${ctx.id}\`)
18049
- ${ctx.description || ""}
18050
- ${embedderNote}
18051
-
18052
- **${itemsTable}** \u2014 documents / items
18053
- id (uuid, primary key)
18054
- name (text)
18055
- external_id (text, nullable)
18056
- archived (boolean, nullable)
18057
- created_by (integer, nullable)
18058
- rights_mode (text, nullable)
18059
- "createdAt" (timestamp)
18060
- "updatedAt" (timestamp)
18061
- -- Custom fields:
18062
- ${customFields}
18063
-
18064
- **${chunksTable}** \u2014 text chunks (source FK \u2192 ${itemsTable}.id)
18065
- id (uuid, primary key)
18066
- source (uuid, FK \u2192 ${itemsTable}.id)
18067
- content (text)
18068
- chunk_index (integer)
18069
- fts (tsvector \u2014 full-text search index)
18070
- embedding (vector \u2014 pgvector, nullable)
18071
- metadata (jsonb, nullable)
18072
- "createdAt" (timestamp)
18073
- "updatedAt" (timestamp)`;
18074
- }).join("\n\n");
18075
- }
18076
-
18077
- // ee/agentic-retrieval/v4/agent-loop.ts
18078
- var import_ai13 = require("ai");
18079
- var MAX_STEPS = 10;
18080
- async function* runAgentLoop2(params) {
18081
- const { query, systemPrompt, tools, model, onStepComplete } = params;
18082
- const output = {
18083
- steps: [],
18084
- reasoning: [],
18085
- chunks: [],
18086
- usage: [],
18087
- totalTokens: 0
18088
- };
18089
- const seenChunkIds = /* @__PURE__ */ new Set();
18090
- const messages = [{ role: "user", content: query }];
18091
- for (let step = 0; step < MAX_STEPS; step++) {
18092
- console.log(`[EXULU] v4 agent loop \u2014 step ${step + 1}/${MAX_STEPS}`);
18093
- let result;
18094
- try {
18095
- result = await withRetry(
18096
- () => (0, import_ai13.generateText)({
18097
- model,
18098
- temperature: 0,
18099
- system: systemPrompt,
18100
- messages,
18101
- tools,
18102
- toolChoice: "auto",
18103
- stopWhen: (0, import_ai13.stepCountIs)(1)
18104
- })
18105
- );
18106
- } catch (err) {
18107
- console.error("[EXULU] v4 generateText failed:", err);
18108
- throw err;
18109
- }
18110
- messages.push(...result.response.messages);
18111
- const rawToolResults = result.toolResults ?? [];
18112
- const stepChunks = [];
18113
- for (const chunk of harvestChunks(rawToolResults)) {
18114
- if (!chunk.chunk_id || !seenChunkIds.has(chunk.chunk_id)) {
18115
- if (chunk.chunk_id) seenChunkIds.add(chunk.chunk_id);
18116
- stepChunks.push(chunk);
18117
- }
18118
- }
18119
- const stepRecord = {
18120
- stepNumber: step + 1,
18121
- text: result.text ?? "",
18122
- toolCalls: result.toolCalls?.map((tc) => ({
18123
- name: tc.toolName,
18124
- id: tc.toolCallId,
18125
- input: tc.input
18126
- })) ?? [],
18127
- chunks: stepChunks,
18128
- tokens: result.usage?.totalTokens ?? 0
18129
- };
18130
- output.steps.push(stepRecord);
18131
- output.reasoning.push({
18132
- text: result.text ?? "",
18133
- tools: result.toolCalls?.map((tc) => ({
18134
- name: tc.toolName,
18135
- id: tc.toolCallId,
18136
- input: tc.input,
18137
- output: rawToolResults.find(
18138
- (r) => (r.toolCallId ?? r.id) === tc.toolCallId
18139
- )?.output
18140
- })) ?? []
18141
- });
18142
- output.chunks.push(...stepChunks);
18143
- output.usage.push(result.usage);
18144
- onStepComplete?.(stepRecord);
18145
- yield { ...output };
18146
- const calledTools = result.toolCalls?.length > 0;
18147
- if (!calledTools) {
18148
- console.log(`[EXULU] v4 \u2014 model finished after step ${step + 1} (no tool calls)`);
18149
- break;
18150
- }
18151
- }
18152
- output.totalTokens = output.usage.reduce((sum, u) => sum + (u?.totalTokens ?? 0), 0);
18153
- }
18154
-
18155
- // ee/agentic-retrieval/v4/index.ts
18156
- async function* executeV4({
18157
- query,
18158
- contexts,
18159
- model,
18160
- user,
18161
- role,
18162
- customInstructions
18163
- }) {
18164
- const sessionId = (0, import_crypto2.randomUUID)();
18165
- const sessionDir = path4.join(os.tmpdir(), `exulu-v4-${sessionId}`);
18166
- console.log("[EXULU] v4 \u2014 starting observe-infer-act retrieval");
18167
- const tools = createTools({ contexts, user, role, sessionDir });
18168
- const systemPrompt = buildSystemPrompt(contexts, customInstructions);
18169
- let finalOutput;
18170
- try {
18171
- for await (const output of runAgentLoop2({
18172
- query,
18173
- systemPrompt,
18174
- tools,
18175
- model
18176
- })) {
18177
- finalOutput = output;
18178
- yield output;
18179
- }
18180
- } finally {
18181
- fs7.rm(sessionDir, { recursive: true, force: true }).catch(() => {
18182
- });
18183
- }
18184
- if (finalOutput) {
18185
- console.log(
18186
- `[EXULU] v4 \u2014 done. steps=${finalOutput.steps.length} chunks=${finalOutput.chunks.length} tokens=${finalOutput.totalTokens}`
18187
- );
18188
- }
18189
- }
18190
- function createAgenticRetrievalToolV4({
18191
- contexts,
18192
- instructions: adminInstructions,
18193
- rerankers,
18194
- user,
18195
- role,
18196
- model
18197
- }) {
18198
- const license = checkLicense();
18199
- if (!license["agentic-retrieval"]) {
18200
- console.warn("[EXULU] Not licensed for agentic retrieval");
18201
- return void 0;
18202
- }
18203
- const contextNames = contexts.map((c) => c.id).join(", ");
18204
- return new ExuluTool({
18205
- id: "agentic_context_search_v4",
18206
- name: "Agentic Context Search (V4)",
18207
- description: `Observe-infer-act retrieval using raw SQL. Searches: ${contextNames}`,
18208
- category: "contexts",
18209
- needsApproval: false,
18210
- type: "context",
18211
- config: [
18212
- {
18213
- name: "instructions",
18214
- description: "Custom instructions for the retrieval agent",
18215
- type: "string",
18216
- default: ""
18217
- },
18218
- {
18219
- name: "reasoning_model",
18220
- description: "Override the model used by the retrieval agent (default: inherits from calling agent)",
18221
- type: "string",
18222
- default: ""
18223
- },
18224
- ...contexts.map((ctx) => ({
18225
- name: ctx.id,
18226
- description: `Enable search in "${ctx.name}". ${ctx.description}`,
18227
- type: "boolean",
18228
- default: true
18229
- }))
18230
- ],
18231
- inputSchema: import_zod19.z.object({
18232
- query: import_zod19.z.string().describe("The question or query to answer"),
18233
- userInstructions: import_zod19.z.string().optional().describe("Additional instructions from the user to guide retrieval")
18234
- }),
18235
- execute: async function* ({
18236
- query,
18237
- userInstructions,
18238
- toolVariablesConfig
18239
- }) {
18240
- if (!model) {
18241
- throw new Error("Model is required for executing the agentic retrieval tool");
18242
- }
18243
- let activeContexts = contexts;
18244
- let configInstructions = "";
18245
- if (toolVariablesConfig) {
18246
- configInstructions = toolVariablesConfig["instructions"] ?? "";
18247
- activeContexts = contexts.filter(
18248
- (ctx) => toolVariablesConfig[ctx.id] === true || toolVariablesConfig[ctx.id] === "true" || toolVariablesConfig[ctx.id] === 1
18249
- );
18250
- if (activeContexts.length === 0) activeContexts = contexts;
18251
- }
18252
- const combinedInstructions = [
18253
- configInstructions ? `Configuration instructions: ${configInstructions}` : "",
18254
- adminInstructions ? `Admin instructions: ${adminInstructions}` : "",
18255
- userInstructions ? `User instructions: ${userInstructions}` : ""
18256
- ].filter(Boolean).join("\n");
18257
- for await (const output of executeV4({
18258
- query,
18259
- contexts: activeContexts,
18260
- model,
18261
- user,
18262
- role,
18263
- customInstructions: combinedInstructions || void 0
18264
- })) {
18265
- yield { result: JSON.stringify(output) };
18266
- }
18267
- }
18268
- });
18269
- }
18270
-
18271
19066
  // src/index.ts
18272
19067
  var ExuluJobs = {
18273
19068
  redis: redisClient
@@ -18276,8 +19071,7 @@ var ExuluDefaultTools = {
18276
19071
  agentic: {
18277
19072
  retrieval: {
18278
19073
  create: {
18279
- v3: createAgenticRetrievalToolV3,
18280
- v4: createAgenticRetrievalToolV4
19074
+ v3: createAgenticRetrievalToolV3
18281
19075
  }
18282
19076
  }
18283
19077
  }
@@ -18318,8 +19112,8 @@ var ExuluVariables = {
18318
19112
  throw new Error(`Variable ${name} not found.`);
18319
19113
  }
18320
19114
  if (variable.encrypted) {
18321
- const bytes = import_crypto_js9.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
18322
- variable.value = bytes.toString(import_crypto_js9.default.enc.Utf8);
19115
+ const bytes = import_crypto_js10.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
19116
+ variable.value = bytes.toString(import_crypto_js10.default.enc.Utf8);
18323
19117
  }
18324
19118
  return variable.value;
18325
19119
  }