@exulu/backend 1.54.0 → 1.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1970 -1176
- package/dist/index.d.cts +6 -29
- package/dist/index.d.ts +6 -29
- package/dist/index.js +1963 -1164
- package/ee/agentic-retrieval/v3/agent-loop.ts +49 -3
- package/ee/agentic-retrieval/v3/classifier.ts +42 -37
- package/ee/agentic-retrieval/v3/index.ts +112 -18
- package/ee/agentic-retrieval/v3/session-tools-registry.ts +20 -0
- package/ee/agentic-retrieval/v3/strategies.ts +28 -24
- package/ee/agentic-retrieval/v3/tools.ts +226 -111
- package/ee/agentic-retrieval/v3/trajectory.ts +227 -14
- package/ee/invoke-skills/create-sandbox.ts +119 -0
- package/ee/python/documents/processing/doc_processor.ts +106 -14
- package/package.json +4 -2
- package/ee/agentic-retrieval/ANALYSIS.md +0 -658
- package/ee/agentic-retrieval/index.ts +0 -1109
- package/ee/agentic-retrieval/logs/README.md +0 -198
- package/ee/agentic-retrieval/v2.ts +0 -1628
- package/ee/agentic-retrieval/v4/agent-loop.ts +0 -121
- package/ee/agentic-retrieval/v4/embed-preprocessor.ts +0 -76
- package/ee/agentic-retrieval/v4/index.ts +0 -181
- package/ee/agentic-retrieval/v4/system-prompt.ts +0 -248
- package/ee/agentic-retrieval/v4/tools.ts +0 -241
- package/ee/agentic-retrieval/v4/types.ts +0 -29
package/dist/index.cjs
CHANGED
|
@@ -101,10 +101,10 @@ async function redisClient() {
|
|
|
101
101
|
}
|
|
102
102
|
|
|
103
103
|
// src/exulu/app/index.ts
|
|
104
|
-
var
|
|
104
|
+
var import_express7 = require("express");
|
|
105
105
|
|
|
106
106
|
// src/exulu/routes.ts
|
|
107
|
-
var
|
|
107
|
+
var import_express4 = require("express");
|
|
108
108
|
|
|
109
109
|
// src/auth/get-token.ts
|
|
110
110
|
var import_jose = require("jose");
|
|
@@ -548,7 +548,7 @@ var STATISTICS_TYPE_ENUM = {
|
|
|
548
548
|
};
|
|
549
549
|
|
|
550
550
|
// src/exulu/routes.ts
|
|
551
|
-
var
|
|
551
|
+
var import_express5 = __toESM(require("express"), 1);
|
|
552
552
|
var import_server4 = require("@apollo/server");
|
|
553
553
|
var import_cors = __toESM(require("cors"), 1);
|
|
554
554
|
var import_reflect_metadata = require("reflect-metadata");
|
|
@@ -807,7 +807,7 @@ var JOB_STATUS_ENUM = {
|
|
|
807
807
|
};
|
|
808
808
|
|
|
809
809
|
// ee/agentic-retrieval/v3/index.ts
|
|
810
|
-
var
|
|
810
|
+
var import_zod9 = require("zod");
|
|
811
811
|
var import_bash_tool = require("bash-tool");
|
|
812
812
|
|
|
813
813
|
// src/exulu/tool.ts
|
|
@@ -1041,10 +1041,25 @@ var createSessionItemsRetrievalTool = async ({
|
|
|
1041
1041
|
return sessionItemsRetrievalTool;
|
|
1042
1042
|
};
|
|
1043
1043
|
|
|
1044
|
+
// ee/agentic-retrieval/v3/session-tools-registry.ts
|
|
1045
|
+
var registry = /* @__PURE__ */ new Map();
|
|
1046
|
+
function registerSessionTools(sessionId, tools) {
|
|
1047
|
+
const existing = registry.get(sessionId) ?? /* @__PURE__ */ new Map();
|
|
1048
|
+
for (const [name, toolDef] of Object.entries(tools)) {
|
|
1049
|
+
existing.set(name, toolDef);
|
|
1050
|
+
}
|
|
1051
|
+
registry.set(sessionId, existing);
|
|
1052
|
+
}
|
|
1053
|
+
function getSessionTools(sessionId) {
|
|
1054
|
+
const toolMap = registry.get(sessionId);
|
|
1055
|
+
if (!toolMap || toolMap.size === 0) return {};
|
|
1056
|
+
return Object.fromEntries(toolMap.entries());
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1044
1059
|
// src/utils/sanitize-tool-name.ts
|
|
1045
1060
|
function sanitizeToolName(name) {
|
|
1046
1061
|
if (typeof name !== "string") return "";
|
|
1047
|
-
let sanitized = name.replace(/[^a-zA-Z0-9_
|
|
1062
|
+
let sanitized = name.replace(/[^a-zA-Z0-9_\:-]+/g, "_");
|
|
1048
1063
|
if (sanitized.length > 0 && !/^[a-zA-Z_]/.test(sanitized)) {
|
|
1049
1064
|
sanitized = "_" + sanitized;
|
|
1050
1065
|
}
|
|
@@ -1066,7 +1081,6 @@ var import_node_crypto = require("crypto");
|
|
|
1066
1081
|
|
|
1067
1082
|
// src/templates/tools/memory-tool.ts
|
|
1068
1083
|
var import_zod3 = require("zod");
|
|
1069
|
-
var import_fs = require("fs");
|
|
1070
1084
|
var createNewMemoryItemTool = (agent, context) => {
|
|
1071
1085
|
const fields = {
|
|
1072
1086
|
name: import_zod3.z.string().describe("The name of the item to create"),
|
|
@@ -1204,9 +1218,9 @@ var getMimeType = (type) => {
|
|
|
1204
1218
|
return "";
|
|
1205
1219
|
}
|
|
1206
1220
|
};
|
|
1207
|
-
var hydrateVariables = async (
|
|
1221
|
+
var hydrateVariables = async (tool5) => {
|
|
1208
1222
|
const { db: db2 } = await postgresClient();
|
|
1209
|
-
const promises2 =
|
|
1223
|
+
const promises2 = tool5.config.map(async (toolConfig) => {
|
|
1210
1224
|
if (!toolConfig.variable) {
|
|
1211
1225
|
return toolConfig;
|
|
1212
1226
|
}
|
|
@@ -1237,9 +1251,9 @@ var hydrateVariables = async (tool6) => {
|
|
|
1237
1251
|
return toolConfig;
|
|
1238
1252
|
});
|
|
1239
1253
|
await Promise.all(promises2);
|
|
1240
|
-
return
|
|
1254
|
+
return tool5;
|
|
1241
1255
|
};
|
|
1242
|
-
var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExuluTools, configs, providerapikey, contexts, rerankers, user, exuluConfig, sessionID, req, project,
|
|
1256
|
+
var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExuluTools, configs, providerapikey, contexts, rerankers, user, exuluConfig, sessionID, req, project, sessionItems, model, agent) => {
|
|
1243
1257
|
if (!currentTools) return {};
|
|
1244
1258
|
if (!allExuluTools) {
|
|
1245
1259
|
allExuluTools = [];
|
|
@@ -1274,13 +1288,13 @@ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExulu
|
|
|
1274
1288
|
currentTools.push(createNewMemoryTool);
|
|
1275
1289
|
}
|
|
1276
1290
|
}
|
|
1277
|
-
console.log("[EXULU] Convert tools array to object, session items",
|
|
1278
|
-
if (
|
|
1291
|
+
console.log("[EXULU] Convert tools array to object, session items", sessionItems);
|
|
1292
|
+
if (sessionItems) {
|
|
1279
1293
|
const sessionItemsRetrievalTool = await createSessionItemsRetrievalTool({
|
|
1280
1294
|
user,
|
|
1281
1295
|
role: user?.role?.id,
|
|
1282
1296
|
contexts,
|
|
1283
|
-
items
|
|
1297
|
+
items: sessionItems
|
|
1284
1298
|
});
|
|
1285
1299
|
if (sessionItemsRetrievalTool) {
|
|
1286
1300
|
currentTools.push(sessionItemsRetrievalTool);
|
|
@@ -1294,10 +1308,11 @@ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExulu
|
|
|
1294
1308
|
rerankers: rerankers || [],
|
|
1295
1309
|
user,
|
|
1296
1310
|
role: user?.role?.id,
|
|
1297
|
-
model
|
|
1311
|
+
model,
|
|
1312
|
+
preselectedItemIds: sessionItems
|
|
1298
1313
|
});
|
|
1299
1314
|
if (agenticSearchTool) {
|
|
1300
|
-
const index = currentTools.findIndex((
|
|
1315
|
+
const index = currentTools.findIndex((tool5) => tool5.id === "agentic_context_search");
|
|
1301
1316
|
if (index !== -1) {
|
|
1302
1317
|
currentTools[index] = {
|
|
1303
1318
|
...currentTools[index],
|
|
@@ -1307,21 +1322,26 @@ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExulu
|
|
|
1307
1322
|
}
|
|
1308
1323
|
}
|
|
1309
1324
|
} else {
|
|
1310
|
-
const agenticSearchTool = currentTools.find((
|
|
1325
|
+
const agenticSearchTool = currentTools.find((tool5) => tool5.id === "agentic_context_search");
|
|
1311
1326
|
if (agenticSearchTool) {
|
|
1312
1327
|
currentTools.splice(currentTools.indexOf(agenticSearchTool), 1);
|
|
1313
1328
|
}
|
|
1314
1329
|
}
|
|
1315
|
-
const sanitizedTools = currentTools ? currentTools.map((
|
|
1316
|
-
...
|
|
1317
|
-
name: sanitizeToolName(
|
|
1330
|
+
const sanitizedTools = currentTools ? currentTools.map((tool5) => ({
|
|
1331
|
+
...tool5,
|
|
1332
|
+
name: sanitizeToolName(tool5.name)
|
|
1318
1333
|
})) : [];
|
|
1319
1334
|
console.log(
|
|
1320
1335
|
"[EXULU] Sanitized tools",
|
|
1321
1336
|
sanitizedTools.map((x) => x.name + " (" + x.id + ")")
|
|
1322
1337
|
);
|
|
1323
1338
|
console.log("[EXULU] Approved tools", approvedTools);
|
|
1339
|
+
const sessionDynamicTools = sessionID ? Object.entries(getSessionTools(sessionID)).reduce((acc, [name, t]) => {
|
|
1340
|
+
acc[name] = { ...t, needsApproval: false };
|
|
1341
|
+
return acc;
|
|
1342
|
+
}, {}) : {};
|
|
1324
1343
|
return {
|
|
1344
|
+
...sessionDynamicTools,
|
|
1325
1345
|
...sanitizedTools?.reduce((prev, cur) => {
|
|
1326
1346
|
let toolVariableConfig = configs?.find((config) => config.id === cur.id);
|
|
1327
1347
|
const userDefinedConfigDescription = toolVariableConfig?.config.find(
|
|
@@ -1523,7 +1543,7 @@ var ExuluTool = class {
|
|
|
1523
1543
|
});
|
|
1524
1544
|
}
|
|
1525
1545
|
execute = async ({
|
|
1526
|
-
agent:
|
|
1546
|
+
agent: agentId2,
|
|
1527
1547
|
config,
|
|
1528
1548
|
user,
|
|
1529
1549
|
inputs,
|
|
@@ -1531,14 +1551,14 @@ var ExuluTool = class {
|
|
|
1531
1551
|
items
|
|
1532
1552
|
}) => {
|
|
1533
1553
|
console.log("[EXULU] Calling tool execute directly", {
|
|
1534
|
-
agentId,
|
|
1554
|
+
agentId: agentId2,
|
|
1535
1555
|
config,
|
|
1536
1556
|
user,
|
|
1537
1557
|
inputs,
|
|
1538
1558
|
project,
|
|
1539
1559
|
items
|
|
1540
1560
|
});
|
|
1541
|
-
const agent = await exuluApp.get().agent(
|
|
1561
|
+
const agent = await exuluApp.get().agent(agentId2);
|
|
1542
1562
|
if (!agent) {
|
|
1543
1563
|
throw new Error("Agent not found.");
|
|
1544
1564
|
}
|
|
@@ -1581,8 +1601,8 @@ var ExuluTool = class {
|
|
|
1581
1601
|
void 0,
|
|
1582
1602
|
agent
|
|
1583
1603
|
);
|
|
1584
|
-
const
|
|
1585
|
-
if (!
|
|
1604
|
+
const tool5 = tools[sanitizeName(this.name)] || tools[this.name] || tools[this.id];
|
|
1605
|
+
if (!tool5?.execute) {
|
|
1586
1606
|
throw new Error("Tool " + sanitizeName(this.name) + " not found in " + JSON.stringify(tools));
|
|
1587
1607
|
}
|
|
1588
1608
|
console.log("[EXULU] Tool found", this.name);
|
|
@@ -1592,7 +1612,7 @@ var ExuluTool = class {
|
|
|
1592
1612
|
toolCallId,
|
|
1593
1613
|
messages: []
|
|
1594
1614
|
});
|
|
1595
|
-
const generator =
|
|
1615
|
+
const generator = tool5.execute(inputs, {
|
|
1596
1616
|
toolCallId,
|
|
1597
1617
|
messages: []
|
|
1598
1618
|
});
|
|
@@ -1722,7 +1742,7 @@ var uploadFile = async (file, fileName, config, options = {}, user, customBucket
|
|
|
1722
1742
|
if (error.name === "SignatureDoesNotMatch" || error.name === "InvalidAccessKeyId" || error.name === "AccessDenied") {
|
|
1723
1743
|
if (attempt < maxRetries) {
|
|
1724
1744
|
const backoffMs = Math.pow(2, attempt) * 1e3;
|
|
1725
|
-
await new Promise((
|
|
1745
|
+
await new Promise((resolve3) => setTimeout(resolve3, backoffMs));
|
|
1726
1746
|
s3Client2 = void 0;
|
|
1727
1747
|
getS3Client(config);
|
|
1728
1748
|
continue;
|
|
@@ -1737,6 +1757,87 @@ var uploadFile = async (file, fileName, config, options = {}, user, customBucket
|
|
|
1737
1757
|
}
|
|
1738
1758
|
return addBucketPrefixToKey(key, customBucket || defaultBucket);
|
|
1739
1759
|
};
|
|
1760
|
+
var listS3ObjectsByPrefix = async (prefix, config) => {
|
|
1761
|
+
if (!config.fileUploads) {
|
|
1762
|
+
throw new Error("File uploads are not configured");
|
|
1763
|
+
}
|
|
1764
|
+
const client2 = getS3Client(config);
|
|
1765
|
+
const bucket = config.fileUploads.s3Bucket;
|
|
1766
|
+
const fullPrefix = addGeneralPrefixToKey(prefix, config);
|
|
1767
|
+
const results = [];
|
|
1768
|
+
let continuationToken;
|
|
1769
|
+
do {
|
|
1770
|
+
const command = new import_client_s32.ListObjectsV2Command({
|
|
1771
|
+
Bucket: bucket,
|
|
1772
|
+
Prefix: fullPrefix,
|
|
1773
|
+
...continuationToken && { ContinuationToken: continuationToken }
|
|
1774
|
+
});
|
|
1775
|
+
const response = await client2.send(command);
|
|
1776
|
+
for (const obj of response.Contents ?? []) {
|
|
1777
|
+
if (obj.Key && obj.Size !== void 0 && obj.LastModified) {
|
|
1778
|
+
results.push({
|
|
1779
|
+
key: obj.Key,
|
|
1780
|
+
size: obj.Size,
|
|
1781
|
+
lastModified: obj.LastModified
|
|
1782
|
+
});
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
continuationToken = response.IsTruncated ? response.NextContinuationToken : void 0;
|
|
1786
|
+
} while (continuationToken);
|
|
1787
|
+
return results;
|
|
1788
|
+
};
|
|
1789
|
+
var copyS3Object = async (sourceKey, destKey, config) => {
|
|
1790
|
+
if (!config.fileUploads) {
|
|
1791
|
+
throw new Error("File uploads are not configured");
|
|
1792
|
+
}
|
|
1793
|
+
const client2 = getS3Client(config);
|
|
1794
|
+
const bucket = config.fileUploads.s3Bucket;
|
|
1795
|
+
const command = new import_client_s32.CopyObjectCommand({
|
|
1796
|
+
Bucket: bucket,
|
|
1797
|
+
CopySource: `${bucket}/${sourceKey}`,
|
|
1798
|
+
Key: destKey
|
|
1799
|
+
});
|
|
1800
|
+
await client2.send(command);
|
|
1801
|
+
};
|
|
1802
|
+
var getS3ObjectContent = async (key, config) => {
|
|
1803
|
+
if (!config.fileUploads) {
|
|
1804
|
+
throw new Error("File uploads are not configured");
|
|
1805
|
+
}
|
|
1806
|
+
const client2 = getS3Client(config);
|
|
1807
|
+
const bucket = config.fileUploads.s3Bucket;
|
|
1808
|
+
const command = new import_client_s32.GetObjectCommand({ Bucket: bucket, Key: key });
|
|
1809
|
+
const response = await client2.send(command);
|
|
1810
|
+
if (!response.Body) {
|
|
1811
|
+
throw new Error(`Empty body for S3 key: ${key}`);
|
|
1812
|
+
}
|
|
1813
|
+
return response.Body.transformToString("utf-8");
|
|
1814
|
+
};
|
|
1815
|
+
var deleteS3Object = async (key, config) => {
|
|
1816
|
+
if (!config.fileUploads) {
|
|
1817
|
+
throw new Error("File uploads are not configured");
|
|
1818
|
+
}
|
|
1819
|
+
const client2 = getS3Client(config);
|
|
1820
|
+
const bucket = config.fileUploads.s3Bucket;
|
|
1821
|
+
const command = new import_client_s32.DeleteObjectCommand({ Bucket: bucket, Key: key });
|
|
1822
|
+
await client2.send(command);
|
|
1823
|
+
};
|
|
1824
|
+
var getS3SignedUploadUrl = async (key, contentType, config) => {
|
|
1825
|
+
if (!config.fileUploads) {
|
|
1826
|
+
throw new Error("File uploads are not configured");
|
|
1827
|
+
}
|
|
1828
|
+
const client2 = getS3Client(config);
|
|
1829
|
+
const bucket = config.fileUploads.s3Bucket;
|
|
1830
|
+
const url = await (0, import_s3_request_presigner.getSignedUrl)(
|
|
1831
|
+
client2,
|
|
1832
|
+
new import_client_s32.PutObjectCommand({
|
|
1833
|
+
Bucket: bucket,
|
|
1834
|
+
Key: key,
|
|
1835
|
+
ContentType: contentType
|
|
1836
|
+
}),
|
|
1837
|
+
{ expiresIn }
|
|
1838
|
+
);
|
|
1839
|
+
return url;
|
|
1840
|
+
};
|
|
1740
1841
|
var createUppyRoutes = async (app, config) => {
|
|
1741
1842
|
if (!config.fileUploads) {
|
|
1742
1843
|
throw new Error("File uploads are not configured");
|
|
@@ -2995,6 +3096,53 @@ var agentSessionsSchema = {
|
|
|
2995
3096
|
}
|
|
2996
3097
|
]
|
|
2997
3098
|
};
|
|
3099
|
+
var skillsSchema = {
|
|
3100
|
+
type: "skills",
|
|
3101
|
+
name: {
|
|
3102
|
+
plural: "skills",
|
|
3103
|
+
singular: "skill"
|
|
3104
|
+
},
|
|
3105
|
+
RBAC: true,
|
|
3106
|
+
fields: [
|
|
3107
|
+
{
|
|
3108
|
+
name: "name",
|
|
3109
|
+
type: "text",
|
|
3110
|
+
index: true,
|
|
3111
|
+
unique: true
|
|
3112
|
+
},
|
|
3113
|
+
{
|
|
3114
|
+
name: "description",
|
|
3115
|
+
type: "text"
|
|
3116
|
+
},
|
|
3117
|
+
{
|
|
3118
|
+
name: "s3folder",
|
|
3119
|
+
type: "text"
|
|
3120
|
+
},
|
|
3121
|
+
{
|
|
3122
|
+
name: "tags",
|
|
3123
|
+
type: "json"
|
|
3124
|
+
},
|
|
3125
|
+
{
|
|
3126
|
+
name: "usage_count",
|
|
3127
|
+
type: "number",
|
|
3128
|
+
default: 0
|
|
3129
|
+
},
|
|
3130
|
+
{
|
|
3131
|
+
name: "favorite_count",
|
|
3132
|
+
type: "number",
|
|
3133
|
+
default: 0
|
|
3134
|
+
},
|
|
3135
|
+
{
|
|
3136
|
+
name: "history",
|
|
3137
|
+
type: "json"
|
|
3138
|
+
},
|
|
3139
|
+
{
|
|
3140
|
+
name: "current_version",
|
|
3141
|
+
type: "number",
|
|
3142
|
+
default: 1
|
|
3143
|
+
}
|
|
3144
|
+
]
|
|
3145
|
+
};
|
|
2998
3146
|
var variablesSchema = {
|
|
2999
3147
|
type: "variables",
|
|
3000
3148
|
name: {
|
|
@@ -3113,6 +3261,10 @@ var agentsSchema = {
|
|
|
3113
3261
|
name: "tools",
|
|
3114
3262
|
type: "json"
|
|
3115
3263
|
},
|
|
3264
|
+
{
|
|
3265
|
+
name: "skills",
|
|
3266
|
+
type: "json"
|
|
3267
|
+
},
|
|
3116
3268
|
{
|
|
3117
3269
|
name: "animation_idle",
|
|
3118
3270
|
type: "text"
|
|
@@ -3372,6 +3524,7 @@ var coreSchemas = {
|
|
|
3372
3524
|
agentSessionsSchema: () => addCoreFields(agentSessionsSchema),
|
|
3373
3525
|
projectsSchema: () => addCoreFields(projectsSchema),
|
|
3374
3526
|
usersSchema: () => addCoreFields(usersSchema),
|
|
3527
|
+
skillsSchema: () => addCoreFields(skillsSchema),
|
|
3375
3528
|
statisticsSchema: () => addCoreFields(statisticsSchema),
|
|
3376
3529
|
variablesSchema: () => addCoreFields(variablesSchema),
|
|
3377
3530
|
platformConfigurationsSchema: () => addCoreFields(platformConfigurationsSchema),
|
|
@@ -3993,9 +4146,6 @@ var mapType = (t, type, name, defaultValue, unique) => {
|
|
|
3993
4146
|
throw new Error("Invalid field type for database: " + type);
|
|
3994
4147
|
};
|
|
3995
4148
|
|
|
3996
|
-
// src/exulu/context.ts
|
|
3997
|
-
var import_zod5 = require("zod");
|
|
3998
|
-
|
|
3999
4149
|
// ee/queues/decorator.ts
|
|
4000
4150
|
var import_bullmq2 = require("bullmq");
|
|
4001
4151
|
var import_uuid = require("uuid");
|
|
@@ -4712,69 +4862,6 @@ var ExuluContext2 = class {
|
|
|
4712
4862
|
`);
|
|
4713
4863
|
return;
|
|
4714
4864
|
};
|
|
4715
|
-
// Exports the context as a tool that can be used by an agent
|
|
4716
|
-
tool = () => {
|
|
4717
|
-
if (this.configuration.enableAsTool === false) {
|
|
4718
|
-
return null;
|
|
4719
|
-
}
|
|
4720
|
-
return new ExuluTool({
|
|
4721
|
-
id: this.id,
|
|
4722
|
-
name: `${this.name}_context_search`,
|
|
4723
|
-
type: "context",
|
|
4724
|
-
category: "contexts",
|
|
4725
|
-
needsApproval: true,
|
|
4726
|
-
// todo make configurable
|
|
4727
|
-
inputSchema: import_zod5.z.object({
|
|
4728
|
-
query: import_zod5.z.string().describe("The original question that the user asked"),
|
|
4729
|
-
keywords: import_zod5.z.array(import_zod5.z.string()).describe(
|
|
4730
|
-
"The keywords that are relevant to the user's question, for example names of specific products, systems or parts, IDs, etc."
|
|
4731
|
-
),
|
|
4732
|
-
method: import_zod5.z.enum(["keyword", "semantic", "hybrid"]).default("hybrid").describe(
|
|
4733
|
-
"Search method: 'hybrid' (best for most queries - combines semantic understanding with exact term matching), 'keyword' (best for exact terms, technical names, IDs, or specific phrases), 'semantic' (best for conceptual queries where synonyms and paraphrasing matter)"
|
|
4734
|
-
)
|
|
4735
|
-
}),
|
|
4736
|
-
config: [],
|
|
4737
|
-
description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
|
|
4738
|
-
execute: async ({ query, keywords, user, role, method }) => {
|
|
4739
|
-
const { db: db2 } = await postgresClient();
|
|
4740
|
-
const result = await vectorSearch({
|
|
4741
|
-
page: 1,
|
|
4742
|
-
limit: this.configuration.maxRetrievalResults ?? 10,
|
|
4743
|
-
query,
|
|
4744
|
-
keywords,
|
|
4745
|
-
itemFilters: [],
|
|
4746
|
-
chunkFilters: [],
|
|
4747
|
-
user,
|
|
4748
|
-
role,
|
|
4749
|
-
method: method === "hybrid" ? "hybridSearch" : method === "keyword" ? "tsvector" : "cosineDistance",
|
|
4750
|
-
context: this,
|
|
4751
|
-
db: db2,
|
|
4752
|
-
sort: void 0,
|
|
4753
|
-
trigger: "agent"
|
|
4754
|
-
});
|
|
4755
|
-
await updateStatistic({
|
|
4756
|
-
name: "count",
|
|
4757
|
-
label: this.name,
|
|
4758
|
-
type: STATISTICS_TYPE_ENUM.TOOL_CALL,
|
|
4759
|
-
trigger: "tool",
|
|
4760
|
-
count: 1,
|
|
4761
|
-
user: user?.id,
|
|
4762
|
-
role: user?.role?.id
|
|
4763
|
-
});
|
|
4764
|
-
return {
|
|
4765
|
-
result: JSON.stringify(
|
|
4766
|
-
result.chunks.map((chunk) => ({
|
|
4767
|
-
...chunk,
|
|
4768
|
-
context: {
|
|
4769
|
-
name: this.name,
|
|
4770
|
-
id: this.id
|
|
4771
|
-
}
|
|
4772
|
-
}))
|
|
4773
|
-
)
|
|
4774
|
-
};
|
|
4775
|
-
}
|
|
4776
|
-
});
|
|
4777
|
-
};
|
|
4778
4865
|
};
|
|
4779
4866
|
|
|
4780
4867
|
// ee/agentic-retrieval/v3/context-sampler.ts
|
|
@@ -4819,7 +4906,27 @@ var ContextSampler = class {
|
|
|
4819
4906
|
|
|
4820
4907
|
// ee/agentic-retrieval/v3/classifier.ts
|
|
4821
4908
|
var import_ai2 = require("ai");
|
|
4822
|
-
var
|
|
4909
|
+
var import_zod5 = require("zod");
|
|
4910
|
+
|
|
4911
|
+
// src/utils/with-retry.ts
|
|
4912
|
+
async function withRetry(generateFn, maxRetries = 3) {
|
|
4913
|
+
let lastError;
|
|
4914
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
4915
|
+
try {
|
|
4916
|
+
return await generateFn();
|
|
4917
|
+
} catch (error) {
|
|
4918
|
+
lastError = error;
|
|
4919
|
+
console.error(`[EXULU] generateText attempt ${attempt} failed:`, error);
|
|
4920
|
+
if (attempt === maxRetries) {
|
|
4921
|
+
throw error;
|
|
4922
|
+
}
|
|
4923
|
+
await new Promise((resolve3) => setTimeout(resolve3, Math.pow(2, attempt) * 1e3));
|
|
4924
|
+
}
|
|
4925
|
+
}
|
|
4926
|
+
throw lastError;
|
|
4927
|
+
}
|
|
4928
|
+
|
|
4929
|
+
// ee/agentic-retrieval/v3/classifier.ts
|
|
4823
4930
|
async function classifyQuery(query, contexts, samples, model) {
|
|
4824
4931
|
const contextDescriptions = contexts.map((ctx) => {
|
|
4825
4932
|
const sample = samples.find((s) => s.contextId === ctx.id);
|
|
@@ -4830,46 +4937,49 @@ async function classifyQuery(query, contexts, samples, model) {
|
|
|
4830
4937
|
Description: ${ctx.description}
|
|
4831
4938
|
Fields: ${fieldList}${exampleStr}`;
|
|
4832
4939
|
}).join("\n\n");
|
|
4833
|
-
const result = await (
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
|
|
4838
|
-
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
|
|
4842
|
-
|
|
4843
|
-
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
|
|
4850
|
-
|
|
4851
|
-
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
-
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
|
|
4860
|
-
|
|
4861
|
-
|
|
4862
|
-
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4865
|
-
|
|
4940
|
+
const result = await withRetry(async () => {
|
|
4941
|
+
const result2 = await (0, import_ai2.generateText)({
|
|
4942
|
+
model,
|
|
4943
|
+
temperature: 0,
|
|
4944
|
+
output: import_ai2.Output.object({
|
|
4945
|
+
schema: import_zod5.z.object({
|
|
4946
|
+
queryType: import_zod5.z.enum(["aggregate", "list", "targeted", "exploratory"]).describe(
|
|
4947
|
+
"aggregate: ONLY use when the user explicitly asks to COUNT how many documents/items/tickets exist in the knowledge base (e.g. 'how many documents about X?', 'total number of tickets'). NEVER use for: real-world statistics stored in a document, intent statements, how-to questions, error/fault descriptions, configuration questions, or any query that does not explicitly ask for a count of knowledge base entries. When in doubt, choose targeted. list: user wants to enumerate matching items/documents (show me all, list documents about). targeted: use for almost everything \u2014 specific fact, answer, configuration, how-to, error/fault, feature/behavior question. Also use for intent statements and short commands describing a desired state (phrases that state what the user wants to do or achieve, even without an explicit question word). Real-world statistics stored in documents also go here. When in doubt, choose targeted over aggregate or exploratory. exploratory: only for broad conceptual questions needing multi-source synthesis (what is the process for Z, explain how X works, general overview of topic Y)."
|
|
4948
|
+
),
|
|
4949
|
+
language: import_zod5.z.string().describe("ISO 639-3 language code of the query (e.g. eng, deu, fra)"),
|
|
4950
|
+
suggestedContextIds: import_zod5.z.array(import_zod5.z.enum(contexts.map((c) => c.id))).describe(
|
|
4951
|
+
"IDs of knowledge bases most likely to contain the answer. Return empty array to search all contexts."
|
|
4952
|
+
)
|
|
4953
|
+
})
|
|
4954
|
+
}),
|
|
4955
|
+
toolChoice: "none",
|
|
4956
|
+
system: `You are a query classifier for a multi-knowledge-base retrieval system.
|
|
4957
|
+
Classify the query and identify which knowledge bases are most relevant.
|
|
4958
|
+
|
|
4959
|
+
Available knowledge bases:
|
|
4960
|
+
${contextDescriptions}
|
|
4961
|
+
|
|
4962
|
+
Guidelines for queryType:
|
|
4963
|
+
- Use "aggregate" ONLY when the query contains explicit counting language (e.g., "how many", "count", "total number", "wie viele"). Short statements, commands, or phrases without a question word are NEVER aggregate \u2014 classify them as targeted.
|
|
4964
|
+
- When in doubt between aggregate and targeted: always choose targeted.
|
|
4965
|
+
|
|
4966
|
+
Guidelines for suggestedContextIds:
|
|
4967
|
+
- Be conservative: only suggest contexts that are genuinely likely to contain the answer.
|
|
4968
|
+
Aim for 2\u20133 focused suggestions rather than listing everything.
|
|
4969
|
+
- Use each knowledge base's name and description (shown above) to judge relevance.
|
|
4970
|
+
- Return an empty array only if you truly cannot determine which contexts are relevant.`,
|
|
4971
|
+
prompt: `Query: ${query}`
|
|
4972
|
+
});
|
|
4973
|
+
return result2.output;
|
|
4974
|
+
}, 3);
|
|
4975
|
+
return result;
|
|
4866
4976
|
}
|
|
4867
4977
|
|
|
4868
4978
|
// ee/agentic-retrieval/v3/tools.ts
|
|
4869
|
-
var
|
|
4979
|
+
var import_zod6 = require("zod");
|
|
4870
4980
|
var import_ai3 = require("ai");
|
|
4871
4981
|
function buildContextEnum(contexts) {
|
|
4872
|
-
return
|
|
4982
|
+
return import_zod6.z.array(import_zod6.z.enum(contexts.map((c) => c.id))).describe(
|
|
4873
4983
|
contexts.map(
|
|
4874
4984
|
(c) => `<knowledge_base id="${c.id}" name="${c.name}">${c.description}</knowledge_base>`
|
|
4875
4985
|
).join("\n")
|
|
@@ -4890,16 +5000,34 @@ function mapSearchMethod(method) {
|
|
|
4890
5000
|
if (method === "keyword") return "tsvector";
|
|
4891
5001
|
return "cosineDistance";
|
|
4892
5002
|
}
|
|
5003
|
+
function parseGlobalItemIds(globalIds) {
|
|
5004
|
+
const map = /* @__PURE__ */ new Map();
|
|
5005
|
+
for (const gid of globalIds) {
|
|
5006
|
+
const slashIdx = gid.indexOf("/");
|
|
5007
|
+
if (slashIdx === -1) {
|
|
5008
|
+
if (gid) map.set(gid, null);
|
|
5009
|
+
continue;
|
|
5010
|
+
}
|
|
5011
|
+
const contextId = gid.slice(0, slashIdx);
|
|
5012
|
+
const itemId = gid.slice(slashIdx + 1);
|
|
5013
|
+
if (!contextId || !itemId) continue;
|
|
5014
|
+
if (map.get(contextId) === null) continue;
|
|
5015
|
+
const existing = map.get(contextId) ?? [];
|
|
5016
|
+
existing.push(itemId);
|
|
5017
|
+
map.set(contextId, existing);
|
|
5018
|
+
}
|
|
5019
|
+
return map;
|
|
5020
|
+
}
|
|
4893
5021
|
function createRetrievalTools(params) {
|
|
4894
|
-
const { contexts, user, role, updateVirtualFiles } = params;
|
|
5022
|
+
const { contexts, user, role, updateVirtualFiles, preselectedItemsByContext } = params;
|
|
4895
5023
|
const ctxEnum = buildContextEnum(contexts);
|
|
4896
5024
|
const count_items_or_chunks = (0, import_ai3.tool)({
|
|
4897
5025
|
description: "Count items or chunks WITHOUT loading them into context. Use for 'how many', 'count', or 'total number of' queries.",
|
|
4898
|
-
inputSchema:
|
|
5026
|
+
inputSchema: import_zod6.z.object({
|
|
4899
5027
|
knowledge_base_ids: ctxEnum,
|
|
4900
|
-
count_what:
|
|
4901
|
-
name_contains:
|
|
4902
|
-
content_query:
|
|
5028
|
+
count_what: import_zod6.z.enum(["items", "chunks"]).describe("Whether to count items (documents) or chunks (pages/sections)"),
|
|
5029
|
+
name_contains: import_zod6.z.string().optional().describe("Only count items whose name contains this text (case-insensitive)"),
|
|
5030
|
+
content_query: import_zod6.z.string().optional().describe(
|
|
4903
5031
|
"Only count chunks matching this search query (uses hybrid search). Only used when count_what is 'chunks'."
|
|
4904
5032
|
)
|
|
4905
5033
|
}),
|
|
@@ -4908,6 +5036,10 @@ function createRetrievalTools(params) {
|
|
|
4908
5036
|
const ctxList = resolveContexts(knowledge_base_ids, contexts);
|
|
4909
5037
|
const counts = await Promise.all(
|
|
4910
5038
|
ctxList.map(async (ctx) => {
|
|
5039
|
+
const contextItemIds = preselectedItemsByContext?.get(ctx.id);
|
|
5040
|
+
if (preselectedItemsByContext && contextItemIds === void 0) {
|
|
5041
|
+
return { context: ctx.id, context_name: ctx.name, count: 0 };
|
|
5042
|
+
}
|
|
4911
5043
|
let count = 0;
|
|
4912
5044
|
if (count_what === "items") {
|
|
4913
5045
|
const tableName = getTableName(ctx.id);
|
|
@@ -4915,19 +5047,23 @@ function createRetrievalTools(params) {
|
|
|
4915
5047
|
if (name_contains) {
|
|
4916
5048
|
q = q.whereRaw("LOWER(name) LIKE ?", [`%${name_contains.toLowerCase()}%`]);
|
|
4917
5049
|
}
|
|
5050
|
+
if (Array.isArray(contextItemIds)) {
|
|
5051
|
+
q = q.whereIn("id", contextItemIds);
|
|
5052
|
+
}
|
|
4918
5053
|
const tableDefinition = convertContextToTableDefinition(ctx);
|
|
4919
5054
|
q = applyAccessControl(tableDefinition, q, user, tableName);
|
|
4920
5055
|
const result = await q.first();
|
|
4921
5056
|
count = Number(result?.count ?? 0);
|
|
4922
5057
|
} else {
|
|
4923
5058
|
const chunksTable = getChunksTableName(ctx.id);
|
|
5059
|
+
const baseItemFilters = Array.isArray(contextItemIds) ? [{ id: { in: contextItemIds } }] : [];
|
|
4924
5060
|
if (content_query) {
|
|
4925
5061
|
const searchResults = await ctx.search({
|
|
4926
5062
|
query: content_query,
|
|
4927
5063
|
method: "hybridSearch",
|
|
4928
5064
|
limit: 1e4,
|
|
4929
5065
|
page: 1,
|
|
4930
|
-
itemFilters:
|
|
5066
|
+
itemFilters: baseItemFilters,
|
|
4931
5067
|
chunkFilters: [],
|
|
4932
5068
|
sort: { field: "updatedAt", direction: "desc" },
|
|
4933
5069
|
user,
|
|
@@ -4935,6 +5071,9 @@ function createRetrievalTools(params) {
|
|
|
4935
5071
|
trigger: "tool"
|
|
4936
5072
|
});
|
|
4937
5073
|
count = searchResults.chunks.length;
|
|
5074
|
+
} else if (Array.isArray(contextItemIds)) {
|
|
5075
|
+
const result = await db2(chunksTable).count("id as count").whereIn("source", contextItemIds).first();
|
|
5076
|
+
count = Number(result?.count ?? 0);
|
|
4938
5077
|
} else {
|
|
4939
5078
|
const result = await db2(chunksTable).count("id as count").first();
|
|
4940
5079
|
count = Number(result?.count ?? 0);
|
|
@@ -4950,11 +5089,13 @@ function createRetrievalTools(params) {
|
|
|
4950
5089
|
}
|
|
4951
5090
|
});
|
|
4952
5091
|
const search_items_by_name = (0, import_ai3.tool)({
|
|
4953
|
-
description: "Search for items by their name or external ID. Use
|
|
4954
|
-
inputSchema:
|
|
5092
|
+
description: "Search for items by their name or external ID. Use when:\n\u2022 The user asks for a document BY TITLE or NAME\n\u2022 The user asks whether a specific named document EXISTS (e.g. 'do you have the X manual?', 'is there a document for Y?')\n\u2022 Any query that references a specific document, manual, or resource by its name rather than by topic\nDo NOT use for topic-based content queries (e.g. 'what are the parameters for X?', 'how do I configure Y?').",
|
|
5093
|
+
inputSchema: import_zod6.z.object({
|
|
4955
5094
|
knowledge_base_ids: ctxEnum,
|
|
4956
|
-
item_name:
|
|
4957
|
-
|
|
5095
|
+
item_name: import_zod6.z.string().describe(
|
|
5096
|
+
"The name or partial name to search for. Uses substring matching, so shorter and more specific terms work better than full phrases. Extract only the core identifying part \u2014 typically the product model, document title, or unique identifier. Do NOT include surrounding descriptors like type words ('manual', 'guide', 'document') or manufacturer names unless they are likely part of the actual document title."
|
|
5097
|
+
),
|
|
5098
|
+
limit: import_zod6.z.number().default(100).describe(
|
|
4958
5099
|
"Max items per knowledge base (max 400). Applies independently to each knowledge base."
|
|
4959
5100
|
)
|
|
4960
5101
|
}),
|
|
@@ -4962,9 +5103,12 @@ function createRetrievalTools(params) {
|
|
|
4962
5103
|
const { db: db2 } = await postgresClient();
|
|
4963
5104
|
const ctxList = resolveContexts(knowledge_base_ids, contexts);
|
|
4964
5105
|
const safeLimit = Math.min(limit ?? 100, 400);
|
|
4965
|
-
const itemFilters = item_name ? [{ name: { contains: item_name } }] : [];
|
|
4966
5106
|
const results = await Promise.all(
|
|
4967
5107
|
ctxList.map(async (ctx) => {
|
|
5108
|
+
const contextItemIds = preselectedItemsByContext?.get(ctx.id);
|
|
5109
|
+
if (preselectedItemsByContext && contextItemIds === void 0) return [];
|
|
5110
|
+
const itemFilters = item_name ? [{ name: { contains: item_name } }] : [];
|
|
5111
|
+
if (Array.isArray(contextItemIds)) itemFilters.push({ id: { in: contextItemIds } });
|
|
4968
5112
|
const tableName = getTableName(ctx.id);
|
|
4969
5113
|
const tableDefinition = convertContextToTableDefinition(ctx);
|
|
4970
5114
|
let q = db2(`${tableName} as items`).select([
|
|
@@ -5000,30 +5144,35 @@ function createRetrievalTools(params) {
|
|
|
5000
5144
|
}
|
|
5001
5145
|
});
|
|
5002
5146
|
const search_content = (0, import_ai3.tool)({
|
|
5003
|
-
description: `Search
|
|
5147
|
+
description: `Search ONE knowledge base for document content using hybrid, keyword, or semantic search.
|
|
5148
|
+
Always make a separate call for each knowledge base you want to search \u2014 never bundle multiple in one call.
|
|
5004
5149
|
|
|
5005
5150
|
Use includeContent: false when you only need to know WHICH documents match (listing, overview, navigation).
|
|
5006
5151
|
Use includeContent: true when you need the ACTUAL text to answer a question.
|
|
5007
5152
|
|
|
5008
5153
|
For listing queries: always start with includeContent: false, then use dynamic tools to fetch specific pages.`,
|
|
5009
|
-
inputSchema:
|
|
5010
|
-
query:
|
|
5011
|
-
|
|
5012
|
-
|
|
5013
|
-
|
|
5154
|
+
inputSchema: import_zod6.z.object({
|
|
5155
|
+
query: import_zod6.z.string().describe("Search query about the content you're looking for"),
|
|
5156
|
+
knowledge_base_id: import_zod6.z.enum(contexts.map((c) => c.id)).describe(
|
|
5157
|
+
contexts.map(
|
|
5158
|
+
(c) => `<knowledge_base id="${c.id}" name="${c.name}">${c.description}</knowledge_base>`
|
|
5159
|
+
).join("\n")
|
|
5160
|
+
),
|
|
5161
|
+
keywords: import_zod6.z.array(import_zod6.z.string()).optional().describe("Keywords extracted from the query"),
|
|
5162
|
+
searchMethod: import_zod6.z.enum(["hybrid", "keyword", "semantic"]).default("hybrid").describe(
|
|
5014
5163
|
"hybrid: best default (semantic + keyword). keyword: exact terms, product codes, IDs. semantic: conceptual/synonyms."
|
|
5015
5164
|
),
|
|
5016
|
-
includeContent:
|
|
5165
|
+
includeContent: import_zod6.z.boolean().default(true).describe(
|
|
5017
5166
|
"false: returns metadata only (document names, scores) \u2014 use for listing/navigation. true: returns full chunk text \u2014 use when you need content to answer a question."
|
|
5018
5167
|
),
|
|
5019
|
-
item_ids:
|
|
5020
|
-
item_names:
|
|
5021
|
-
item_external_ids:
|
|
5022
|
-
limit:
|
|
5168
|
+
item_ids: import_zod6.z.array(import_zod6.z.string()).optional().describe("Filter results to specific item IDs"),
|
|
5169
|
+
item_names: import_zod6.z.array(import_zod6.z.string()).optional().describe("Filter results to items whose name contains one of these strings"),
|
|
5170
|
+
item_external_ids: import_zod6.z.array(import_zod6.z.string()).optional().describe("Filter results to specific external IDs"),
|
|
5171
|
+
limit: import_zod6.z.number().default(20).describe("Max chunks with content (max 20). Without content, up to 200 are returned.")
|
|
5023
5172
|
}),
|
|
5024
5173
|
execute: async ({
|
|
5025
5174
|
query,
|
|
5026
|
-
|
|
5175
|
+
knowledge_base_id,
|
|
5027
5176
|
keywords,
|
|
5028
5177
|
searchMethod,
|
|
5029
5178
|
includeContent,
|
|
@@ -5032,67 +5181,81 @@ For listing queries: always start with includeContent: false, then use dynamic t
|
|
|
5032
5181
|
item_external_ids,
|
|
5033
5182
|
limit
|
|
5034
5183
|
}) => {
|
|
5035
|
-
const
|
|
5036
|
-
const effectiveLimit = includeContent ? Math.min(limit ??
|
|
5037
|
-
const
|
|
5038
|
-
|
|
5039
|
-
|
|
5040
|
-
|
|
5041
|
-
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
const
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
if (!ctx.embedder) {
|
|
5048
|
-
console.error(`[EXULU] context "${ctx.id}" does not have an embedder, falling back to tsvector search`);
|
|
5049
|
-
method = "tsvector";
|
|
5050
|
-
}
|
|
5051
|
-
}
|
|
5052
|
-
try {
|
|
5053
|
-
const { chunks } = await ctx.search({
|
|
5054
|
-
query: effectiveQuery,
|
|
5055
|
-
keywords,
|
|
5056
|
-
method,
|
|
5057
|
-
limit: effectiveLimit,
|
|
5058
|
-
page: 1,
|
|
5059
|
-
itemFilters,
|
|
5060
|
-
chunkFilters: [],
|
|
5061
|
-
sort: { field: "updatedAt", direction: "desc" },
|
|
5062
|
-
user,
|
|
5063
|
-
role,
|
|
5064
|
-
trigger: "tool"
|
|
5065
|
-
});
|
|
5066
|
-
return chunks.map(
|
|
5067
|
-
(chunk) => ({
|
|
5068
|
-
item_name: chunk.item_name,
|
|
5069
|
-
item_id: chunk.item_id,
|
|
5070
|
-
context: chunk.context?.id ?? ctx.id,
|
|
5071
|
-
chunk_id: chunk.chunk_id,
|
|
5072
|
-
chunk_index: chunk.chunk_index,
|
|
5073
|
-
chunk_content: includeContent ? chunk.chunk_content : void 0,
|
|
5074
|
-
metadata: {
|
|
5075
|
-
...chunk.chunk_metadata,
|
|
5076
|
-
cosine_distance: chunk.chunk_cosine_distance,
|
|
5077
|
-
fts_rank: chunk.chunk_fts_rank,
|
|
5078
|
-
hybrid_score: chunk.chunk_hybrid_score
|
|
5079
|
-
}
|
|
5080
|
-
})
|
|
5081
|
-
);
|
|
5082
|
-
} catch (err) {
|
|
5083
|
-
console.error(`[EXULU] search_content failed for context "${ctx.id}":`, err);
|
|
5084
|
-
return [];
|
|
5184
|
+
const [ctx] = resolveContexts([knowledge_base_id], contexts);
|
|
5185
|
+
const effectiveLimit = includeContent ? Math.min(limit ?? 20, 20) : Math.min((limit ?? 20) * 20, 400);
|
|
5186
|
+
const itemFilters = [];
|
|
5187
|
+
if (preselectedItemsByContext) {
|
|
5188
|
+
const contextItemIds = preselectedItemsByContext.get(knowledge_base_id);
|
|
5189
|
+
if (contextItemIds === void 0) {
|
|
5190
|
+
return JSON.stringify([]);
|
|
5191
|
+
}
|
|
5192
|
+
if (Array.isArray(contextItemIds)) {
|
|
5193
|
+
const intersection = item_ids?.length ? item_ids.filter((id) => contextItemIds.includes(id)) : contextItemIds;
|
|
5194
|
+
if (!intersection.length) {
|
|
5195
|
+
return JSON.stringify([]);
|
|
5085
5196
|
}
|
|
5086
|
-
|
|
5087
|
-
|
|
5088
|
-
|
|
5197
|
+
itemFilters.push({ id: { in: intersection } });
|
|
5198
|
+
} else if (item_ids?.length) {
|
|
5199
|
+
itemFilters.push({ id: { in: item_ids } });
|
|
5200
|
+
}
|
|
5201
|
+
} else if (item_ids?.length) {
|
|
5202
|
+
itemFilters.push({ id: { in: item_ids } });
|
|
5203
|
+
}
|
|
5204
|
+
if (item_names)
|
|
5205
|
+
itemFilters.push({ name: { or: item_names.map((n) => ({ contains: n })) } });
|
|
5206
|
+
if (item_external_ids) itemFilters.push({ external_id: { in: item_external_ids } });
|
|
5207
|
+
const effectiveQuery = query || keywords?.join(" ") || "";
|
|
5208
|
+
let method = mapSearchMethod(searchMethod ?? "hybrid");
|
|
5209
|
+
if (method === "hybridSearch" || method === "cosineDistance") {
|
|
5210
|
+
if (!ctx.embedder) {
|
|
5211
|
+
console.error(`[EXULU] context "${ctx.id}" does not have an embedder, falling back to tsvector search`);
|
|
5212
|
+
method = "tsvector";
|
|
5213
|
+
}
|
|
5214
|
+
}
|
|
5215
|
+
try {
|
|
5216
|
+
const { chunks } = await ctx.search({
|
|
5217
|
+
query: effectiveQuery,
|
|
5218
|
+
keywords,
|
|
5219
|
+
method,
|
|
5220
|
+
limit: effectiveLimit,
|
|
5221
|
+
page: 1,
|
|
5222
|
+
itemFilters,
|
|
5223
|
+
chunkFilters: [],
|
|
5224
|
+
sort: { field: "updatedAt", direction: "desc" },
|
|
5225
|
+
user,
|
|
5226
|
+
role,
|
|
5227
|
+
trigger: "tool"
|
|
5228
|
+
});
|
|
5229
|
+
return JSON.stringify(
|
|
5230
|
+
chunks.map(
|
|
5231
|
+
(chunk) => ({
|
|
5232
|
+
item_name: chunk.item_name,
|
|
5233
|
+
item_id: chunk.item_id,
|
|
5234
|
+
context: chunk.context?.id ?? ctx.id,
|
|
5235
|
+
chunk_id: chunk.chunk_id,
|
|
5236
|
+
chunk_index: chunk.chunk_index,
|
|
5237
|
+
chunk_content: includeContent ? chunk.chunk_content : void 0,
|
|
5238
|
+
metadata: {
|
|
5239
|
+
...chunk.chunk_metadata,
|
|
5240
|
+
cosine_distance: chunk.chunk_cosine_distance,
|
|
5241
|
+
fts_rank: chunk.chunk_fts_rank,
|
|
5242
|
+
hybrid_score: chunk.chunk_hybrid_score
|
|
5243
|
+
}
|
|
5244
|
+
})
|
|
5245
|
+
)
|
|
5246
|
+
);
|
|
5247
|
+
} catch (err) {
|
|
5248
|
+
console.error(`[EXULU] search_content failed for context "${ctx.id}":`, err);
|
|
5249
|
+
return JSON.stringify([]);
|
|
5250
|
+
}
|
|
5089
5251
|
}
|
|
5090
5252
|
});
|
|
5091
5253
|
const save_search_results = (0, import_ai3.tool)({
|
|
5092
|
-
description: `Execute a search and save ALL results to the virtual filesystem WITHOUT loading them into context.
|
|
5254
|
+
description: `Execute a search on ONE knowledge base and save ALL results to the virtual filesystem WITHOUT loading them into context.
|
|
5255
|
+
Always make a separate call for each knowledge base you want to search.
|
|
5093
5256
|
|
|
5094
5257
|
Use this when you expect many results (>20) and need to filter iteratively:
|
|
5095
|
-
1. Call save_search_results to save up to 1000 results to /
|
|
5258
|
+
1. Call save_search_results (once per knowledge base) to save up to 1000 results to /search_results_{knowledge_base_id}.txt
|
|
5096
5259
|
2. Use bash grep/awk to identify relevant chunks by pattern
|
|
5097
5260
|
3. Use dynamic get_content tools to load only the specific chunks you need
|
|
5098
5261
|
|
|
@@ -5107,40 +5270,49 @@ SCORE: ...
|
|
|
5107
5270
|
---CONTENT START---
|
|
5108
5271
|
(content or placeholder)
|
|
5109
5272
|
---CONTENT END---`,
|
|
5110
|
-
inputSchema:
|
|
5111
|
-
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5273
|
+
inputSchema: import_zod6.z.object({
|
|
5274
|
+
knowledge_base_id: import_zod6.z.enum(contexts.map((c) => c.id)).describe(
|
|
5275
|
+
contexts.map(
|
|
5276
|
+
(c) => `<knowledge_base id="${c.id}" name="${c.name}">${c.description}</knowledge_base>`
|
|
5277
|
+
).join("\n")
|
|
5278
|
+
),
|
|
5279
|
+
query: import_zod6.z.string().describe("Search query"),
|
|
5280
|
+
searchMethod: import_zod6.z.enum(["hybrid", "keyword", "semantic"]).default("hybrid"),
|
|
5281
|
+
limit: import_zod6.z.number().max(1e3).default(100).describe("Max results to save (max 1000)"),
|
|
5282
|
+
includeContent: import_zod6.z.boolean().default(true).describe(
|
|
5116
5283
|
"Whether to include chunk text in the saved file. False saves tokens \u2014 use true only if you need to grep content."
|
|
5117
5284
|
)
|
|
5118
5285
|
}),
|
|
5119
|
-
execute: async ({ query,
|
|
5120
|
-
const
|
|
5121
|
-
const
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5125
|
-
|
|
5126
|
-
|
|
5127
|
-
|
|
5128
|
-
|
|
5129
|
-
|
|
5130
|
-
|
|
5131
|
-
|
|
5132
|
-
|
|
5133
|
-
|
|
5134
|
-
|
|
5135
|
-
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5286
|
+
execute: async ({ query, knowledge_base_id, searchMethod, limit, includeContent }) => {
|
|
5287
|
+
const [ctx] = resolveContexts([knowledge_base_id], contexts);
|
|
5288
|
+
const contextItemIds = preselectedItemsByContext?.get(knowledge_base_id);
|
|
5289
|
+
if (preselectedItemsByContext && contextItemIds === void 0) {
|
|
5290
|
+
return JSON.stringify({
|
|
5291
|
+
success: true,
|
|
5292
|
+
results_count: 0,
|
|
5293
|
+
message: `Context "${knowledge_base_id}" not in preselected scope \u2014 skipped.`
|
|
5294
|
+
});
|
|
5295
|
+
}
|
|
5296
|
+
const itemFilters = Array.isArray(contextItemIds) ? [{ id: { in: contextItemIds } }] : [];
|
|
5297
|
+
let chunks = [];
|
|
5298
|
+
try {
|
|
5299
|
+
const result = await ctx.search({
|
|
5300
|
+
query,
|
|
5301
|
+
method: mapSearchMethod(searchMethod ?? "hybrid"),
|
|
5302
|
+
limit: Math.min(limit ?? 100, 1e3),
|
|
5303
|
+
page: 1,
|
|
5304
|
+
itemFilters,
|
|
5305
|
+
chunkFilters: [],
|
|
5306
|
+
sort: { field: "updatedAt", direction: "desc" },
|
|
5307
|
+
user,
|
|
5308
|
+
role,
|
|
5309
|
+
trigger: "tool"
|
|
5310
|
+
});
|
|
5311
|
+
chunks = result.chunks;
|
|
5312
|
+
} catch (err) {
|
|
5313
|
+
console.error(`[EXULU] save_search_results failed for context "${ctx.id}":`, err);
|
|
5314
|
+
}
|
|
5315
|
+
const fileName = `search_results_${ctx.id}.txt`;
|
|
5144
5316
|
const fileContent = chunks.map(
|
|
5145
5317
|
(chunk, i) => `### RESULT ${i + 1} ###
|
|
5146
5318
|
ITEM_NAME: ${chunk.item_name}
|
|
@@ -5155,14 +5327,14 @@ ${includeContent && chunk.chunk_content ? chunk.chunk_content : "[use includeCon
|
|
|
5155
5327
|
`
|
|
5156
5328
|
).join("\n");
|
|
5157
5329
|
await updateVirtualFiles([
|
|
5158
|
-
{ path:
|
|
5330
|
+
{ path: fileName, content: fileContent },
|
|
5159
5331
|
{
|
|
5160
|
-
path:
|
|
5332
|
+
path: `search_metadata_${ctx.id}.json`,
|
|
5161
5333
|
content: JSON.stringify({
|
|
5162
5334
|
query,
|
|
5163
5335
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5164
5336
|
results_count: chunks.length,
|
|
5165
|
-
|
|
5337
|
+
context: ctx.id,
|
|
5166
5338
|
method: searchMethod
|
|
5167
5339
|
})
|
|
5168
5340
|
}
|
|
@@ -5170,11 +5342,11 @@ ${includeContent && chunk.chunk_content ? chunk.chunk_content : "[use includeCon
|
|
|
5170
5342
|
return JSON.stringify({
|
|
5171
5343
|
success: true,
|
|
5172
5344
|
results_count: chunks.length,
|
|
5173
|
-
message: `Saved ${chunks.length} results to
|
|
5345
|
+
message: `Saved ${chunks.length} results to /${fileName}`,
|
|
5174
5346
|
grep_examples: [
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5347
|
+
`grep -i 'keyword' ${fileName} | head -20`,
|
|
5348
|
+
`grep 'ITEM_NAME:' ${fileName}`,
|
|
5349
|
+
`grep -B 5 'pattern' ${fileName} | grep 'CHUNK_ID:'`
|
|
5178
5350
|
]
|
|
5179
5351
|
});
|
|
5180
5352
|
}
|
|
@@ -5196,9 +5368,10 @@ another component will do that based on what you retrieve.
|
|
|
5196
5368
|
Always respond in the SAME LANGUAGE as the user's query.
|
|
5197
5369
|
Always write search queries in the SAME LANGUAGE as the user's query \u2014 do NOT translate to English.
|
|
5198
5370
|
|
|
5199
|
-
SEARCH APPROACH \u2014
|
|
5200
|
-
1.
|
|
5201
|
-
|
|
5371
|
+
SEARCH APPROACH \u2014 one knowledge base at a time, then go deep:
|
|
5372
|
+
1. search_content and save_search_results accept ONE knowledge base per call. Make a separate
|
|
5373
|
+
call for each knowledge base you need to cover \u2014 never skip one. Search all relevant
|
|
5374
|
+
knowledge bases before concluding, even if the first one already returned good results.
|
|
5202
5375
|
2. After finding a relevant document, use get_more_content_from_{item} dynamic tools to load
|
|
5203
5376
|
additional pages/sections. The specific answer is often NOT in the first retrieved chunk \u2014
|
|
5204
5377
|
always explore adjacent content before concluding.
|
|
@@ -5214,9 +5387,8 @@ var AGGREGATE_INSTRUCTIONS = `
|
|
|
5214
5387
|
${BASE_INSTRUCTIONS}
|
|
5215
5388
|
|
|
5216
5389
|
STRATEGY: This is a COUNTING or AGGREGATION query.
|
|
5217
|
-
- Use count_items_or_chunks exclusively
|
|
5390
|
+
- Use count_items_or_chunks exclusively \u2014 it accepts multiple knowledge bases in one call for efficiency
|
|
5218
5391
|
- Do NOT use search_content \u2014 it loads unnecessary data
|
|
5219
|
-
- Search ALL contexts in parallel in a single tool call
|
|
5220
5392
|
- Return immediately after counting \u2014 one step is sufficient
|
|
5221
5393
|
- If the count needs a content filter, use content_query parameter
|
|
5222
5394
|
`.trim();
|
|
@@ -5249,9 +5421,23 @@ Search language:
|
|
|
5249
5421
|
- Always write search queries in the SAME LANGUAGE as the user's query.
|
|
5250
5422
|
- Do NOT translate the query to English \u2014 the documents are indexed in their original language.
|
|
5251
5423
|
|
|
5252
|
-
Step 1 \u2014
|
|
5253
|
-
|
|
5254
|
-
|
|
5424
|
+
Step 1 \u2014 match the opening move to what the query actually needs:
|
|
5425
|
+
|
|
5426
|
+
Query references a SPECIFIC NAMED DOCUMENT (product manual, titled report, named file):
|
|
5427
|
+
\u2192 ALWAYS start with search_items_by_name \u2014 searches document name/title directly
|
|
5428
|
+
\u2192 Only proceed to load content if the document is found
|
|
5429
|
+
|
|
5430
|
+
Query asks WHETHER a topic EXISTS or WHAT documents cover a topic (no specific title given):
|
|
5431
|
+
\u2192 search_content with includeContent: false
|
|
5432
|
+
\u2192 Returns matching document names without loading chunk text \u2014 efficient and precise
|
|
5433
|
+
\u2192 Load content with dynamic get_{item}_page_{n}_content tools only if needed in step 2
|
|
5434
|
+
|
|
5435
|
+
Query asks for CONTENT itself (procedures, parameters, explanations, how-to):
|
|
5436
|
+
\u2192 search_content with includeContent: true, limit 20, searchMethod: "hybrid"
|
|
5437
|
+
\u2192 Make one call per knowledge base \u2014 search each separately before concluding
|
|
5438
|
+
|
|
5439
|
+
Query provides an EXACT TERM (error code, product code, ID, parameter name):
|
|
5440
|
+
\u2192 search_content with searchMethod: "keyword"
|
|
5255
5441
|
|
|
5256
5442
|
Step 2+ \u2014 depth and follow-up:
|
|
5257
5443
|
- For any relevant document found with fewer than 5 chunks, use get_more_content_from_{item}
|
|
@@ -5261,19 +5447,9 @@ Step 2+ \u2014 depth and follow-up:
|
|
|
5261
5447
|
- Try alternative phrasings if the first query doesn't surface the right answer.
|
|
5262
5448
|
|
|
5263
5449
|
Product-specific filtering:
|
|
5264
|
-
- When the query mentions a specific
|
|
5265
|
-
item_names: ["<
|
|
5450
|
+
- When the query mentions a specific named entity (product, model, version), you MAY use
|
|
5451
|
+
item_names: ["<entity>"] on a follow-up search to narrow results \u2014 but only after an initial
|
|
5266
5452
|
wide search. Never start with item_names filtering alone.
|
|
5267
|
-
|
|
5268
|
-
Two-step approach \u2014 use includeContent: false first:
|
|
5269
|
-
- Only when you expect many results (>20) and need to identify the right document first.
|
|
5270
|
-
- Step 1: search_content with includeContent: false \u2192 see which documents/chunks match.
|
|
5271
|
-
- Step 2: use dynamic get_{item}_page_{n}_content tools to load specific pages.
|
|
5272
|
-
|
|
5273
|
-
Search method selection:
|
|
5274
|
-
- hybrid (default): best for most queries
|
|
5275
|
-
- keyword: exact product codes, document IDs, error codes
|
|
5276
|
-
- semantic: conceptual questions, synonyms, paraphrasing
|
|
5277
5453
|
`.trim();
|
|
5278
5454
|
var EXPLORATORY_INSTRUCTIONS = `
|
|
5279
5455
|
${BASE_INSTRUCTIONS}
|
|
@@ -5281,13 +5457,13 @@ ${BASE_INSTRUCTIONS}
|
|
|
5281
5457
|
STRATEGY: This is an EXPLORATORY query \u2014 general question requiring broad search.
|
|
5282
5458
|
|
|
5283
5459
|
Recommended approach:
|
|
5284
|
-
1.
|
|
5460
|
+
1. Search each relevant knowledge base separately with hybrid search (includeContent: true, limit: 20) \u2014 one call per knowledge base
|
|
5285
5461
|
2. If results are insufficient: try alternative search terms or different search method
|
|
5286
5462
|
3. Use save_search_results + bash grep when you need to scan many results without context bloat
|
|
5287
5463
|
4. Use dynamic get_more_content_from_{item} tools to read adjacent pages when a relevant item is found
|
|
5288
5464
|
|
|
5289
5465
|
When to declare done:
|
|
5290
|
-
- You have retrieved chunks that cover the key aspects of the query
|
|
5466
|
+
- You have retrieved chunks that cover the key aspects of the query from all relevant knowledge bases
|
|
5291
5467
|
- OR you have tried 3+ different search strategies and found nothing relevant
|
|
5292
5468
|
|
|
5293
5469
|
Do NOT use count_items_or_chunks for exploratory queries \u2014 the user wants content, not statistics.
|
|
@@ -5302,7 +5478,7 @@ var STRATEGIES = {
|
|
|
5302
5478
|
},
|
|
5303
5479
|
list: {
|
|
5304
5480
|
queryType: "list",
|
|
5305
|
-
stepBudget:
|
|
5481
|
+
stepBudget: 3,
|
|
5306
5482
|
retrieval_tools: ["count_items_or_chunks", "search_items_by_name", "search_content"],
|
|
5307
5483
|
include_bash: false,
|
|
5308
5484
|
instructions: LIST_INSTRUCTIONS
|
|
@@ -5316,7 +5492,7 @@ var STRATEGIES = {
|
|
|
5316
5492
|
},
|
|
5317
5493
|
exploratory: {
|
|
5318
5494
|
queryType: "exploratory",
|
|
5319
|
-
stepBudget:
|
|
5495
|
+
stepBudget: 5,
|
|
5320
5496
|
retrieval_tools: [
|
|
5321
5497
|
"count_items_or_chunks",
|
|
5322
5498
|
"search_items_by_name",
|
|
@@ -5330,28 +5506,10 @@ var STRATEGIES = {
|
|
|
5330
5506
|
|
|
5331
5507
|
// ee/agentic-retrieval/v3/agent-loop.ts
|
|
5332
5508
|
var import_ai5 = require("ai");
|
|
5333
|
-
var
|
|
5334
|
-
|
|
5335
|
-
// src/utils/with-retry.ts
|
|
5336
|
-
async function withRetry(generateFn, maxRetries = 3) {
|
|
5337
|
-
let lastError;
|
|
5338
|
-
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
5339
|
-
try {
|
|
5340
|
-
return await generateFn();
|
|
5341
|
-
} catch (error) {
|
|
5342
|
-
lastError = error;
|
|
5343
|
-
console.error(`[EXULU] generateText attempt ${attempt} failed:`, error);
|
|
5344
|
-
if (attempt === maxRetries) {
|
|
5345
|
-
throw error;
|
|
5346
|
-
}
|
|
5347
|
-
await new Promise((resolve4) => setTimeout(resolve4, Math.pow(2, attempt) * 1e3));
|
|
5348
|
-
}
|
|
5349
|
-
}
|
|
5350
|
-
throw lastError;
|
|
5351
|
-
}
|
|
5509
|
+
var import_zod8 = require("zod");
|
|
5352
5510
|
|
|
5353
5511
|
// ee/agentic-retrieval/v3/dynamic-tools.ts
|
|
5354
|
-
var
|
|
5512
|
+
var import_zod7 = require("zod");
|
|
5355
5513
|
var import_ai4 = require("ai");
|
|
5356
5514
|
async function createDynamicTools(chunks, hadExcludedContent) {
|
|
5357
5515
|
const { db: db2 } = await postgresClient();
|
|
@@ -5370,9 +5528,9 @@ async function createDynamicTools(chunks, hadExcludedContent) {
|
|
|
5370
5528
|
const capturedChunk = chunk;
|
|
5371
5529
|
tools[browseToolName] = (0, import_ai4.tool)({
|
|
5372
5530
|
description: `"${chunk.item_name}" has ${total} pages/chunks. Use this to read a range of pages from it.`,
|
|
5373
|
-
inputSchema:
|
|
5374
|
-
from_index:
|
|
5375
|
-
to_index:
|
|
5531
|
+
inputSchema: import_zod7.z.object({
|
|
5532
|
+
from_index: import_zod7.z.number().min(1).default(1).describe("Starting chunk index (1-based)"),
|
|
5533
|
+
to_index: import_zod7.z.number().max(total).describe(`Ending chunk index (max ${total})`)
|
|
5376
5534
|
}),
|
|
5377
5535
|
execute: async ({ from_index, to_index }) => {
|
|
5378
5536
|
const { db: db22 } = await postgresClient();
|
|
@@ -5401,8 +5559,8 @@ async function createDynamicTools(chunks, hadExcludedContent) {
|
|
|
5401
5559
|
const capturedChunk = chunk;
|
|
5402
5560
|
tools[pageToolName] = (0, import_ai4.tool)({
|
|
5403
5561
|
description: `Load the full text of page ${chunk.chunk_index} from "${chunk.item_name}"`,
|
|
5404
|
-
inputSchema:
|
|
5405
|
-
reasoning:
|
|
5562
|
+
inputSchema: import_zod7.z.object({
|
|
5563
|
+
reasoning: import_zod7.z.string().describe("Why you need this specific page's content")
|
|
5406
5564
|
}),
|
|
5407
5565
|
execute: async () => {
|
|
5408
5566
|
const { db: db22 } = await postgresClient();
|
|
@@ -5429,8 +5587,8 @@ async function createDynamicTools(chunks, hadExcludedContent) {
|
|
|
5429
5587
|
var FINISH_TOOL_NAME = "finish_retrieval";
|
|
5430
5588
|
var finishRetrievalTool = (0, import_ai5.tool)({
|
|
5431
5589
|
description: "Call this tool when you have retrieved sufficient information and no further searches are needed. You MUST call this tool to signal that retrieval is complete \u2014 do not write a text conclusion.",
|
|
5432
|
-
inputSchema:
|
|
5433
|
-
reasoning:
|
|
5590
|
+
inputSchema: import_zod8.z.object({
|
|
5591
|
+
reasoning: import_zod8.z.string().describe("One sentence explaining why retrieval is complete")
|
|
5434
5592
|
}),
|
|
5435
5593
|
execute: async ({ reasoning }) => JSON.stringify({ finished: true, reasoning })
|
|
5436
5594
|
});
|
|
@@ -5463,7 +5621,7 @@ function extractChunksFromToolResults(toolResults) {
|
|
|
5463
5621
|
return chunks;
|
|
5464
5622
|
}
|
|
5465
5623
|
async function* runAgentLoop(params) {
|
|
5466
|
-
const { query, strategy, tools, model, reranker, contextGuidance, customInstructions, onStepComplete } = params;
|
|
5624
|
+
const { query, strategy, tools, model, reranker, contextGuidance, customInstructions, sessionId, onStepComplete, onTrajectoryStep } = params;
|
|
5467
5625
|
const output = {
|
|
5468
5626
|
steps: [],
|
|
5469
5627
|
reasoning: [],
|
|
@@ -5519,6 +5677,13 @@ ${customInstructions}` : ""
|
|
|
5519
5677
|
}
|
|
5520
5678
|
messages.push(...result.response.messages);
|
|
5521
5679
|
let stepChunks = extractChunksFromToolResults(result.toolResults);
|
|
5680
|
+
const seenChunkIds = /* @__PURE__ */ new Set();
|
|
5681
|
+
stepChunks = stepChunks.filter((c) => {
|
|
5682
|
+
if (!c.chunk_id) return true;
|
|
5683
|
+
if (seenChunkIds.has(c.chunk_id)) return false;
|
|
5684
|
+
seenChunkIds.add(c.chunk_id);
|
|
5685
|
+
return true;
|
|
5686
|
+
});
|
|
5522
5687
|
const hadExcludedContent = result.toolCalls?.some(
|
|
5523
5688
|
(tc) => tc.toolName === "search_content" && tc.input?.includeContent === false || tc.toolName === "search_items_by_name"
|
|
5524
5689
|
);
|
|
@@ -5528,9 +5693,15 @@ ${customInstructions}` : ""
|
|
|
5528
5693
|
}
|
|
5529
5694
|
const newDynamic = await createDynamicTools(stepChunks, hadExcludedContent);
|
|
5530
5695
|
Object.assign(dynamicTools, newDynamic);
|
|
5696
|
+
if (sessionId && Object.keys(newDynamic).length > 0) {
|
|
5697
|
+
registerSessionTools(sessionId, newDynamic);
|
|
5698
|
+
}
|
|
5531
5699
|
forceDepthExploration = stepChunks.length > 0 && stepChunks.length < 5 && Object.keys(newDynamic).length > 0 && step < strategy.stepBudget - 2;
|
|
5532
5700
|
for (const tc of result.toolCalls ?? []) {
|
|
5533
5701
|
if (SEARCH_TOOL_NAMES.has(tc.toolName)) {
|
|
5702
|
+
if (tc.input?.knowledge_base_id) {
|
|
5703
|
+
searchedContextIds.add(tc.input.knowledge_base_id);
|
|
5704
|
+
}
|
|
5534
5705
|
for (const id of tc.input?.knowledge_base_ids ?? []) {
|
|
5535
5706
|
searchedContextIds.add(id);
|
|
5536
5707
|
}
|
|
@@ -5565,9 +5736,30 @@ ${customInstructions}` : ""
|
|
|
5565
5736
|
output: stepChunks
|
|
5566
5737
|
})) ?? []
|
|
5567
5738
|
});
|
|
5568
|
-
output.chunks.
|
|
5739
|
+
const existingChunkIds = new Set(output.chunks.map((c) => c.chunk_id).filter(Boolean));
|
|
5740
|
+
output.chunks.push(...stepChunks.filter((c) => !c.chunk_id || !existingChunkIds.has(c.chunk_id)));
|
|
5569
5741
|
output.usage.push(result.usage);
|
|
5570
5742
|
onStepComplete?.(stepRecord);
|
|
5743
|
+
if (onTrajectoryStep) {
|
|
5744
|
+
const toolResultMap = /* @__PURE__ */ new Map();
|
|
5745
|
+
for (const tr of result.toolResults ?? []) {
|
|
5746
|
+
toolResultMap.set(tr.toolCallId, tr.output ?? tr.result);
|
|
5747
|
+
}
|
|
5748
|
+
onTrajectoryStep({
|
|
5749
|
+
stepNumber: step + 1,
|
|
5750
|
+
systemPrompt: stepSystemPrompt,
|
|
5751
|
+
text: result.text ?? "",
|
|
5752
|
+
toolCalls: result.toolCalls?.map((tc) => ({
|
|
5753
|
+
name: tc.toolName,
|
|
5754
|
+
id: tc.toolCallId,
|
|
5755
|
+
input: tc.input,
|
|
5756
|
+
output: toolResultMap.get(tc.toolCallId)
|
|
5757
|
+
})) ?? [],
|
|
5758
|
+
chunks: stepChunks,
|
|
5759
|
+
dynamicToolsCreated: Object.keys(newDynamic),
|
|
5760
|
+
tokens: result.usage?.totalTokens ?? 0
|
|
5761
|
+
});
|
|
5762
|
+
}
|
|
5571
5763
|
yield { ...output };
|
|
5572
5764
|
const calledFinish = result.toolCalls?.some(
|
|
5573
5765
|
(tc) => tc.toolName === FINISH_TOOL_NAME
|
|
@@ -5585,21 +5777,23 @@ ${customInstructions}` : ""
|
|
|
5585
5777
|
}
|
|
5586
5778
|
|
|
5587
5779
|
// ee/agentic-retrieval/v3/trajectory.ts
|
|
5588
|
-
var
|
|
5780
|
+
var fs = __toESM(require("fs/promises"), 1);
|
|
5589
5781
|
var path = __toESM(require("path"), 1);
|
|
5590
5782
|
var trajectoryRegistry = {
|
|
5591
5783
|
lastFile: void 0
|
|
5592
5784
|
};
|
|
5593
5785
|
var TrajectoryLogger = class {
|
|
5594
5786
|
data;
|
|
5787
|
+
richSteps = [];
|
|
5595
5788
|
startTime = Date.now();
|
|
5596
5789
|
logDir;
|
|
5597
|
-
constructor(query, classification, logDir = path.join(process.cwd(), "ee/agentic-retrieval/logs")) {
|
|
5790
|
+
constructor(query, classification, logDir = path.join(process.cwd(), "ee/agentic-retrieval/logs"), preselectedItemIds) {
|
|
5598
5791
|
this.logDir = logDir;
|
|
5599
5792
|
this.data = {
|
|
5600
5793
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5601
5794
|
query,
|
|
5602
5795
|
classification,
|
|
5796
|
+
preselectedItemIds: preselectedItemIds?.length ? preselectedItemIds : void 0,
|
|
5603
5797
|
steps: [],
|
|
5604
5798
|
final: {
|
|
5605
5799
|
total_chunks: 0,
|
|
@@ -5620,23 +5814,181 @@ var TrajectoryLogger = class {
|
|
|
5620
5814
|
tokens: step.tokens
|
|
5621
5815
|
});
|
|
5622
5816
|
}
|
|
5623
|
-
|
|
5817
|
+
recordRichStep(data) {
|
|
5818
|
+
this.richSteps.push(data);
|
|
5819
|
+
}
|
|
5820
|
+
toMarkdown(durationMs, success, error) {
|
|
5821
|
+
const totalTokens = this.richSteps.reduce((sum, s) => sum + s.tokens, 0);
|
|
5822
|
+
const totalChunks = this.richSteps.reduce((sum, s) => sum + s.chunks.length, 0);
|
|
5823
|
+
const status = success ? "\u2713 Success" : `\u2717 Failed${error ? `: ${error.message}` : ""}`;
|
|
5824
|
+
const lines = [];
|
|
5825
|
+
lines.push(`# Agentic Retrieval \u2014 ${this.data.timestamp}`);
|
|
5826
|
+
lines.push("");
|
|
5827
|
+
lines.push(`**Query:** ${this.data.query} `);
|
|
5828
|
+
lines.push(
|
|
5829
|
+
`**Duration:** ${(durationMs / 1e3).toFixed(1)}s | **Tokens:** ${totalTokens} | **Status:** ${status}`
|
|
5830
|
+
);
|
|
5831
|
+
lines.push("");
|
|
5832
|
+
lines.push("## Classification");
|
|
5833
|
+
lines.push("");
|
|
5834
|
+
lines.push(`- **Type:** \`${this.data.classification.queryType}\``);
|
|
5835
|
+
lines.push(`- **Language:** \`${this.data.classification.language}\``);
|
|
5836
|
+
const suggested = this.data.classification.suggestedContextIds;
|
|
5837
|
+
lines.push(
|
|
5838
|
+
`- **Suggested contexts:** ${suggested.length > 0 ? suggested.map((id) => `\`${id}\``).join(", ") : "*(all)*"}`
|
|
5839
|
+
);
|
|
5840
|
+
if (this.data.preselectedItemIds?.length) {
|
|
5841
|
+
lines.push(
|
|
5842
|
+
`- **Preselected item IDs:** ${this.data.preselectedItemIds.map((id) => `\`${id}\``).join(", ")}`
|
|
5843
|
+
);
|
|
5844
|
+
}
|
|
5845
|
+
lines.push("");
|
|
5846
|
+
lines.push("---");
|
|
5847
|
+
lines.push("");
|
|
5848
|
+
const firstStep = this.richSteps[0];
|
|
5849
|
+
if (firstStep) {
|
|
5850
|
+
lines.push("## System Prompt");
|
|
5851
|
+
lines.push("");
|
|
5852
|
+
lines.push("<details>");
|
|
5853
|
+
lines.push("<summary>View system prompt</summary>");
|
|
5854
|
+
lines.push("");
|
|
5855
|
+
lines.push("```");
|
|
5856
|
+
lines.push(firstStep.systemPrompt);
|
|
5857
|
+
lines.push("```");
|
|
5858
|
+
lines.push("");
|
|
5859
|
+
lines.push("</details>");
|
|
5860
|
+
lines.push("");
|
|
5861
|
+
lines.push("---");
|
|
5862
|
+
lines.push("");
|
|
5863
|
+
}
|
|
5864
|
+
for (const step of this.richSteps) {
|
|
5865
|
+
const toolLabel = step.toolCalls.map((tc) => `\`${tc.name}\``).join(", ") || "*(no tool calls)*";
|
|
5866
|
+
lines.push(`## Step ${step.stepNumber} \u2014 ${toolLabel}`);
|
|
5867
|
+
lines.push("");
|
|
5868
|
+
const dynLabel = step.dynamicToolsCreated.length > 0 ? step.dynamicToolsCreated.map((t) => `\`${t}\``).join(", ") : "none";
|
|
5869
|
+
lines.push(
|
|
5870
|
+
`**Tokens:** ${step.tokens} | **Chunks retrieved:** ${step.chunks.length} | **Dynamic tools created:** ${dynLabel}`
|
|
5871
|
+
);
|
|
5872
|
+
lines.push("");
|
|
5873
|
+
if (step.text) {
|
|
5874
|
+
lines.push("### Reasoning");
|
|
5875
|
+
lines.push("");
|
|
5876
|
+
lines.push(step.text);
|
|
5877
|
+
lines.push("");
|
|
5878
|
+
}
|
|
5879
|
+
if (step.toolCalls.length > 0) {
|
|
5880
|
+
lines.push("### Tool Calls");
|
|
5881
|
+
lines.push("");
|
|
5882
|
+
for (const [i, tc] of step.toolCalls.entries()) {
|
|
5883
|
+
lines.push(`#### ${i + 1}. \`${tc.name}\``);
|
|
5884
|
+
lines.push("");
|
|
5885
|
+
lines.push("**Input:**");
|
|
5886
|
+
lines.push("```json");
|
|
5887
|
+
lines.push(JSON.stringify(tc.input, null, 2));
|
|
5888
|
+
lines.push("```");
|
|
5889
|
+
lines.push("");
|
|
5890
|
+
if (tc.output !== void 0) {
|
|
5891
|
+
let parsedOutput;
|
|
5892
|
+
try {
|
|
5893
|
+
parsedOutput = typeof tc.output === "string" ? JSON.parse(tc.output) : tc.output;
|
|
5894
|
+
} catch {
|
|
5895
|
+
parsedOutput = tc.output;
|
|
5896
|
+
}
|
|
5897
|
+
const outputStr = JSON.stringify(parsedOutput, null, 2);
|
|
5898
|
+
const truncated = outputStr.length > 2e3;
|
|
5899
|
+
lines.push("**Output:**");
|
|
5900
|
+
lines.push("```json");
|
|
5901
|
+
lines.push(truncated ? `${outputStr.slice(0, 2e3)}
|
|
5902
|
+
\u2026 (truncated)` : outputStr);
|
|
5903
|
+
lines.push("```");
|
|
5904
|
+
lines.push("");
|
|
5905
|
+
}
|
|
5906
|
+
}
|
|
5907
|
+
}
|
|
5908
|
+
if (step.chunks.length > 0) {
|
|
5909
|
+
lines.push("### Chunks Retrieved");
|
|
5910
|
+
lines.push("");
|
|
5911
|
+
lines.push("| # | Item | Context | Chunk | Score |");
|
|
5912
|
+
lines.push("|---|------|---------|-------|-------|");
|
|
5913
|
+
for (const [i, c] of step.chunks.entries()) {
|
|
5914
|
+
const score = c.metadata?.hybrid_score ?? c.metadata?.cosine_distance ?? c.metadata?.fts_rank ?? "\u2014";
|
|
5915
|
+
const scoreStr = typeof score === "number" ? score.toFixed(4) : String(score);
|
|
5916
|
+
lines.push(
|
|
5917
|
+
`| ${i + 1} | ${c.item_name ?? "\u2014"} | \`${c.context}\` | ${c.chunk_index ?? "\u2014"} | ${scoreStr} |`
|
|
5918
|
+
);
|
|
5919
|
+
}
|
|
5920
|
+
lines.push("");
|
|
5921
|
+
const withContent = step.chunks.filter((c) => c.chunk_content);
|
|
5922
|
+
if (withContent.length > 0) {
|
|
5923
|
+
lines.push("<details>");
|
|
5924
|
+
lines.push("<summary>View chunk content</summary>");
|
|
5925
|
+
lines.push("");
|
|
5926
|
+
for (const c of withContent) {
|
|
5927
|
+
lines.push(`**${c.item_name} (chunk ${c.chunk_index}):**`);
|
|
5928
|
+
lines.push("");
|
|
5929
|
+
const content = (c.chunk_content ?? "").trim();
|
|
5930
|
+
lines.push(`> ${content.split("\n").join("\n> ")}`);
|
|
5931
|
+
lines.push("");
|
|
5932
|
+
}
|
|
5933
|
+
lines.push("</details>");
|
|
5934
|
+
lines.push("");
|
|
5935
|
+
}
|
|
5936
|
+
}
|
|
5937
|
+
if (firstStep && step.stepNumber > 1 && step.systemPrompt !== firstStep.systemPrompt) {
|
|
5938
|
+
const addendum = step.systemPrompt.slice(firstStep.systemPrompt.length).trim();
|
|
5939
|
+
if (addendum) {
|
|
5940
|
+
lines.push("<details>");
|
|
5941
|
+
lines.push("<summary>System prompt addendum (this step only)</summary>");
|
|
5942
|
+
lines.push("");
|
|
5943
|
+
lines.push("```");
|
|
5944
|
+
lines.push(addendum);
|
|
5945
|
+
lines.push("```");
|
|
5946
|
+
lines.push("");
|
|
5947
|
+
lines.push("</details>");
|
|
5948
|
+
lines.push("");
|
|
5949
|
+
}
|
|
5950
|
+
}
|
|
5951
|
+
lines.push("---");
|
|
5952
|
+
lines.push("");
|
|
5953
|
+
}
|
|
5954
|
+
lines.push("## Summary");
|
|
5955
|
+
lines.push("");
|
|
5956
|
+
lines.push("| Metric | Value |");
|
|
5957
|
+
lines.push("|--------|-------|");
|
|
5958
|
+
lines.push(`| Steps | ${this.richSteps.length} |`);
|
|
5959
|
+
lines.push(`| Total chunks | ${totalChunks} |`);
|
|
5960
|
+
lines.push(`| Total tokens | ${totalTokens} |`);
|
|
5961
|
+
lines.push(`| Duration | ${(durationMs / 1e3).toFixed(1)}s |`);
|
|
5962
|
+
lines.push(`| Status | ${status} |`);
|
|
5963
|
+
if (error) {
|
|
5964
|
+
lines.push(`| Error | ${error.message} |`);
|
|
5965
|
+
}
|
|
5966
|
+
lines.push("");
|
|
5967
|
+
return lines.join("\n");
|
|
5968
|
+
}
|
|
5969
|
+
async finalize(output, success, error, writeFiles = false) {
|
|
5970
|
+
const durationMs = Date.now() - this.startTime;
|
|
5624
5971
|
this.data.final = {
|
|
5625
5972
|
total_chunks: output.chunks.length,
|
|
5626
5973
|
total_steps: output.steps.length,
|
|
5627
5974
|
total_tokens: output.totalTokens,
|
|
5628
|
-
duration_ms:
|
|
5975
|
+
duration_ms: durationMs,
|
|
5629
5976
|
success,
|
|
5630
5977
|
error: error?.message
|
|
5631
5978
|
};
|
|
5979
|
+
if (!writeFiles) return void 0;
|
|
5632
5980
|
try {
|
|
5633
|
-
await
|
|
5634
|
-
const
|
|
5635
|
-
const
|
|
5636
|
-
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
|
|
5981
|
+
await fs.mkdir(this.logDir, { recursive: true });
|
|
5982
|
+
const ts = Date.now();
|
|
5983
|
+
const jsonPath = path.join(this.logDir, `trajectory_${ts}.json`);
|
|
5984
|
+
const mdPath = path.join(this.logDir, `trajectory_${ts}.md`);
|
|
5985
|
+
await Promise.all([
|
|
5986
|
+
fs.writeFile(jsonPath, JSON.stringify(this.data, null, 2), "utf-8"),
|
|
5987
|
+
fs.writeFile(mdPath, this.toMarkdown(durationMs, success, error), "utf-8")
|
|
5988
|
+
]);
|
|
5989
|
+
console.log(`[EXULU] v3 trajectory saved: trajectory_${ts}.json + trajectory_${ts}.md`);
|
|
5990
|
+
trajectoryRegistry.lastFile = jsonPath;
|
|
5991
|
+
return jsonPath;
|
|
5640
5992
|
} catch (e) {
|
|
5641
5993
|
console.error("[EXULU] v3 failed to write trajectory:", e);
|
|
5642
5994
|
return void 0;
|
|
@@ -5653,14 +6005,19 @@ async function* executeV3({
|
|
|
5653
6005
|
model,
|
|
5654
6006
|
user,
|
|
5655
6007
|
role,
|
|
5656
|
-
customInstructions
|
|
6008
|
+
customInstructions,
|
|
6009
|
+
logTrajectory,
|
|
6010
|
+
sessionId,
|
|
6011
|
+
preselectedItemIds
|
|
5657
6012
|
}) {
|
|
6013
|
+
const preselectedByContext = preselectedItemIds?.length ? parseGlobalItemIds(preselectedItemIds) : void 0;
|
|
6014
|
+
const activeContexts = preselectedByContext?.size ? contexts.filter((c) => preselectedByContext.has(c.id)) : contexts;
|
|
5658
6015
|
console.log("[EXULU] v3 \u2014 sampling contexts");
|
|
5659
|
-
const samples = await sampler.getSamples(
|
|
6016
|
+
const samples = await sampler.getSamples(activeContexts, user, role);
|
|
5660
6017
|
console.log("[EXULU] v3 \u2014 classifying query");
|
|
5661
6018
|
let classification;
|
|
5662
6019
|
try {
|
|
5663
|
-
classification = await classifyQuery(query,
|
|
6020
|
+
classification = await classifyQuery(query, activeContexts, samples, model);
|
|
5664
6021
|
} catch (err) {
|
|
5665
6022
|
console.warn("[EXULU] v3 \u2014 classification failed, falling back to exploratory:", err);
|
|
5666
6023
|
classification = {
|
|
@@ -5672,15 +6029,18 @@ async function* executeV3({
|
|
|
5672
6029
|
console.log("[EXULU] v3 \u2014 classified as:", classification);
|
|
5673
6030
|
const strategy = STRATEGIES[classification.queryType];
|
|
5674
6031
|
const suggestedIds = classification.suggestedContextIds;
|
|
5675
|
-
const fallbackIds =
|
|
5676
|
-
const
|
|
6032
|
+
const fallbackIds = activeContexts.filter((c) => !suggestedIds.includes(c.id)).map((c) => c.id);
|
|
6033
|
+
const contextBase = suggestedIds.length > 0 ? `Suggested priority contexts: [${suggestedIds.join(", ")}]. Also available: [${fallbackIds.join(", ")}]. Custom instructions may require searching additional or all contexts \u2014 follow them.` : `All contexts available: [${activeContexts.map((c) => c.id).join(", ")}].`;
|
|
6034
|
+
const preselectedNote = preselectedByContext?.size ? `
|
|
6035
|
+
SCOPE CONSTRAINT: Retrieval is scoped to preselected items/contexts. Per context: ${[...preselectedByContext.entries()].map(([ctx, ids]) => ids === null ? `${ctx} (full context)` : `${ctx} (${ids.length} item${ids.length === 1 ? "" : "s"})`).join(", ")}. All tools enforce this scope automatically. For full-context entries you may search freely; for item-restricted entries do NOT use search_items_by_name for discovery \u2014 go directly to search_content or save_search_results.` : "";
|
|
6036
|
+
const contextGuidance = contextBase + preselectedNote;
|
|
5677
6037
|
const bashToolkit = await (0, import_bash_tool.createBashTool)({ files: {} });
|
|
5678
6038
|
const retrievalTools = createRetrievalTools({
|
|
5679
|
-
contexts,
|
|
5680
|
-
// ALL contexts — agent decides which to search based on context guidance
|
|
6039
|
+
contexts: activeContexts,
|
|
5681
6040
|
user,
|
|
5682
6041
|
role,
|
|
5683
|
-
updateVirtualFiles: (files) => bashToolkit.sandbox.writeFiles(files)
|
|
6042
|
+
updateVirtualFiles: (files) => bashToolkit.sandbox.writeFiles(files),
|
|
6043
|
+
preselectedItemsByContext: preselectedByContext
|
|
5684
6044
|
});
|
|
5685
6045
|
const activeTools = {};
|
|
5686
6046
|
for (const name of strategy.retrieval_tools) {
|
|
@@ -5691,7 +6051,7 @@ async function* executeV3({
|
|
|
5691
6051
|
if (strategy.include_bash) {
|
|
5692
6052
|
Object.assign(activeTools, bashToolkit.tools);
|
|
5693
6053
|
}
|
|
5694
|
-
const trajectory = new TrajectoryLogger(query, classification);
|
|
6054
|
+
const trajectory = new TrajectoryLogger(query, classification, void 0, preselectedItemIds);
|
|
5695
6055
|
let finalOutput;
|
|
5696
6056
|
let executionError;
|
|
5697
6057
|
try {
|
|
@@ -5704,7 +6064,9 @@ async function* executeV3({
|
|
|
5704
6064
|
contextGuidance,
|
|
5705
6065
|
customInstructions,
|
|
5706
6066
|
classification,
|
|
5707
|
-
|
|
6067
|
+
sessionId,
|
|
6068
|
+
onStepComplete: (step) => trajectory.recordStep(step),
|
|
6069
|
+
onTrajectoryStep: (data) => trajectory.recordRichStep(data)
|
|
5708
6070
|
})) {
|
|
5709
6071
|
finalOutput = output;
|
|
5710
6072
|
yield output;
|
|
@@ -5715,7 +6077,7 @@ async function* executeV3({
|
|
|
5715
6077
|
throw err;
|
|
5716
6078
|
} finally {
|
|
5717
6079
|
if (finalOutput) {
|
|
5718
|
-
const trajectoryFile = await trajectory.finalize(finalOutput, !executionError, executionError);
|
|
6080
|
+
const trajectoryFile = await trajectory.finalize(finalOutput, !executionError, executionError, logTrajectory);
|
|
5719
6081
|
if (trajectoryFile) {
|
|
5720
6082
|
finalOutput.trajectoryFile = trajectoryFile;
|
|
5721
6083
|
}
|
|
@@ -5728,7 +6090,8 @@ function createAgenticRetrievalToolV3({
|
|
|
5728
6090
|
rerankers,
|
|
5729
6091
|
user,
|
|
5730
6092
|
role,
|
|
5731
|
-
model
|
|
6093
|
+
model,
|
|
6094
|
+
preselectedItemIds
|
|
5732
6095
|
}) {
|
|
5733
6096
|
const license = checkLicense();
|
|
5734
6097
|
if (!license["agentic-retrieval"]) {
|
|
@@ -5756,6 +6119,12 @@ function createAgenticRetrievalToolV3({
|
|
|
5756
6119
|
type: "string",
|
|
5757
6120
|
default: "none"
|
|
5758
6121
|
},
|
|
6122
|
+
{
|
|
6123
|
+
name: "managed_context",
|
|
6124
|
+
description: "Makes sure the user defines which items from which contexts the agentic retrieval tool will search in",
|
|
6125
|
+
type: "boolean",
|
|
6126
|
+
default: false
|
|
6127
|
+
},
|
|
5759
6128
|
{
|
|
5760
6129
|
name: "reasoning_model",
|
|
5761
6130
|
description: "By default the agentic retrieval tool uses the model from the agent calling the tool, but you can overwrite this here for the reasoning phase",
|
|
@@ -5768,6 +6137,18 @@ function createAgenticRetrievalToolV3({
|
|
|
5768
6137
|
type: "string",
|
|
5769
6138
|
default: ""
|
|
5770
6139
|
},
|
|
6140
|
+
{
|
|
6141
|
+
name: "require_preselected_contexts",
|
|
6142
|
+
description: "Require the user to preselect contexts before executing the tool, meaning the user will be asked to select the contexts they want to search in",
|
|
6143
|
+
type: "boolean",
|
|
6144
|
+
default: false
|
|
6145
|
+
},
|
|
6146
|
+
{
|
|
6147
|
+
name: "log_trajectories",
|
|
6148
|
+
description: "Save a detailed markdown + JSON log of every retrieval execution to disk. Useful for debugging and evaluation.",
|
|
6149
|
+
type: "boolean",
|
|
6150
|
+
default: false
|
|
6151
|
+
},
|
|
5771
6152
|
...contexts.map((ctx) => ({
|
|
5772
6153
|
name: ctx.id,
|
|
5773
6154
|
description: `Enable search in "${ctx.name}". ${ctx.description}`,
|
|
@@ -5775,32 +6156,61 @@ function createAgenticRetrievalToolV3({
|
|
|
5775
6156
|
default: true
|
|
5776
6157
|
}))
|
|
5777
6158
|
],
|
|
5778
|
-
inputSchema:
|
|
5779
|
-
query:
|
|
5780
|
-
userInstructions:
|
|
6159
|
+
inputSchema: import_zod9.z.object({
|
|
6160
|
+
query: import_zod9.z.string().describe("The question or query to answer"),
|
|
6161
|
+
userInstructions: import_zod9.z.string().optional().describe("Additional instructions from the user to guide retrieval"),
|
|
6162
|
+
confirmedContextIds: import_zod9.z.array(import_zod9.z.string()).optional().describe(
|
|
6163
|
+
"Knowledge base IDs explicitly confirmed by the user to be used in the retrieval. When presen only searches these contexts. "
|
|
6164
|
+
)
|
|
5781
6165
|
}),
|
|
5782
6166
|
execute: async function* ({
|
|
5783
6167
|
query,
|
|
5784
6168
|
userInstructions,
|
|
5785
|
-
|
|
6169
|
+
confirmedContextIds,
|
|
6170
|
+
toolVariablesConfig,
|
|
6171
|
+
sessionID
|
|
5786
6172
|
}) {
|
|
5787
6173
|
if (!model) {
|
|
5788
|
-
|
|
6174
|
+
yield { result: "Model is required for executing the agentic retrieval tool" };
|
|
6175
|
+
return;
|
|
5789
6176
|
}
|
|
5790
6177
|
let activeContexts = contexts;
|
|
5791
6178
|
let configuredReranker;
|
|
5792
6179
|
let configInstructions = "";
|
|
6180
|
+
let logTrajectory = false;
|
|
6181
|
+
let requiresPreselectedContexts = false;
|
|
6182
|
+
let managedContextEnabled = false;
|
|
5793
6183
|
if (toolVariablesConfig) {
|
|
5794
6184
|
configInstructions = toolVariablesConfig["instructions"] ?? "";
|
|
6185
|
+
logTrajectory = toolVariablesConfig["log_trajectories"] === true || toolVariablesConfig["log_trajectories"] === "true";
|
|
6186
|
+
managedContextEnabled = toolVariablesConfig["managed_context"] === true || toolVariablesConfig["managed_context"] === "true";
|
|
5795
6187
|
activeContexts = contexts.filter(
|
|
5796
6188
|
(ctx) => toolVariablesConfig[ctx.id] === true || toolVariablesConfig[ctx.id] === "true" || toolVariablesConfig[ctx.id] === 1
|
|
5797
6189
|
);
|
|
5798
6190
|
if (activeContexts.length === 0) activeContexts = contexts;
|
|
6191
|
+
requiresPreselectedContexts = toolVariablesConfig["require_preselected_contexts"] === true || toolVariablesConfig["require_preselected_contexts"] === "true";
|
|
5799
6192
|
const rerankerId = toolVariablesConfig["reranker"];
|
|
5800
6193
|
if (rerankerId && rerankerId !== "none") {
|
|
5801
6194
|
configuredReranker = rerankers.find((r) => r.id === rerankerId);
|
|
5802
6195
|
}
|
|
5803
6196
|
}
|
|
6197
|
+
console.log("[EXULU] Managed context enabled:", managedContextEnabled);
|
|
6198
|
+
console.log("[EXULU] Preselected item IDs:", preselectedItemIds);
|
|
6199
|
+
if (managedContextEnabled && !preselectedItemIds?.length) {
|
|
6200
|
+
console.log("[EXULU] Managed context was enabled for the agentic retrieval tool. This means that the user must preselect items that the agentic retrieval tool will search in, please notify the user to preselect items before executing the tool.");
|
|
6201
|
+
yield { result: "Managed context was enabled for the agentic retrieval tool. This means that the user must preselect items that the agentic retrieval tool will search in, please notify the user to preselect items before executing the tool." };
|
|
6202
|
+
return;
|
|
6203
|
+
}
|
|
6204
|
+
if (requiresPreselectedContexts && !confirmedContextIds?.length && !preselectedItemIds?.length) {
|
|
6205
|
+
console.log("[EXULU] The user must choose between the available contexts before executing the tool. The available contexts are: " + activeContexts.map((c) => c.id).join(", ") + ". If the question_ask tool is available use that to ask the user which contexts they want to search in, otherwise just ask them in plain text.");
|
|
6206
|
+
yield { result: "The user must choose between the available contexts before executing the tool, the available contexts are: " + activeContexts.map((c) => c.id).join(", ") + ". If the question_ask tool is available use that to ask the user which contexts they want to search in, otherwise just ask them in plain text." };
|
|
6207
|
+
return;
|
|
6208
|
+
}
|
|
6209
|
+
if (confirmedContextIds?.length) {
|
|
6210
|
+
const confirmed = new Set(confirmedContextIds);
|
|
6211
|
+
const filtered = activeContexts.filter((c) => confirmed.has(c.id));
|
|
6212
|
+
if (filtered.length > 0) activeContexts = filtered;
|
|
6213
|
+
}
|
|
5804
6214
|
const combinedInstructions = [
|
|
5805
6215
|
configInstructions ? `Configuration instructions: ${configInstructions}` : "",
|
|
5806
6216
|
adminInstructions ? `Admin instructions: ${adminInstructions}` : "",
|
|
@@ -5813,10 +6223,14 @@ function createAgenticRetrievalToolV3({
|
|
|
5813
6223
|
model,
|
|
5814
6224
|
user,
|
|
5815
6225
|
role,
|
|
5816
|
-
customInstructions: combinedInstructions || void 0
|
|
6226
|
+
customInstructions: combinedInstructions || void 0,
|
|
6227
|
+
logTrajectory,
|
|
6228
|
+
sessionId: sessionID,
|
|
6229
|
+
preselectedItemIds
|
|
5817
6230
|
})) {
|
|
5818
6231
|
yield { result: JSON.stringify(output) };
|
|
5819
6232
|
}
|
|
6233
|
+
return;
|
|
5820
6234
|
}
|
|
5821
6235
|
});
|
|
5822
6236
|
}
|
|
@@ -5971,9 +6385,9 @@ var addProviderFields = async (args, requestedFields, providers, result, tools,
|
|
|
5971
6385
|
if (result.tools) {
|
|
5972
6386
|
result.tools = await Promise.all(
|
|
5973
6387
|
result.tools.map(
|
|
5974
|
-
async (
|
|
6388
|
+
async (tool5) => {
|
|
5975
6389
|
let hydrated;
|
|
5976
|
-
if (
|
|
6390
|
+
if (tool5.id === "agentic_context_search") {
|
|
5977
6391
|
const instance2 = createAgenticRetrievalToolV3({
|
|
5978
6392
|
contexts: [],
|
|
5979
6393
|
rerankers: [],
|
|
@@ -5989,23 +6403,23 @@ var addProviderFields = async (args, requestedFields, providers, result, tools,
|
|
|
5989
6403
|
name: instance2.name,
|
|
5990
6404
|
description: instance2.description,
|
|
5991
6405
|
category: instance2.category,
|
|
5992
|
-
config:
|
|
6406
|
+
config: tool5.config
|
|
5993
6407
|
};
|
|
5994
6408
|
}
|
|
5995
|
-
if (
|
|
5996
|
-
if (
|
|
6409
|
+
if (tool5.type === "agent") {
|
|
6410
|
+
if (tool5.id === result.id) {
|
|
5997
6411
|
return null;
|
|
5998
6412
|
}
|
|
5999
|
-
const instance2 = await exuluApp.get().agent(
|
|
6413
|
+
const instance2 = await exuluApp.get().agent(tool5.id);
|
|
6000
6414
|
if (!instance2) {
|
|
6001
6415
|
throw new Error(
|
|
6002
|
-
"Trying to load a tool of type 'agent', but the associated agent with id " +
|
|
6416
|
+
"Trying to load a tool of type 'agent', but the associated agent with id " + tool5.id + " was not found in the database."
|
|
6003
6417
|
);
|
|
6004
6418
|
}
|
|
6005
6419
|
const provider2 = providers.find((a) => a.id === instance2.provider);
|
|
6006
6420
|
if (!provider2) {
|
|
6007
6421
|
throw new Error(
|
|
6008
|
-
"Trying to load a tool of type 'agent', but the associated agent with id " +
|
|
6422
|
+
"Trying to load a tool of type 'agent', but the associated agent with id " + tool5.id + " does not have a provider set for it."
|
|
6009
6423
|
);
|
|
6010
6424
|
}
|
|
6011
6425
|
const hasAccessToAgent = await checkRecordAccess(instance2, "read", user);
|
|
@@ -6014,13 +6428,13 @@ var addProviderFields = async (args, requestedFields, providers, result, tools,
|
|
|
6014
6428
|
}
|
|
6015
6429
|
hydrated = await provider2.tool(instance2.id, providers, contexts, rerankers);
|
|
6016
6430
|
} else {
|
|
6017
|
-
hydrated = tools.find((t) => t.id ===
|
|
6431
|
+
hydrated = tools.find((t) => t.id === tool5.id);
|
|
6018
6432
|
}
|
|
6019
6433
|
const hydratedTool = {
|
|
6020
|
-
...
|
|
6434
|
+
...tool5,
|
|
6021
6435
|
name: hydrated?.name || "",
|
|
6022
6436
|
description: hydrated?.description || "",
|
|
6023
|
-
category:
|
|
6437
|
+
category: tool5?.category || "default"
|
|
6024
6438
|
};
|
|
6025
6439
|
console.log("[EXULU] hydratedTool", hydratedTool);
|
|
6026
6440
|
return hydratedTool;
|
|
@@ -6038,7 +6452,7 @@ var addProviderFields = async (args, requestedFields, providers, result, tools,
|
|
|
6038
6452
|
result.tools.unshift(projectTool);
|
|
6039
6453
|
}
|
|
6040
6454
|
}
|
|
6041
|
-
result.tools = result.tools.filter((
|
|
6455
|
+
result.tools = result.tools.filter((tool5) => tool5 !== null);
|
|
6042
6456
|
} else {
|
|
6043
6457
|
result.tools = [];
|
|
6044
6458
|
}
|
|
@@ -7544,7 +7958,7 @@ var getEnabledTools = async (agent, allExuluTools, allContexts, allRerankers, di
|
|
|
7544
7958
|
}
|
|
7545
7959
|
console.log("[EXULU] available tools", enabledTools?.length);
|
|
7546
7960
|
console.log("[EXULU] disabled tools", disabledTools?.length);
|
|
7547
|
-
enabledTools = enabledTools.filter((
|
|
7961
|
+
enabledTools = enabledTools.filter((tool5) => !disabledTools.includes(tool5.id));
|
|
7548
7962
|
return enabledTools;
|
|
7549
7963
|
};
|
|
7550
7964
|
|
|
@@ -7689,7 +8103,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
|
|
|
7689
8103
|
);
|
|
7690
8104
|
if (attempt < retries) {
|
|
7691
8105
|
const backoffMs = 500 * Math.pow(2, attempt - 1);
|
|
7692
|
-
await new Promise((
|
|
8106
|
+
await new Promise((resolve3) => setTimeout(resolve3, backoffMs));
|
|
7693
8107
|
}
|
|
7694
8108
|
}
|
|
7695
8109
|
}
|
|
@@ -7892,7 +8306,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
|
|
|
7892
8306
|
} = await validateWorkflowPayload(data, providers);
|
|
7893
8307
|
const retries2 = 3;
|
|
7894
8308
|
let attempts = 0;
|
|
7895
|
-
const promise = new Promise(async (
|
|
8309
|
+
const promise = new Promise(async (resolve3, reject) => {
|
|
7896
8310
|
while (attempts < retries2) {
|
|
7897
8311
|
try {
|
|
7898
8312
|
const messages2 = await processUiMessagesFlow({
|
|
@@ -7907,7 +8321,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
|
|
|
7907
8321
|
config,
|
|
7908
8322
|
variables: data.inputs
|
|
7909
8323
|
});
|
|
7910
|
-
|
|
8324
|
+
resolve3(messages2);
|
|
7911
8325
|
break;
|
|
7912
8326
|
} catch (error) {
|
|
7913
8327
|
console.error(
|
|
@@ -7918,7 +8332,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
|
|
|
7918
8332
|
if (attempts >= retries2) {
|
|
7919
8333
|
reject(new Error(error instanceof Error ? error.message : String(error)));
|
|
7920
8334
|
}
|
|
7921
|
-
await new Promise((
|
|
8335
|
+
await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
|
|
7922
8336
|
}
|
|
7923
8337
|
}
|
|
7924
8338
|
});
|
|
@@ -7968,7 +8382,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
|
|
|
7968
8382
|
} = await validateEvalPayload(data, providers);
|
|
7969
8383
|
const retries2 = 3;
|
|
7970
8384
|
let attempts = 0;
|
|
7971
|
-
const promise = new Promise(async (
|
|
8385
|
+
const promise = new Promise(async (resolve3, reject) => {
|
|
7972
8386
|
while (attempts < retries2) {
|
|
7973
8387
|
try {
|
|
7974
8388
|
const messages2 = await processUiMessagesFlow({
|
|
@@ -7982,7 +8396,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
|
|
|
7982
8396
|
tools,
|
|
7983
8397
|
config
|
|
7984
8398
|
});
|
|
7985
|
-
|
|
8399
|
+
resolve3(messages2);
|
|
7986
8400
|
break;
|
|
7987
8401
|
} catch (error) {
|
|
7988
8402
|
console.error(
|
|
@@ -7993,7 +8407,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
|
|
|
7993
8407
|
if (attempts >= retries2) {
|
|
7994
8408
|
reject(new Error(error instanceof Error ? error.message : String(error)));
|
|
7995
8409
|
}
|
|
7996
|
-
await new Promise((
|
|
8410
|
+
await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
|
|
7997
8411
|
}
|
|
7998
8412
|
}
|
|
7999
8413
|
});
|
|
@@ -8468,7 +8882,7 @@ var pollJobResult = async ({
|
|
|
8468
8882
|
attempts++;
|
|
8469
8883
|
const job = await import_bullmq3.Job.fromId(queue.queue, jobId);
|
|
8470
8884
|
if (!job) {
|
|
8471
|
-
await new Promise((
|
|
8885
|
+
await new Promise((resolve3) => setTimeout((resolve4) => resolve4(true), 2e3));
|
|
8472
8886
|
continue;
|
|
8473
8887
|
}
|
|
8474
8888
|
const elapsedTime = Date.now() - startTime;
|
|
@@ -8498,7 +8912,7 @@ var pollJobResult = async ({
|
|
|
8498
8912
|
console.log(`[EXULU] eval function ${job.id} result: ${result}`);
|
|
8499
8913
|
break;
|
|
8500
8914
|
}
|
|
8501
|
-
await new Promise((
|
|
8915
|
+
await new Promise((resolve3) => setTimeout(() => resolve3(true), 2e3));
|
|
8502
8916
|
}
|
|
8503
8917
|
return result;
|
|
8504
8918
|
};
|
|
@@ -8606,7 +9020,7 @@ var processUiMessagesFlow = async ({
|
|
|
8606
9020
|
label: agent.name,
|
|
8607
9021
|
trigger: "agent"
|
|
8608
9022
|
};
|
|
8609
|
-
messageHistory = await new Promise(async (
|
|
9023
|
+
messageHistory = await new Promise(async (resolve3, reject) => {
|
|
8610
9024
|
const startTime = Date.now();
|
|
8611
9025
|
try {
|
|
8612
9026
|
const result = await provider.generateStream({
|
|
@@ -8614,7 +9028,7 @@ var processUiMessagesFlow = async ({
|
|
|
8614
9028
|
rerankers,
|
|
8615
9029
|
agent,
|
|
8616
9030
|
user,
|
|
8617
|
-
approvedTools: tools.map((
|
|
9031
|
+
approvedTools: tools.map((tool5) => "tool-" + sanitizeToolName(tool5.name)),
|
|
8618
9032
|
instructions: agent.instructions,
|
|
8619
9033
|
session: void 0,
|
|
8620
9034
|
previousMessages: messageHistory.messages,
|
|
@@ -8683,7 +9097,7 @@ var processUiMessagesFlow = async ({
|
|
|
8683
9097
|
})
|
|
8684
9098
|
] : []
|
|
8685
9099
|
]);
|
|
8686
|
-
|
|
9100
|
+
resolve3({
|
|
8687
9101
|
messages,
|
|
8688
9102
|
metadata: {
|
|
8689
9103
|
tokens: {
|
|
@@ -9431,7 +9845,7 @@ type PageInfo {
|
|
|
9431
9845
|
} = await validateWorkflowPayload(jobData, providers);
|
|
9432
9846
|
const retries = 3;
|
|
9433
9847
|
let attempts = 0;
|
|
9434
|
-
const promise = new Promise(async (
|
|
9848
|
+
const promise = new Promise(async (resolve3, reject) => {
|
|
9435
9849
|
while (attempts < retries) {
|
|
9436
9850
|
try {
|
|
9437
9851
|
const messages2 = await processUiMessagesFlow({
|
|
@@ -9446,7 +9860,7 @@ type PageInfo {
|
|
|
9446
9860
|
config,
|
|
9447
9861
|
variables: args.variables
|
|
9448
9862
|
});
|
|
9449
|
-
|
|
9863
|
+
resolve3(messages2);
|
|
9450
9864
|
break;
|
|
9451
9865
|
} catch (error) {
|
|
9452
9866
|
console.error(
|
|
@@ -9460,7 +9874,7 @@ type PageInfo {
|
|
|
9460
9874
|
if (attempts >= retries) {
|
|
9461
9875
|
reject(error instanceof Error ? error : new Error(String(error)));
|
|
9462
9876
|
}
|
|
9463
|
-
await new Promise((
|
|
9877
|
+
await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
|
|
9464
9878
|
}
|
|
9465
9879
|
}
|
|
9466
9880
|
});
|
|
@@ -9713,10 +10127,10 @@ type PageInfo {
|
|
|
9713
10127
|
contexts.map(async (context2) => {
|
|
9714
10128
|
let processor = null;
|
|
9715
10129
|
if (context2.processor) {
|
|
9716
|
-
processor = await new Promise(async (
|
|
10130
|
+
processor = await new Promise(async (resolve3, reject) => {
|
|
9717
10131
|
const config2 = context2.processor?.config;
|
|
9718
10132
|
const queue = await config2?.queue;
|
|
9719
|
-
|
|
10133
|
+
resolve3({
|
|
9720
10134
|
name: context2.processor.name,
|
|
9721
10135
|
description: context2.processor.description,
|
|
9722
10136
|
queue: queue?.queue?.name || void 0,
|
|
@@ -9797,10 +10211,10 @@ type PageInfo {
|
|
|
9797
10211
|
}
|
|
9798
10212
|
let processor = null;
|
|
9799
10213
|
if (data.processor) {
|
|
9800
|
-
processor = await new Promise(async (
|
|
10214
|
+
processor = await new Promise(async (resolve3, reject) => {
|
|
9801
10215
|
const config2 = data.processor?.config;
|
|
9802
10216
|
const queue = await config2?.queue;
|
|
9803
|
-
|
|
10217
|
+
resolve3({
|
|
9804
10218
|
name: data.processor.name,
|
|
9805
10219
|
description: data.processor.description,
|
|
9806
10220
|
queue: queue?.queue?.name || void 0,
|
|
@@ -9889,7 +10303,7 @@ type PageInfo {
|
|
|
9889
10303
|
})
|
|
9890
10304
|
);
|
|
9891
10305
|
let agenticRetrievalTool = void 0;
|
|
9892
|
-
const filtered = agentTools.filter((
|
|
10306
|
+
const filtered = agentTools.filter((tool5) => tool5 !== null);
|
|
9893
10307
|
let allTools = [...filtered, ...tools];
|
|
9894
10308
|
if (contexts?.length) {
|
|
9895
10309
|
agenticRetrievalTool = createAgenticRetrievalToolV3({
|
|
@@ -9907,21 +10321,21 @@ type PageInfo {
|
|
|
9907
10321
|
if (search && search.trim()) {
|
|
9908
10322
|
const searchTerm = search.toLowerCase().trim();
|
|
9909
10323
|
allTools = allTools.filter(
|
|
9910
|
-
(
|
|
10324
|
+
(tool5) => tool5.name?.toLowerCase().includes(searchTerm) || tool5.description?.toLowerCase().includes(searchTerm)
|
|
9911
10325
|
);
|
|
9912
10326
|
}
|
|
9913
10327
|
if (category && category.trim()) {
|
|
9914
|
-
allTools = allTools.filter((
|
|
10328
|
+
allTools = allTools.filter((tool5) => tool5.category === category);
|
|
9915
10329
|
}
|
|
9916
10330
|
const total = allTools.length;
|
|
9917
10331
|
const start = page * limit;
|
|
9918
10332
|
const end = start + limit;
|
|
9919
10333
|
const paginatedTools = allTools.slice(start, end);
|
|
9920
10334
|
return {
|
|
9921
|
-
items: paginatedTools.map((
|
|
10335
|
+
items: paginatedTools.map((tool5) => {
|
|
9922
10336
|
const object = {};
|
|
9923
10337
|
requestedFields.forEach((field) => {
|
|
9924
|
-
object[field] =
|
|
10338
|
+
object[field] = tool5[field];
|
|
9925
10339
|
});
|
|
9926
10340
|
return object;
|
|
9927
10341
|
}),
|
|
@@ -9931,7 +10345,7 @@ type PageInfo {
|
|
|
9931
10345
|
};
|
|
9932
10346
|
};
|
|
9933
10347
|
resolvers.Query["toolCategories"] = async () => {
|
|
9934
|
-
const array = tools.map((
|
|
10348
|
+
const array = tools.map((tool5) => tool5.category).filter((category) => category && typeof category === "string");
|
|
9935
10349
|
array.push("contexts");
|
|
9936
10350
|
array.push("agents");
|
|
9937
10351
|
return [...new Set(array)].sort();
|
|
@@ -10291,13 +10705,13 @@ type AgentWorldAgent {
|
|
|
10291
10705
|
}
|
|
10292
10706
|
|
|
10293
10707
|
// src/exulu/routes.ts
|
|
10294
|
-
var
|
|
10708
|
+
var import_express52 = require("@as-integrations/express5");
|
|
10295
10709
|
var import_utils5 = require("@apollo/utils.keyvaluecache");
|
|
10296
10710
|
var import_body_parser = __toESM(require("body-parser"), 1);
|
|
10297
|
-
var
|
|
10711
|
+
var import_crypto_js7 = __toESM(require("crypto-js"), 1);
|
|
10298
10712
|
var import_openai = __toESM(require("openai"), 1);
|
|
10299
|
-
var
|
|
10300
|
-
var
|
|
10713
|
+
var import_fs2 = __toESM(require("fs"), 1);
|
|
10714
|
+
var import_node_crypto5 = require("crypto");
|
|
10301
10715
|
var import_api2 = require("@opentelemetry/api");
|
|
10302
10716
|
var import_sdk = __toESM(require("@anthropic-ai/sdk"), 1);
|
|
10303
10717
|
|
|
@@ -10327,11 +10741,11 @@ var CLAUDE_MESSAGES = {
|
|
|
10327
10741
|
};
|
|
10328
10742
|
|
|
10329
10743
|
// src/exulu/routes.ts
|
|
10330
|
-
var
|
|
10744
|
+
var import_ai10 = require("ai");
|
|
10331
10745
|
var import_cookie_parser = __toESM(require("cookie-parser"), 1);
|
|
10332
10746
|
|
|
10333
10747
|
// src/exulu/provider.ts
|
|
10334
|
-
var
|
|
10748
|
+
var import_zod10 = require("zod");
|
|
10335
10749
|
var import_ai8 = require("ai");
|
|
10336
10750
|
|
|
10337
10751
|
// src/utils/generate-slug.ts
|
|
@@ -10389,7 +10803,7 @@ async function clearSessionCurrentTask(session) {
|
|
|
10389
10803
|
}
|
|
10390
10804
|
|
|
10391
10805
|
// src/exulu/provider.ts
|
|
10392
|
-
var
|
|
10806
|
+
var import_fs = __toESM(require("fs"), 1);
|
|
10393
10807
|
var ExuluProvider = class {
|
|
10394
10808
|
// Must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or
|
|
10395
10809
|
// underscores and be a max length of 80 characters and at least 5 characters long.
|
|
@@ -10471,9 +10885,9 @@ var ExuluProvider = class {
|
|
|
10471
10885
|
name: `${agent.name}`,
|
|
10472
10886
|
type: "agent",
|
|
10473
10887
|
category: "agents",
|
|
10474
|
-
inputSchema:
|
|
10475
|
-
prompt:
|
|
10476
|
-
information:
|
|
10888
|
+
inputSchema: import_zod10.z.object({
|
|
10889
|
+
prompt: import_zod10.z.string().describe("The prompt (usually a question for the agent) to send to the agent."),
|
|
10890
|
+
information: import_zod10.z.string().describe("A summary of relevant context / information from the current session")
|
|
10477
10891
|
}),
|
|
10478
10892
|
description: `This tool calls an agent named: ${agent.name}. The agent does the following: ${agent.description}.`,
|
|
10479
10893
|
config: [],
|
|
@@ -10676,12 +11090,12 @@ var ExuluProvider = class {
|
|
|
10676
11090
|
system += "\n\n" + memoryContext;
|
|
10677
11091
|
}
|
|
10678
11092
|
const includesContextSearchTool = currentTools?.some(
|
|
10679
|
-
(
|
|
11093
|
+
(tool5) => tool5.name.toLowerCase().includes("context_search") || tool5.id.includes("context_search") || tool5.type === "context"
|
|
10680
11094
|
);
|
|
10681
11095
|
const includesWebSearchTool = currentTools?.some(
|
|
10682
|
-
(
|
|
11096
|
+
(tool5) => tool5.name.toLowerCase().includes("web_search") || tool5.id.includes("web_search") || tool5.type === "web_search"
|
|
10683
11097
|
);
|
|
10684
|
-
console.log("[EXULU] Current tools: " + currentTools?.map((
|
|
11098
|
+
console.log("[EXULU] Current tools: " + currentTools?.map((tool5) => tool5.name).join("\n"));
|
|
10685
11099
|
console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
|
|
10686
11100
|
if (includesContextSearchTool) {
|
|
10687
11101
|
system += `
|
|
@@ -11054,7 +11468,7 @@ ${extractedText}
|
|
|
11054
11468
|
// todo make this configurable?
|
|
11055
11469
|
page: 1
|
|
11056
11470
|
});
|
|
11057
|
-
|
|
11471
|
+
import_fs.default.writeFileSync("pre-fetched-relevant-information.json", JSON.stringify(result2, null, 2));
|
|
11058
11472
|
if (result2?.chunks?.length) {
|
|
11059
11473
|
memoryContext = `
|
|
11060
11474
|
<pre-fetched relevant information for this query>:
|
|
@@ -11079,12 +11493,12 @@ ${extractedText}
|
|
|
11079
11493
|
system += "\n\n" + memoryContext;
|
|
11080
11494
|
}
|
|
11081
11495
|
const includesContextSearchTool = currentTools?.some(
|
|
11082
|
-
(
|
|
11496
|
+
(tool5) => tool5.name.toLowerCase().includes("context_search") || tool5.id.includes("context_search") || tool5.type === "context"
|
|
11083
11497
|
);
|
|
11084
11498
|
const includesWebSearchTool = currentTools?.some(
|
|
11085
|
-
(
|
|
11499
|
+
(tool5) => tool5.name.toLowerCase().includes("web_search") || tool5.id.includes("web_search") || tool5.type === "web_search"
|
|
11086
11500
|
);
|
|
11087
|
-
console.log("[EXULU] Current tools: " + currentTools?.map((
|
|
11501
|
+
console.log("[EXULU] Current tools: " + currentTools?.map((tool5) => tool5.name).join("\n"));
|
|
11088
11502
|
console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
|
|
11089
11503
|
console.log("[EXULU] Includes web search tool: " + includesWebSearchTool);
|
|
11090
11504
|
if (includesContextSearchTool) {
|
|
@@ -11129,7 +11543,7 @@ ${extractedText}
|
|
|
11129
11543
|
|
|
11130
11544
|
When a tool execution is not approved by the user, do not retry it unless explicitly asked by the user. ' +
|
|
11131
11545
|
'Inform the user that the action was not performed.`;
|
|
11132
|
-
|
|
11546
|
+
import_fs.default.writeFileSync("system-prompt.txt", system);
|
|
11133
11547
|
const result = (0, import_ai8.streamText)({
|
|
11134
11548
|
temperature: 0,
|
|
11135
11549
|
// TODO Make this configurable
|
|
@@ -11274,13 +11688,458 @@ var providerRateLimiter = async (key, windowSeconds, limit, points) => {
|
|
|
11274
11688
|
}
|
|
11275
11689
|
};
|
|
11276
11690
|
|
|
11691
|
+
// src/exulu/openai-gateway.ts
|
|
11692
|
+
var import_express2 = require("express");
|
|
11693
|
+
var import_ai9 = require("ai");
|
|
11694
|
+
var import_node_crypto4 = require("crypto");
|
|
11695
|
+
var import_crypto_js6 = __toESM(require("crypto-js"), 1);
|
|
11696
|
+
var import_express3 = __toESM(require("express"), 1);
|
|
11697
|
+
function convertOpenAIMessagesToCoreMessages(messages) {
|
|
11698
|
+
const systemParts = [];
|
|
11699
|
+
const coreMessages = [];
|
|
11700
|
+
for (const msg of messages) {
|
|
11701
|
+
if (msg.role === "system") {
|
|
11702
|
+
systemParts.push(typeof msg.content === "string" ? msg.content : "");
|
|
11703
|
+
continue;
|
|
11704
|
+
}
|
|
11705
|
+
if (msg.role === "user") {
|
|
11706
|
+
if (typeof msg.content === "string") {
|
|
11707
|
+
coreMessages.push({ role: "user", content: msg.content });
|
|
11708
|
+
} else if (Array.isArray(msg.content)) {
|
|
11709
|
+
const parts = msg.content.flatMap((part) => {
|
|
11710
|
+
if (part.type === "text") return [{ type: "text", text: part.text }];
|
|
11711
|
+
if (part.type === "image_url") return [{ type: "image", image: part.image_url.url }];
|
|
11712
|
+
return [];
|
|
11713
|
+
});
|
|
11714
|
+
coreMessages.push({ role: "user", content: parts });
|
|
11715
|
+
}
|
|
11716
|
+
continue;
|
|
11717
|
+
}
|
|
11718
|
+
if (msg.role === "assistant") {
|
|
11719
|
+
if (msg.tool_calls && msg.tool_calls.length > 0) {
|
|
11720
|
+
const parts = [];
|
|
11721
|
+
if (typeof msg.content === "string" && msg.content) {
|
|
11722
|
+
parts.push({ type: "text", text: msg.content });
|
|
11723
|
+
}
|
|
11724
|
+
for (const tc of msg.tool_calls) {
|
|
11725
|
+
parts.push({
|
|
11726
|
+
type: "tool-call",
|
|
11727
|
+
toolCallId: tc.id,
|
|
11728
|
+
toolName: tc.function.name,
|
|
11729
|
+
args: JSON.parse(tc.function.arguments || "{}")
|
|
11730
|
+
});
|
|
11731
|
+
}
|
|
11732
|
+
coreMessages.push({ role: "assistant", content: parts });
|
|
11733
|
+
} else {
|
|
11734
|
+
coreMessages.push({
|
|
11735
|
+
role: "assistant",
|
|
11736
|
+
content: typeof msg.content === "string" ? msg.content : ""
|
|
11737
|
+
});
|
|
11738
|
+
}
|
|
11739
|
+
continue;
|
|
11740
|
+
}
|
|
11741
|
+
if (msg.role === "tool") {
|
|
11742
|
+
coreMessages.push({
|
|
11743
|
+
role: "tool",
|
|
11744
|
+
content: [
|
|
11745
|
+
{
|
|
11746
|
+
type: "tool-result",
|
|
11747
|
+
toolCallId: msg.tool_call_id ?? "",
|
|
11748
|
+
result: msg.content
|
|
11749
|
+
}
|
|
11750
|
+
]
|
|
11751
|
+
});
|
|
11752
|
+
}
|
|
11753
|
+
}
|
|
11754
|
+
return { systemPrompt: systemParts.join("\n\n"), coreMessages };
|
|
11755
|
+
}
|
|
11756
|
+
async function writeStatistics(agent, project, user, inputTokens, outputTokens) {
|
|
11757
|
+
const label = agent.name;
|
|
11758
|
+
const trigger = "agent";
|
|
11759
|
+
const projectId = project?.id ? { project: project.id } : {};
|
|
11760
|
+
await Promise.all([
|
|
11761
|
+
updateStatistic({
|
|
11762
|
+
name: "count",
|
|
11763
|
+
label,
|
|
11764
|
+
type: STATISTICS_TYPE_ENUM.AGENT_RUN,
|
|
11765
|
+
trigger,
|
|
11766
|
+
count: 1,
|
|
11767
|
+
user: user.id,
|
|
11768
|
+
role: user.role?.id,
|
|
11769
|
+
...projectId
|
|
11770
|
+
}),
|
|
11771
|
+
...inputTokens ? [
|
|
11772
|
+
updateStatistic({
|
|
11773
|
+
name: "inputTokens",
|
|
11774
|
+
label,
|
|
11775
|
+
type: STATISTICS_TYPE_ENUM.AGENT_RUN,
|
|
11776
|
+
trigger,
|
|
11777
|
+
count: inputTokens,
|
|
11778
|
+
user: user.id,
|
|
11779
|
+
role: user.role?.id,
|
|
11780
|
+
...projectId
|
|
11781
|
+
})
|
|
11782
|
+
] : [],
|
|
11783
|
+
...outputTokens ? [
|
|
11784
|
+
updateStatistic({
|
|
11785
|
+
name: "outputTokens",
|
|
11786
|
+
label,
|
|
11787
|
+
type: STATISTICS_TYPE_ENUM.AGENT_RUN,
|
|
11788
|
+
trigger,
|
|
11789
|
+
count: outputTokens,
|
|
11790
|
+
user: user.id,
|
|
11791
|
+
role: user.role?.id,
|
|
11792
|
+
...projectId
|
|
11793
|
+
})
|
|
11794
|
+
] : []
|
|
11795
|
+
]);
|
|
11796
|
+
}
|
|
11797
|
+
var registerOpenAIGatewayRoutes = async (app, providers, tools, contexts, config, rerankers) => {
|
|
11798
|
+
const { agentsSchema: agentsSchema4, projectsSchema: projectsSchema4 } = coreSchemas.get();
|
|
11799
|
+
app.get(
|
|
11800
|
+
"/gateway/open-ai/v1/models",
|
|
11801
|
+
async (req, res) => {
|
|
11802
|
+
try {
|
|
11803
|
+
const authResult = await requestValidators.authenticate(req);
|
|
11804
|
+
if (!authResult.user?.id) {
|
|
11805
|
+
res.status(authResult.code || 401).json({ error: { message: authResult.message, type: "authentication_error" } });
|
|
11806
|
+
return;
|
|
11807
|
+
}
|
|
11808
|
+
const { db: db2 } = await postgresClient();
|
|
11809
|
+
let projectsQuery = db2("projects").select("id", "name");
|
|
11810
|
+
projectsQuery = applyAccessControl(projectsSchema4(), projectsQuery, authResult.user);
|
|
11811
|
+
const projects = await projectsQuery;
|
|
11812
|
+
let agentsQuery = db2("agents").select("id", "name");
|
|
11813
|
+
agentsQuery = applyAccessControl(agentsSchema4(), agentsQuery, authResult.user);
|
|
11814
|
+
const agents = await agentsQuery;
|
|
11815
|
+
const data = projects.flatMap(
|
|
11816
|
+
(p) => agents.map((a) => ({
|
|
11817
|
+
id: `${p.name}/${a.name}`,
|
|
11818
|
+
object: "model",
|
|
11819
|
+
created: 0,
|
|
11820
|
+
owned_by: "exulu"
|
|
11821
|
+
}))
|
|
11822
|
+
);
|
|
11823
|
+
res.json({ object: "list", data });
|
|
11824
|
+
} catch (error) {
|
|
11825
|
+
console.error("[OPENAI GATEWAY] /v1/models error:", error);
|
|
11826
|
+
res.status(500).json({ error: { message: error.message, type: "server_error" } });
|
|
11827
|
+
}
|
|
11828
|
+
}
|
|
11829
|
+
);
|
|
11830
|
+
app.get(
|
|
11831
|
+
"/gateway/open-ai/v1/models/:projectId/:agentId",
|
|
11832
|
+
async (req, res) => {
|
|
11833
|
+
try {
|
|
11834
|
+
const authResult = await requestValidators.authenticate(req);
|
|
11835
|
+
if (!authResult.user?.id) {
|
|
11836
|
+
res.status(authResult.code || 401).json({ error: { message: authResult.message, type: "authentication_error" } });
|
|
11837
|
+
return;
|
|
11838
|
+
}
|
|
11839
|
+
const { db: db2 } = await postgresClient();
|
|
11840
|
+
let projectQuery = db2("projects").select("id", "name");
|
|
11841
|
+
projectQuery = applyAccessControl(projectsSchema4(), projectQuery, authResult.user);
|
|
11842
|
+
projectQuery.where({ id: req.params.projectId });
|
|
11843
|
+
const project = await projectQuery.first();
|
|
11844
|
+
let agentQuery = db2("agents").select("id", "name");
|
|
11845
|
+
agentQuery = applyAccessControl(agentsSchema4(), agentQuery, authResult.user);
|
|
11846
|
+
agentQuery.where({ id: req.params.agentId });
|
|
11847
|
+
const agent = await agentQuery.first();
|
|
11848
|
+
if (!project || !agent) {
|
|
11849
|
+
res.status(404).json({ error: { message: "Model not found", type: "invalid_request_error" } });
|
|
11850
|
+
return;
|
|
11851
|
+
}
|
|
11852
|
+
res.json({
|
|
11853
|
+
id: `${project.name}/${agent.name}`,
|
|
11854
|
+
object: "model",
|
|
11855
|
+
created: 0,
|
|
11856
|
+
owned_by: "exulu"
|
|
11857
|
+
});
|
|
11858
|
+
} catch (error) {
|
|
11859
|
+
console.error("[OPENAI GATEWAY] /v1/models/:id error:", error);
|
|
11860
|
+
res.status(500).json({ error: { message: error.message, type: "server_error" } });
|
|
11861
|
+
}
|
|
11862
|
+
}
|
|
11863
|
+
);
|
|
11864
|
+
app.post(
|
|
11865
|
+
"/gateway/open-ai/v1/chat/completions",
|
|
11866
|
+
import_express3.default.json({ limit: REQUEST_SIZE_LIMIT }),
|
|
11867
|
+
async (req, res) => {
|
|
11868
|
+
try {
|
|
11869
|
+
const { db: db2 } = await postgresClient();
|
|
11870
|
+
const authResult = await requestValidators.authenticate(req);
|
|
11871
|
+
if (!authResult.user?.id) {
|
|
11872
|
+
res.status(authResult.code || 401).json({ error: { message: authResult.message, type: "authentication_error" } });
|
|
11873
|
+
return;
|
|
11874
|
+
}
|
|
11875
|
+
const user = authResult.user;
|
|
11876
|
+
const modelId = req.body.model;
|
|
11877
|
+
if (!modelId) {
|
|
11878
|
+
res.status(400).json({
|
|
11879
|
+
error: { message: "Missing required field: model", type: "invalid_request_error" }
|
|
11880
|
+
});
|
|
11881
|
+
return;
|
|
11882
|
+
}
|
|
11883
|
+
const separatorIndex = modelId.indexOf("/");
|
|
11884
|
+
if (separatorIndex === -1) {
|
|
11885
|
+
res.status(400).json({
|
|
11886
|
+
error: { message: "Invalid model format. Expected: 'projectname/agentname'", type: "invalid_request_error" }
|
|
11887
|
+
});
|
|
11888
|
+
return;
|
|
11889
|
+
}
|
|
11890
|
+
const projectName = modelId.substring(0, separatorIndex);
|
|
11891
|
+
const agentName = modelId.substring(separatorIndex + 1);
|
|
11892
|
+
let agentQuery = db2("agents").select("*");
|
|
11893
|
+
agentQuery = applyAccessControl(agentsSchema4(), agentQuery, user);
|
|
11894
|
+
agentQuery.where({ name: agentName });
|
|
11895
|
+
const agent = await agentQuery.first();
|
|
11896
|
+
if (!agent) {
|
|
11897
|
+
res.status(404).json({
|
|
11898
|
+
error: {
|
|
11899
|
+
message: `Agent '${agentName}' not found or you do not have access to it.`,
|
|
11900
|
+
type: "invalid_request_error"
|
|
11901
|
+
}
|
|
11902
|
+
});
|
|
11903
|
+
return;
|
|
11904
|
+
}
|
|
11905
|
+
let project = null;
|
|
11906
|
+
if (projectName) {
|
|
11907
|
+
let projectQuery = db2("projects").select("*");
|
|
11908
|
+
projectQuery = applyAccessControl(projectsSchema4(), projectQuery, user);
|
|
11909
|
+
projectQuery.where({ name: projectName });
|
|
11910
|
+
project = await projectQuery.first();
|
|
11911
|
+
}
|
|
11912
|
+
if (!process.env.NEXTAUTH_SECRET) {
|
|
11913
|
+
res.status(500).json({ error: { message: "Server configuration error", type: "server_error" } });
|
|
11914
|
+
return;
|
|
11915
|
+
}
|
|
11916
|
+
if (!agent.providerapikey) {
|
|
11917
|
+
res.status(400).json({
|
|
11918
|
+
error: { message: "Agent has no API key configured", type: "invalid_request_error" }
|
|
11919
|
+
});
|
|
11920
|
+
return;
|
|
11921
|
+
}
|
|
11922
|
+
const variable = await db2.from("variables").where({ name: agent.providerapikey }).first();
|
|
11923
|
+
if (!variable) {
|
|
11924
|
+
res.status(400).json({
|
|
11925
|
+
error: { message: "API key variable not found", type: "invalid_request_error" }
|
|
11926
|
+
});
|
|
11927
|
+
return;
|
|
11928
|
+
}
|
|
11929
|
+
if (!variable.encrypted) {
|
|
11930
|
+
res.status(400).json({
|
|
11931
|
+
error: { message: "API key variable must be encrypted", type: "invalid_request_error" }
|
|
11932
|
+
});
|
|
11933
|
+
return;
|
|
11934
|
+
}
|
|
11935
|
+
const bytes = import_crypto_js6.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
11936
|
+
const providerapikey = bytes.toString(import_crypto_js6.default.enc.Utf8);
|
|
11937
|
+
const provider = providers.find((p) => p.id === agent.provider);
|
|
11938
|
+
if (!provider?.config?.model?.create) {
|
|
11939
|
+
res.status(400).json({
|
|
11940
|
+
error: { message: "No provider configured for this agent", type: "invalid_request_error" }
|
|
11941
|
+
});
|
|
11942
|
+
return;
|
|
11943
|
+
}
|
|
11944
|
+
const languageModel = provider.config.model.create({ apiKey: providerapikey });
|
|
11945
|
+
const disabledTools = req.body.disabledTools ?? [];
|
|
11946
|
+
const enabledTools = await getEnabledTools(
|
|
11947
|
+
agent,
|
|
11948
|
+
tools,
|
|
11949
|
+
contexts ?? [],
|
|
11950
|
+
rerankers ?? [],
|
|
11951
|
+
disabledTools,
|
|
11952
|
+
providers,
|
|
11953
|
+
user
|
|
11954
|
+
);
|
|
11955
|
+
const convertedTools = await convertExuluToolsToAiSdkTools(
|
|
11956
|
+
enabledTools,
|
|
11957
|
+
[],
|
|
11958
|
+
tools,
|
|
11959
|
+
agent.tools,
|
|
11960
|
+
providerapikey,
|
|
11961
|
+
contexts,
|
|
11962
|
+
rerankers,
|
|
11963
|
+
user,
|
|
11964
|
+
config,
|
|
11965
|
+
void 0,
|
|
11966
|
+
req,
|
|
11967
|
+
project?.id,
|
|
11968
|
+
void 0,
|
|
11969
|
+
languageModel,
|
|
11970
|
+
agent
|
|
11971
|
+
);
|
|
11972
|
+
const openaiMessages = req.body.messages ?? [];
|
|
11973
|
+
const { systemPrompt: requestSystemPrompt, coreMessages } = convertOpenAIMessagesToCoreMessages(openaiMessages);
|
|
11974
|
+
const agentInstructions = agent.instructions ?? "";
|
|
11975
|
+
const systemParts = [
|
|
11976
|
+
agentInstructions ? `You are an agent named: ${agent.name}
|
|
11977
|
+
Here are your instructions: ${agentInstructions}` : `You are an agent named: ${agent.name}`,
|
|
11978
|
+
project ? `The project you are working on is: ${project.name}${project.description ? `
|
|
11979
|
+
${project.description}` : ""}` : "",
|
|
11980
|
+
requestSystemPrompt
|
|
11981
|
+
].filter(Boolean);
|
|
11982
|
+
const systemPrompt = systemParts.join("\n\n");
|
|
11983
|
+
const completionId = `chatcmpl-${(0, import_node_crypto4.randomUUID)()}`;
|
|
11984
|
+
const created = Math.floor(Date.now() / 1e3);
|
|
11985
|
+
const hasTools = Object.keys(convertedTools).length > 0;
|
|
11986
|
+
if (req.body.stream === true) {
|
|
11987
|
+
res.setHeader("Content-Type", "text/event-stream");
|
|
11988
|
+
res.setHeader("Cache-Control", "no-cache");
|
|
11989
|
+
res.setHeader("Connection", "keep-alive");
|
|
11990
|
+
const result = (0, import_ai9.streamText)({
|
|
11991
|
+
model: languageModel,
|
|
11992
|
+
system: systemPrompt || void 0,
|
|
11993
|
+
messages: coreMessages,
|
|
11994
|
+
tools: hasTools ? convertedTools : void 0,
|
|
11995
|
+
maxRetries: 2,
|
|
11996
|
+
stopWhen: [(0, import_ai9.stepCountIs)(5)],
|
|
11997
|
+
onError: (error) => {
|
|
11998
|
+
console.error("[OPENAI GATEWAY] stream error:", error);
|
|
11999
|
+
}
|
|
12000
|
+
});
|
|
12001
|
+
res.write(
|
|
12002
|
+
`data: ${JSON.stringify({
|
|
12003
|
+
id: completionId,
|
|
12004
|
+
object: "chat.completion.chunk",
|
|
12005
|
+
created,
|
|
12006
|
+
model: modelId,
|
|
12007
|
+
choices: [{ index: 0, delta: { role: "assistant", content: "" }, finish_reason: null }]
|
|
12008
|
+
})}
|
|
12009
|
+
|
|
12010
|
+
`
|
|
12011
|
+
);
|
|
12012
|
+
let inputTokens = 0;
|
|
12013
|
+
let outputTokens = 0;
|
|
12014
|
+
for await (const chunk of result.fullStream) {
|
|
12015
|
+
if (chunk.type === "text-delta") {
|
|
12016
|
+
res.write(
|
|
12017
|
+
`data: ${JSON.stringify({
|
|
12018
|
+
id: completionId,
|
|
12019
|
+
object: "chat.completion.chunk",
|
|
12020
|
+
created,
|
|
12021
|
+
model: modelId,
|
|
12022
|
+
choices: [{ index: 0, delta: { content: chunk.text }, finish_reason: null }]
|
|
12023
|
+
})}
|
|
12024
|
+
|
|
12025
|
+
`
|
|
12026
|
+
);
|
|
12027
|
+
} else if (chunk.type === "tool-input-start") {
|
|
12028
|
+
res.write(
|
|
12029
|
+
`data: ${JSON.stringify({
|
|
12030
|
+
id: completionId,
|
|
12031
|
+
object: "chat.completion.chunk",
|
|
12032
|
+
created,
|
|
12033
|
+
model: modelId,
|
|
12034
|
+
choices: [
|
|
12035
|
+
{
|
|
12036
|
+
index: 0,
|
|
12037
|
+
delta: {
|
|
12038
|
+
tool_calls: [
|
|
12039
|
+
{
|
|
12040
|
+
index: 0,
|
|
12041
|
+
id: chunk.id,
|
|
12042
|
+
type: "function",
|
|
12043
|
+
function: { name: chunk.toolName, arguments: "" }
|
|
12044
|
+
}
|
|
12045
|
+
]
|
|
12046
|
+
},
|
|
12047
|
+
finish_reason: null
|
|
12048
|
+
}
|
|
12049
|
+
]
|
|
12050
|
+
})}
|
|
12051
|
+
|
|
12052
|
+
`
|
|
12053
|
+
);
|
|
12054
|
+
} else if (chunk.type === "tool-input-delta") {
|
|
12055
|
+
res.write(
|
|
12056
|
+
`data: ${JSON.stringify({
|
|
12057
|
+
id: completionId,
|
|
12058
|
+
object: "chat.completion.chunk",
|
|
12059
|
+
created,
|
|
12060
|
+
model: modelId,
|
|
12061
|
+
choices: [
|
|
12062
|
+
{
|
|
12063
|
+
index: 0,
|
|
12064
|
+
delta: { tool_calls: [{ index: 0, function: { arguments: chunk.delta } }] },
|
|
12065
|
+
finish_reason: null
|
|
12066
|
+
}
|
|
12067
|
+
]
|
|
12068
|
+
})}
|
|
12069
|
+
|
|
12070
|
+
`
|
|
12071
|
+
);
|
|
12072
|
+
} else if (chunk.type === "finish") {
|
|
12073
|
+
inputTokens = chunk.usage?.inputTokens ?? 0;
|
|
12074
|
+
outputTokens = chunk.usage?.outputTokens ?? 0;
|
|
12075
|
+
const finishReason = chunk.finishReason === "tool-calls" ? "tool_calls" : "stop";
|
|
12076
|
+
res.write(
|
|
12077
|
+
`data: ${JSON.stringify({
|
|
12078
|
+
id: completionId,
|
|
12079
|
+
object: "chat.completion.chunk",
|
|
12080
|
+
created,
|
|
12081
|
+
model: modelId,
|
|
12082
|
+
choices: [{ index: 0, delta: {}, finish_reason: finishReason }],
|
|
12083
|
+
usage: {
|
|
12084
|
+
prompt_tokens: inputTokens,
|
|
12085
|
+
completion_tokens: outputTokens,
|
|
12086
|
+
total_tokens: inputTokens + outputTokens
|
|
12087
|
+
}
|
|
12088
|
+
})}
|
|
12089
|
+
|
|
12090
|
+
`
|
|
12091
|
+
);
|
|
12092
|
+
}
|
|
12093
|
+
}
|
|
12094
|
+
res.write("data: [DONE]\n\n");
|
|
12095
|
+
res.end();
|
|
12096
|
+
await writeStatistics(agent, project, user, inputTokens, outputTokens);
|
|
12097
|
+
} else {
|
|
12098
|
+
const { text, usage } = await (0, import_ai9.generateText)({
|
|
12099
|
+
model: languageModel,
|
|
12100
|
+
system: systemPrompt || void 0,
|
|
12101
|
+
messages: coreMessages,
|
|
12102
|
+
tools: hasTools ? convertedTools : void 0,
|
|
12103
|
+
maxRetries: 2,
|
|
12104
|
+
stopWhen: [(0, import_ai9.stepCountIs)(5)]
|
|
12105
|
+
});
|
|
12106
|
+
res.json({
|
|
12107
|
+
id: completionId,
|
|
12108
|
+
object: "chat.completion",
|
|
12109
|
+
created,
|
|
12110
|
+
model: agentId,
|
|
12111
|
+
choices: [
|
|
12112
|
+
{
|
|
12113
|
+
index: 0,
|
|
12114
|
+
message: { role: "assistant", content: text },
|
|
12115
|
+
finish_reason: "stop"
|
|
12116
|
+
}
|
|
12117
|
+
],
|
|
12118
|
+
usage: {
|
|
12119
|
+
prompt_tokens: usage.promptTokens,
|
|
12120
|
+
completion_tokens: usage.completionTokens,
|
|
12121
|
+
total_tokens: usage.totalTokens
|
|
12122
|
+
}
|
|
12123
|
+
});
|
|
12124
|
+
await writeStatistics(agent, project, user, usage.promptTokens, usage.completionTokens);
|
|
12125
|
+
}
|
|
12126
|
+
} catch (error) {
|
|
12127
|
+
console.error("[OPENAI GATEWAY] /v1/chat/completions error:", error);
|
|
12128
|
+
if (!res.headersSent) {
|
|
12129
|
+
res.status(500).json({ error: { message: error.message, type: "server_error" } });
|
|
12130
|
+
}
|
|
12131
|
+
}
|
|
12132
|
+
}
|
|
12133
|
+
);
|
|
12134
|
+
};
|
|
12135
|
+
|
|
11277
12136
|
// src/exulu/routes.ts
|
|
11278
12137
|
var import_zod_from_json_schema = require("zod-from-json-schema");
|
|
11279
12138
|
var REQUEST_SIZE_LIMIT = "50mb";
|
|
11280
12139
|
var getExuluVersionNumber = async () => {
|
|
11281
12140
|
try {
|
|
11282
|
-
const
|
|
11283
|
-
const packageJson =
|
|
12141
|
+
const path3 = process.cwd();
|
|
12142
|
+
const packageJson = import_fs2.default.readFileSync(path3 + "/package.json", "utf8");
|
|
11284
12143
|
const packageData = JSON.parse(packageJson);
|
|
11285
12144
|
const exuluVersion = packageData.dependencies["@exulu/backend"];
|
|
11286
12145
|
console.log(`[EXULU] Installed exulu-backend version: ${exuluVersion}`);
|
|
@@ -11305,6 +12164,7 @@ var {
|
|
|
11305
12164
|
agentMessagesSchema: agentMessagesSchema2,
|
|
11306
12165
|
rolesSchema: rolesSchema2,
|
|
11307
12166
|
usersSchema: usersSchema2,
|
|
12167
|
+
skillsSchema: skillsSchema2,
|
|
11308
12168
|
variablesSchema: variablesSchema2,
|
|
11309
12169
|
workflowTemplatesSchema: workflowTemplatesSchema2,
|
|
11310
12170
|
rbacSchema: rbacSchema2,
|
|
@@ -11324,7 +12184,7 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
|
|
|
11324
12184
|
if (tracer) {
|
|
11325
12185
|
console.log("[EXULU] tracer configured", tracer);
|
|
11326
12186
|
}
|
|
11327
|
-
app.use(
|
|
12187
|
+
app.use(import_express5.default.json({ limit: REQUEST_SIZE_LIMIT }));
|
|
11328
12188
|
app.use((0, import_cors.default)(corsOptions));
|
|
11329
12189
|
app.use(import_body_parser.default.urlencoded({ extended: true, limit: REQUEST_SIZE_LIMIT }));
|
|
11330
12190
|
app.use(import_body_parser.default.json({ limit: REQUEST_SIZE_LIMIT }));
|
|
@@ -11349,6 +12209,7 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
|
|
|
11349
12209
|
const schema = createSDL(
|
|
11350
12210
|
[
|
|
11351
12211
|
usersSchema2(),
|
|
12212
|
+
skillsSchema2(),
|
|
11352
12213
|
rolesSchema2(),
|
|
11353
12214
|
agentsSchema2(),
|
|
11354
12215
|
feedbackSchema2(),
|
|
@@ -11384,8 +12245,8 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
|
|
|
11384
12245
|
app.use(
|
|
11385
12246
|
"/graphql",
|
|
11386
12247
|
(0, import_cors.default)(corsOptions),
|
|
11387
|
-
|
|
11388
|
-
(0,
|
|
12248
|
+
import_express5.default.json({ limit: REQUEST_SIZE_LIMIT }),
|
|
12249
|
+
(0, import_express52.expressMiddleware)(server, {
|
|
11389
12250
|
context: async ({ req }) => {
|
|
11390
12251
|
const authenticationResult = await requestValidators.authenticate(req);
|
|
11391
12252
|
if (!authenticationResult.user?.id) {
|
|
@@ -11506,8 +12367,8 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
|
|
|
11506
12367
|
return;
|
|
11507
12368
|
}
|
|
11508
12369
|
if (variable.encrypted) {
|
|
11509
|
-
const bytes =
|
|
11510
|
-
providerapikey = bytes.toString(
|
|
12370
|
+
const bytes = import_crypto_js7.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
12371
|
+
providerapikey = bytes.toString(import_crypto_js7.default.enc.Utf8);
|
|
11511
12372
|
}
|
|
11512
12373
|
const openai = new import_openai.default({
|
|
11513
12374
|
apiKey: providerapikey
|
|
@@ -11556,7 +12417,7 @@ Mood: friendly and intelligent.
|
|
|
11556
12417
|
});
|
|
11557
12418
|
return;
|
|
11558
12419
|
}
|
|
11559
|
-
const uuid = (0,
|
|
12420
|
+
const uuid = (0, import_node_crypto5.randomUUID)();
|
|
11560
12421
|
const image_url = await uploadFile(Buffer.from(image_base64, "base64"), `${uuid}.png`, config, {
|
|
11561
12422
|
contentType: "image/png"
|
|
11562
12423
|
}, authenticationResult.user?.id, void 0, true);
|
|
@@ -11725,8 +12586,8 @@ Mood: friendly and intelligent.
|
|
|
11725
12586
|
return;
|
|
11726
12587
|
}
|
|
11727
12588
|
if (variable.encrypted) {
|
|
11728
|
-
const bytes =
|
|
11729
|
-
providerapikey = bytes.toString(
|
|
12589
|
+
const bytes = import_crypto_js7.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
12590
|
+
providerapikey = bytes.toString(import_crypto_js7.default.enc.Utf8);
|
|
11730
12591
|
}
|
|
11731
12592
|
}
|
|
11732
12593
|
if (!!headers.stream) {
|
|
@@ -11794,7 +12655,7 @@ ${customInstructions}` : agent.instructions;
|
|
|
11794
12655
|
}
|
|
11795
12656
|
return JSON.stringify(error);
|
|
11796
12657
|
},
|
|
11797
|
-
generateMessageId: (0,
|
|
12658
|
+
generateMessageId: (0, import_ai10.createIdGenerator)({
|
|
11798
12659
|
prefix: "msg_",
|
|
11799
12660
|
size: 16
|
|
11800
12661
|
}),
|
|
@@ -11903,7 +12764,7 @@ ${customInstructions}` : agent.instructions;
|
|
|
11903
12764
|
});
|
|
11904
12765
|
app.use(
|
|
11905
12766
|
"/gateway/anthropic/:agent/:project",
|
|
11906
|
-
|
|
12767
|
+
import_express5.default.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }),
|
|
11907
12768
|
async (req, res) => {
|
|
11908
12769
|
try {
|
|
11909
12770
|
if (!req.body.tools) {
|
|
@@ -11987,8 +12848,8 @@ ${customInstructions}` : agent.instructions;
|
|
|
11987
12848
|
return;
|
|
11988
12849
|
}
|
|
11989
12850
|
if (variable.encrypted) {
|
|
11990
|
-
const bytes =
|
|
11991
|
-
anthropicApiKey = bytes.toString(
|
|
12851
|
+
const bytes = import_crypto_js7.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
12852
|
+
anthropicApiKey = bytes.toString(import_crypto_js7.default.enc.Utf8);
|
|
11992
12853
|
}
|
|
11993
12854
|
const headers = {
|
|
11994
12855
|
"x-api-key": anthropicApiKey,
|
|
@@ -12067,14 +12928,14 @@ ${customInstructions}` : agent?.instructions;
|
|
|
12067
12928
|
console.log("[EXULU] Using tool", toolName);
|
|
12068
12929
|
const inputs = event.message?.input;
|
|
12069
12930
|
const id = event.message?.id;
|
|
12070
|
-
const
|
|
12071
|
-
(
|
|
12931
|
+
const tool5 = enabledTools.find(
|
|
12932
|
+
(tool6) => tool6.id === toolName.replace("exulu_", "")
|
|
12072
12933
|
);
|
|
12073
|
-
if (!
|
|
12934
|
+
if (!tool5 || !tool5.tool.execute) {
|
|
12074
12935
|
console.error("[EXULU] Tool not found or not enabled.", toolName);
|
|
12075
12936
|
continue;
|
|
12076
12937
|
}
|
|
12077
|
-
const toolResult = await
|
|
12938
|
+
const toolResult = await tool5.tool.execute(inputs, {
|
|
12078
12939
|
toolCallId: id,
|
|
12079
12940
|
messages: [
|
|
12080
12941
|
{
|
|
@@ -12166,33 +13027,437 @@ data: ${JSON.stringify(event)}
|
|
|
12166
13027
|
}
|
|
12167
13028
|
}
|
|
12168
13029
|
);
|
|
12169
|
-
|
|
12170
|
-
|
|
12171
|
-
|
|
12172
|
-
|
|
12173
|
-
|
|
12174
|
-
|
|
12175
|
-
|
|
12176
|
-
|
|
12177
|
-
|
|
12178
|
-
|
|
13030
|
+
function buildFileTree(files, stripPrefix) {
|
|
13031
|
+
const root = { name: "/", path: "/", key: "", type: "folder", children: [] };
|
|
13032
|
+
for (const file of files) {
|
|
13033
|
+
const relativePath = file.key.startsWith(stripPrefix) ? file.key.slice(stripPrefix.length) : file.key;
|
|
13034
|
+
const parts = relativePath.split("/").filter(Boolean);
|
|
13035
|
+
let current = root;
|
|
13036
|
+
for (let i = 0; i < parts.length; i++) {
|
|
13037
|
+
const part = parts[i];
|
|
13038
|
+
const isFile = i === parts.length - 1;
|
|
13039
|
+
const existingChild = current.children?.find((c) => c.name === part);
|
|
13040
|
+
if (existingChild) {
|
|
13041
|
+
current = existingChild;
|
|
13042
|
+
} else {
|
|
13043
|
+
const nodePath = "/" + parts.slice(0, i + 1).join("/");
|
|
13044
|
+
const node = isFile ? {
|
|
13045
|
+
name: part,
|
|
13046
|
+
path: nodePath,
|
|
13047
|
+
key: file.key,
|
|
13048
|
+
type: "file",
|
|
13049
|
+
size: file.size,
|
|
13050
|
+
lastModified: file.lastModified
|
|
13051
|
+
} : { name: part, path: nodePath, key: "", type: "folder", children: [] };
|
|
13052
|
+
current.children = current.children ?? [];
|
|
13053
|
+
current.children.push(node);
|
|
13054
|
+
current = node;
|
|
13055
|
+
}
|
|
12179
13056
|
}
|
|
12180
|
-
|
|
12181
|
-
|
|
12182
|
-
|
|
12183
|
-
|
|
12184
|
-
|
|
12185
|
-
|
|
12186
|
-
|
|
12187
|
-
|
|
12188
|
-
|
|
12189
|
-
|
|
12190
|
-
|
|
12191
|
-
|
|
12192
|
-
|
|
12193
|
-
|
|
12194
|
-
|
|
12195
|
-
|
|
13057
|
+
}
|
|
13058
|
+
return root;
|
|
13059
|
+
}
|
|
13060
|
+
app.post("/skills/:skillId/init", async (req, res) => {
|
|
13061
|
+
const authResult = await requestValidators.authenticate(req);
|
|
13062
|
+
if (!authResult.user?.id) {
|
|
13063
|
+
res.status(authResult.code ?? 401).json({ detail: authResult.message });
|
|
13064
|
+
return;
|
|
13065
|
+
}
|
|
13066
|
+
const { skillId } = req.params;
|
|
13067
|
+
const { name = "Skill", description = "" } = req.body;
|
|
13068
|
+
const skillMdContent = [
|
|
13069
|
+
`# ${name}`,
|
|
13070
|
+
"",
|
|
13071
|
+
description || "Describe what this skill does and when to use it.",
|
|
13072
|
+
"",
|
|
13073
|
+
"## Overview",
|
|
13074
|
+
"",
|
|
13075
|
+
"...",
|
|
13076
|
+
"",
|
|
13077
|
+
"## Usage",
|
|
13078
|
+
"",
|
|
13079
|
+
"..."
|
|
13080
|
+
].join("\n");
|
|
13081
|
+
const s3Key = `skills/${skillId}/v1/SKILL.md`;
|
|
13082
|
+
try {
|
|
13083
|
+
await uploadFile(Buffer.from(skillMdContent, "utf-8"), s3Key, config, { contentType: "text/markdown" }, void 0, void 0, true);
|
|
13084
|
+
} catch (err) {
|
|
13085
|
+
console.error("[SKILLS] Failed to create SKILL.md in S3", err);
|
|
13086
|
+
res.status(500).json({ detail: "Failed to initialise skill folder in S3." });
|
|
13087
|
+
return;
|
|
13088
|
+
}
|
|
13089
|
+
const { db: db2 } = await postgresClient();
|
|
13090
|
+
await db2("skills").where({ id: skillId }).update({
|
|
13091
|
+
s3folder: `skills/${skillId}`,
|
|
13092
|
+
current_version: 1,
|
|
13093
|
+
history: JSON.stringify([
|
|
13094
|
+
{ version: 1, created_at: (/* @__PURE__ */ new Date()).toISOString(), label: "Initial" }
|
|
13095
|
+
])
|
|
13096
|
+
});
|
|
13097
|
+
res.json({ version: 1, skillMdKey: s3Key });
|
|
13098
|
+
});
|
|
13099
|
+
app.get("/skills/:skillId/files", async (req, res) => {
|
|
13100
|
+
const authResult = await requestValidators.authenticate(req);
|
|
13101
|
+
if (!authResult.user?.id) {
|
|
13102
|
+
res.status(authResult.code ?? 401).json({ detail: authResult.message });
|
|
13103
|
+
return;
|
|
13104
|
+
}
|
|
13105
|
+
const { skillId } = req.params;
|
|
13106
|
+
const { db: db2 } = await postgresClient();
|
|
13107
|
+
const skill = await db2("skills").where({ id: skillId }).first();
|
|
13108
|
+
if (!skill) {
|
|
13109
|
+
res.status(404).json({ detail: "Skill not found." });
|
|
13110
|
+
return;
|
|
13111
|
+
}
|
|
13112
|
+
const version = req.query.version ? Number(req.query.version) : skill.current_version ?? 1;
|
|
13113
|
+
const prefix = `skills/${skillId}/v${version}/`;
|
|
13114
|
+
const files = await listS3ObjectsByPrefix(prefix, config);
|
|
13115
|
+
const tree = buildFileTree(files, config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` + prefix : prefix);
|
|
13116
|
+
res.json({ version, tree, fileCount: files.length });
|
|
13117
|
+
});
|
|
13118
|
+
app.post("/skills/:skillId/sign", async (req, res) => {
|
|
13119
|
+
const authResult = await requestValidators.authenticate(req);
|
|
13120
|
+
if (!authResult.user?.id) {
|
|
13121
|
+
res.status(authResult.code ?? 401).json({ detail: authResult.message });
|
|
13122
|
+
return;
|
|
13123
|
+
}
|
|
13124
|
+
const { skillId } = req.params;
|
|
13125
|
+
const { filePath, contentType = "application/octet-stream" } = req.body;
|
|
13126
|
+
if (!filePath || typeof filePath !== "string") {
|
|
13127
|
+
res.status(400).json({ detail: "Missing filePath in request body." });
|
|
13128
|
+
return;
|
|
13129
|
+
}
|
|
13130
|
+
const { db: db2 } = await postgresClient();
|
|
13131
|
+
const skill = await db2("skills").where({ id: skillId }).first();
|
|
13132
|
+
if (!skill) {
|
|
13133
|
+
res.status(404).json({ detail: "Skill not found." });
|
|
13134
|
+
return;
|
|
13135
|
+
}
|
|
13136
|
+
const version = skill.current_version ?? 1;
|
|
13137
|
+
const safePath = filePath.replace(/^\/+/, "").replace(/\.\.\//g, "");
|
|
13138
|
+
const s3Key = `skills/${skillId}/v${version}/${safePath}`;
|
|
13139
|
+
const fullKey = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/${s3Key}` : s3Key;
|
|
13140
|
+
const url = await getS3SignedUploadUrl(fullKey, contentType, config);
|
|
13141
|
+
res.json({ key: s3Key, url, method: "PUT" });
|
|
13142
|
+
});
|
|
13143
|
+
app.get("/skills/:skillId/file", async (req, res) => {
|
|
13144
|
+
const authResult = await requestValidators.authenticate(req);
|
|
13145
|
+
if (!authResult.user?.id) {
|
|
13146
|
+
res.status(authResult.code ?? 401).json({ detail: authResult.message });
|
|
13147
|
+
return;
|
|
13148
|
+
}
|
|
13149
|
+
const { skillId } = req.params;
|
|
13150
|
+
const { key } = req.query;
|
|
13151
|
+
if (!key || typeof key !== "string") {
|
|
13152
|
+
res.status(400).json({ detail: "Missing key query parameter." });
|
|
13153
|
+
return;
|
|
13154
|
+
}
|
|
13155
|
+
if (!key.startsWith(`skills/${skillId}/`)) {
|
|
13156
|
+
res.status(403).json({ detail: "Key does not belong to this skill." });
|
|
13157
|
+
return;
|
|
13158
|
+
}
|
|
13159
|
+
const { db: db2 } = await postgresClient();
|
|
13160
|
+
const skill = await db2("skills").where({ id: skillId }).first();
|
|
13161
|
+
if (!skill) {
|
|
13162
|
+
res.status(404).json({ detail: "Skill not found." });
|
|
13163
|
+
return;
|
|
13164
|
+
}
|
|
13165
|
+
const fullKey = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/${key}` : key;
|
|
13166
|
+
const TEXT_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".txt", ".py", ".js", ".ts", ".json", ".yaml", ".yml", ".sh", ".env", ".toml", ".xml", ".html", ".css"]);
|
|
13167
|
+
const ext = key.slice(key.lastIndexOf(".")).toLowerCase();
|
|
13168
|
+
const isText = TEXT_EXTENSIONS.has(ext);
|
|
13169
|
+
const MAX_INLINE_BYTES = 200 * 1024;
|
|
13170
|
+
let content;
|
|
13171
|
+
if (isText) {
|
|
13172
|
+
try {
|
|
13173
|
+
const raw = await getS3ObjectContent(fullKey, config);
|
|
13174
|
+
if (raw.length <= MAX_INLINE_BYTES) {
|
|
13175
|
+
content = raw;
|
|
13176
|
+
}
|
|
13177
|
+
} catch {
|
|
13178
|
+
}
|
|
13179
|
+
}
|
|
13180
|
+
const bucket = config.fileUploads?.s3Bucket ?? "";
|
|
13181
|
+
const url = await getPresignedUrl(bucket, fullKey, config);
|
|
13182
|
+
res.json({ url, content, key });
|
|
13183
|
+
});
|
|
13184
|
+
app.delete("/skills/:skillId/file", async (req, res) => {
|
|
13185
|
+
const authResult = await requestValidators.authenticate(req);
|
|
13186
|
+
if (!authResult.user?.id) {
|
|
13187
|
+
res.status(authResult.code ?? 401).json({ detail: authResult.message });
|
|
13188
|
+
return;
|
|
13189
|
+
}
|
|
13190
|
+
const { skillId } = req.params;
|
|
13191
|
+
const { key, prefix } = req.query;
|
|
13192
|
+
if (!key && !prefix) {
|
|
13193
|
+
res.status(400).json({ detail: "Provide either key or prefix query parameter." });
|
|
13194
|
+
return;
|
|
13195
|
+
}
|
|
13196
|
+
const guard = (s) => !s.startsWith(`skills/${skillId}/`);
|
|
13197
|
+
if (key && typeof key === "string") {
|
|
13198
|
+
if (guard(key)) {
|
|
13199
|
+
res.status(403).json({ detail: "Key does not belong to this skill." });
|
|
13200
|
+
return;
|
|
13201
|
+
}
|
|
13202
|
+
}
|
|
13203
|
+
if (prefix && typeof prefix === "string") {
|
|
13204
|
+
if (guard(prefix)) {
|
|
13205
|
+
res.status(403).json({ detail: "Prefix does not belong to this skill." });
|
|
13206
|
+
return;
|
|
13207
|
+
}
|
|
13208
|
+
}
|
|
13209
|
+
const { db: db2 } = await postgresClient();
|
|
13210
|
+
const skill = await db2("skills").where({ id: skillId }).first();
|
|
13211
|
+
if (!skill) {
|
|
13212
|
+
res.status(404).json({ detail: "Skill not found." });
|
|
13213
|
+
return;
|
|
13214
|
+
}
|
|
13215
|
+
const s3Prefix = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
|
|
13216
|
+
if (key && typeof key === "string") {
|
|
13217
|
+
const fullKey = s3Prefix + key;
|
|
13218
|
+
await deleteS3Object(fullKey, config);
|
|
13219
|
+
res.json({ deleted: 1 });
|
|
13220
|
+
return;
|
|
13221
|
+
}
|
|
13222
|
+
if (prefix && typeof prefix === "string") {
|
|
13223
|
+
const files = await listS3ObjectsByPrefix(prefix, config);
|
|
13224
|
+
await Promise.all(files.map((f) => deleteS3Object(f.key, config)));
|
|
13225
|
+
res.json({ deleted: files.length });
|
|
13226
|
+
return;
|
|
13227
|
+
}
|
|
13228
|
+
});
|
|
13229
|
+
app.post("/skills/:skillId/version", async (req, res) => {
|
|
13230
|
+
const authResult = await requestValidators.authenticate(req);
|
|
13231
|
+
if (!authResult.user?.id) {
|
|
13232
|
+
res.status(authResult.code ?? 401).json({ detail: authResult.message });
|
|
13233
|
+
return;
|
|
13234
|
+
}
|
|
13235
|
+
const { skillId } = req.params;
|
|
13236
|
+
const { label } = req.body;
|
|
13237
|
+
const { db: db2 } = await postgresClient();
|
|
13238
|
+
const skill = await db2("skills").where({ id: skillId }).first();
|
|
13239
|
+
if (!skill) {
|
|
13240
|
+
res.status(404).json({ detail: "Skill not found." });
|
|
13241
|
+
return;
|
|
13242
|
+
}
|
|
13243
|
+
const currentVersion = skill.current_version ?? 1;
|
|
13244
|
+
const newVersion = currentVersion + 1;
|
|
13245
|
+
const currentPrefix = `skills/${skillId}/v${currentVersion}/`;
|
|
13246
|
+
const newPrefix = `skills/${skillId}/v${newVersion}/`;
|
|
13247
|
+
const files = await listS3ObjectsByPrefix(currentPrefix, config);
|
|
13248
|
+
if (files.length === 0) {
|
|
13249
|
+
res.status(400).json({ detail: "No files found in current version to snapshot." });
|
|
13250
|
+
return;
|
|
13251
|
+
}
|
|
13252
|
+
const s3GeneralPrefix = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
|
|
13253
|
+
for (const file of files) {
|
|
13254
|
+
const destKey = file.key.replace(s3GeneralPrefix + currentPrefix, s3GeneralPrefix + newPrefix);
|
|
13255
|
+
await copyS3Object(file.key, destKey, config);
|
|
13256
|
+
}
|
|
13257
|
+
const existingHistory = Array.isArray(skill.history) ? skill.history : [];
|
|
13258
|
+
const newHistory = [
|
|
13259
|
+
...existingHistory,
|
|
13260
|
+
{ version: newVersion, created_at: (/* @__PURE__ */ new Date()).toISOString(), label: label ?? `v${newVersion}` }
|
|
13261
|
+
];
|
|
13262
|
+
await db2("skills").where({ id: skillId }).update({
|
|
13263
|
+
current_version: newVersion,
|
|
13264
|
+
history: JSON.stringify(newHistory)
|
|
13265
|
+
});
|
|
13266
|
+
res.json({ newVersion, fileCount: files.length });
|
|
13267
|
+
});
|
|
13268
|
+
app.post("/skills/:skillId/rename", async (req, res) => {
|
|
13269
|
+
const authResult = await requestValidators.authenticate(req);
|
|
13270
|
+
if (!authResult.user?.id) {
|
|
13271
|
+
res.status(authResult.code ?? 401).json({ detail: authResult.message });
|
|
13272
|
+
return;
|
|
13273
|
+
}
|
|
13274
|
+
const { skillId } = req.params;
|
|
13275
|
+
const { sourceKey, destPath } = req.body;
|
|
13276
|
+
if (!sourceKey || !destPath) {
|
|
13277
|
+
res.status(400).json({ detail: "sourceKey and destPath are required." });
|
|
13278
|
+
return;
|
|
13279
|
+
}
|
|
13280
|
+
if (!sourceKey.startsWith(`skills/${skillId}/`)) {
|
|
13281
|
+
res.status(403).json({ detail: "sourceKey does not belong to this skill." });
|
|
13282
|
+
return;
|
|
13283
|
+
}
|
|
13284
|
+
const { db: db2 } = await postgresClient();
|
|
13285
|
+
const skill = await db2("skills").where({ id: skillId }).first();
|
|
13286
|
+
if (!skill) {
|
|
13287
|
+
res.status(404).json({ detail: "Skill not found." });
|
|
13288
|
+
return;
|
|
13289
|
+
}
|
|
13290
|
+
const version = skill.current_version ?? 1;
|
|
13291
|
+
const safeDest = destPath.replace(/^\/+/, "").replace(/\.\.\//g, "");
|
|
13292
|
+
const destKey = `skills/${skillId}/v${version}/${safeDest}`;
|
|
13293
|
+
const s3Prefix = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
|
|
13294
|
+
const fullSourceKey = s3Prefix + sourceKey;
|
|
13295
|
+
const fullDestKey = s3Prefix + destKey;
|
|
13296
|
+
await copyS3Object(fullSourceKey, fullDestKey, config);
|
|
13297
|
+
await deleteS3Object(fullSourceKey, config);
|
|
13298
|
+
res.json({ newKey: destKey });
|
|
13299
|
+
});
|
|
13300
|
+
app.get("/skills/:skillId/diff", async (req, res) => {
|
|
13301
|
+
const authResult = await requestValidators.authenticate(req);
|
|
13302
|
+
if (!authResult.user?.id) {
|
|
13303
|
+
res.status(authResult.code ?? 401).json({ detail: authResult.message });
|
|
13304
|
+
return;
|
|
13305
|
+
}
|
|
13306
|
+
const { skillId } = req.params;
|
|
13307
|
+
const fromVersion = Number(req.query.fromVersion);
|
|
13308
|
+
const toVersion = Number(req.query.toVersion);
|
|
13309
|
+
if (!fromVersion || !toVersion) {
|
|
13310
|
+
res.status(400).json({ detail: "fromVersion and toVersion query params are required." });
|
|
13311
|
+
return;
|
|
13312
|
+
}
|
|
13313
|
+
const { db: db2 } = await postgresClient();
|
|
13314
|
+
const skill = await db2("skills").where({ id: skillId }).first();
|
|
13315
|
+
if (!skill) {
|
|
13316
|
+
res.status(404).json({ detail: "Skill not found." });
|
|
13317
|
+
return;
|
|
13318
|
+
}
|
|
13319
|
+
const s3Prefix = config.fileUploads?.s3prefix ? `${config.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
|
|
13320
|
+
const fromPrefix = `skills/${skillId}/v${fromVersion}/`;
|
|
13321
|
+
const toPrefix = `skills/${skillId}/v${toVersion}/`;
|
|
13322
|
+
const [fromFiles, toFiles] = await Promise.all([
|
|
13323
|
+
listS3ObjectsByPrefix(fromPrefix, config),
|
|
13324
|
+
listS3ObjectsByPrefix(toPrefix, config)
|
|
13325
|
+
]);
|
|
13326
|
+
const relativise = (files, prefix) => {
|
|
13327
|
+
const full = s3Prefix + prefix;
|
|
13328
|
+
return new Map(files.map((f) => [f.key.replace(full, ""), f]));
|
|
13329
|
+
};
|
|
13330
|
+
const fromMap = relativise(fromFiles, fromPrefix);
|
|
13331
|
+
const toMap = relativise(toFiles, toPrefix);
|
|
13332
|
+
const allPaths = /* @__PURE__ */ new Set([...fromMap.keys(), ...toMap.keys()]);
|
|
13333
|
+
const TEXT_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".txt", ".py", ".js", ".ts", ".json", ".yaml", ".yml", ".sh", ".toml"]);
|
|
13334
|
+
const MAX_DIFF_BYTES = 500 * 1024;
|
|
13335
|
+
const fileDiffs = await Promise.all(
|
|
13336
|
+
[...allPaths].map(async (path3) => {
|
|
13337
|
+
const inFrom = fromMap.has(path3);
|
|
13338
|
+
const inTo = toMap.has(path3);
|
|
13339
|
+
const status = !inFrom ? "added" : !inTo ? "removed" : "modified";
|
|
13340
|
+
if (status === "modified") {
|
|
13341
|
+
const ext = path3.slice(path3.lastIndexOf(".")).toLowerCase();
|
|
13342
|
+
if (!TEXT_EXTENSIONS.has(ext)) {
|
|
13343
|
+
return { path: path3, status };
|
|
13344
|
+
}
|
|
13345
|
+
try {
|
|
13346
|
+
const [fromContent, toContent] = await Promise.all([
|
|
13347
|
+
getS3ObjectContent(s3Prefix + fromPrefix + path3, config),
|
|
13348
|
+
getS3ObjectContent(s3Prefix + toPrefix + path3, config)
|
|
13349
|
+
]);
|
|
13350
|
+
if (fromContent === toContent) {
|
|
13351
|
+
return { path: path3, status: "unchanged" };
|
|
13352
|
+
}
|
|
13353
|
+
if (fromContent.length + toContent.length > MAX_DIFF_BYTES) {
|
|
13354
|
+
return { path: path3, status };
|
|
13355
|
+
}
|
|
13356
|
+
const fromLines = fromContent.split("\n");
|
|
13357
|
+
const toLines = toContent.split("\n");
|
|
13358
|
+
const diff = buildUnifiedDiff(fromLines, toLines, `v${fromVersion}/${path3}`, `v${toVersion}/${path3}`);
|
|
13359
|
+
return { path: path3, status, diff };
|
|
13360
|
+
} catch {
|
|
13361
|
+
return { path: path3, status };
|
|
13362
|
+
}
|
|
13363
|
+
}
|
|
13364
|
+
return { path: path3, status };
|
|
13365
|
+
})
|
|
13366
|
+
);
|
|
13367
|
+
res.json({
|
|
13368
|
+
fromVersion,
|
|
13369
|
+
toVersion,
|
|
13370
|
+
files: fileDiffs.filter((f) => f.status !== "unchanged")
|
|
13371
|
+
});
|
|
13372
|
+
});
|
|
13373
|
+
app.use(import_express5.default.static("public"));
|
|
13374
|
+
await registerOpenAIGatewayRoutes(app, providers, tools, contexts, config, rerankers);
|
|
13375
|
+
return app;
|
|
13376
|
+
};
|
|
13377
|
+
function buildUnifiedDiff(fromLines, toLines, fromLabel, toLabel) {
|
|
13378
|
+
function lcs(a, b) {
|
|
13379
|
+
const m = a.length;
|
|
13380
|
+
const n = b.length;
|
|
13381
|
+
const dp = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0));
|
|
13382
|
+
for (let i = 1; i <= m; i++) {
|
|
13383
|
+
for (let j = 1; j <= n; j++) {
|
|
13384
|
+
dp[i][j] = a[i - 1] === b[j - 1] ? (dp[i - 1][j - 1] ?? 0) + 1 : Math.max(dp[i - 1][j] ?? 0, dp[i][j - 1] ?? 0);
|
|
13385
|
+
}
|
|
13386
|
+
}
|
|
13387
|
+
return dp;
|
|
13388
|
+
}
|
|
13389
|
+
function diff(a, b) {
|
|
13390
|
+
const table = lcs(a, b);
|
|
13391
|
+
const result = [];
|
|
13392
|
+
let i = a.length;
|
|
13393
|
+
let j = b.length;
|
|
13394
|
+
while (i > 0 || j > 0) {
|
|
13395
|
+
if (i > 0 && j > 0 && a[i - 1] === b[j - 1]) {
|
|
13396
|
+
result.unshift({ op: "=", line: a[i - 1] });
|
|
13397
|
+
i--;
|
|
13398
|
+
j--;
|
|
13399
|
+
} else if (j > 0 && (i === 0 || (table[i][j - 1] ?? 0) >= (table[i - 1][j] ?? 0))) {
|
|
13400
|
+
result.unshift({ op: "+", line: b[j - 1] });
|
|
13401
|
+
j--;
|
|
13402
|
+
} else {
|
|
13403
|
+
result.unshift({ op: "-", line: a[i - 1] });
|
|
13404
|
+
i--;
|
|
13405
|
+
}
|
|
13406
|
+
}
|
|
13407
|
+
return result;
|
|
13408
|
+
}
|
|
13409
|
+
const CONTEXT = 3;
|
|
13410
|
+
const hunks = diff(fromLines, toLines);
|
|
13411
|
+
const lines = [`--- ${fromLabel}`, `+++ ${toLabel}`];
|
|
13412
|
+
let hunkStart = -1;
|
|
13413
|
+
for (let idx = 0; idx < hunks.length; idx++) {
|
|
13414
|
+
const h = hunks[idx];
|
|
13415
|
+
if (h.op !== "=") {
|
|
13416
|
+
if (hunkStart < 0) {
|
|
13417
|
+
hunkStart = Math.max(0, idx - CONTEXT);
|
|
13418
|
+
}
|
|
13419
|
+
} else if (hunkStart >= 0 && idx - hunkStart > CONTEXT * 2) {
|
|
13420
|
+
const slice = hunks.slice(hunkStart, Math.min(idx, hunkStart + (idx - hunkStart)));
|
|
13421
|
+
lines.push(`@@ -${hunkStart + 1} +${hunkStart + 1} @@`);
|
|
13422
|
+
for (const s of slice) {
|
|
13423
|
+
lines.push((s.op === "=" ? " " : s.op) + s.line);
|
|
13424
|
+
}
|
|
13425
|
+
hunkStart = -1;
|
|
13426
|
+
}
|
|
13427
|
+
}
|
|
13428
|
+
if (hunkStart >= 0) {
|
|
13429
|
+
const slice = hunks.slice(hunkStart, Math.min(hunks.length, hunkStart + hunks.length));
|
|
13430
|
+
lines.push(`@@ -${hunkStart + 1} +${hunkStart + 1} @@`);
|
|
13431
|
+
for (const s of slice) {
|
|
13432
|
+
lines.push((s.op === "=" ? " " : s.op) + s.line);
|
|
13433
|
+
}
|
|
13434
|
+
}
|
|
13435
|
+
return lines.join("\n");
|
|
13436
|
+
}
|
|
13437
|
+
var createCustomAnthropicStreamingMessage = (message) => {
|
|
13438
|
+
const responseData = {
|
|
13439
|
+
type: "message",
|
|
13440
|
+
content: [
|
|
13441
|
+
{
|
|
13442
|
+
type: "text",
|
|
13443
|
+
text: message
|
|
13444
|
+
}
|
|
13445
|
+
]
|
|
13446
|
+
};
|
|
13447
|
+
const jsonString = JSON.stringify(responseData);
|
|
13448
|
+
const arrayBuffer = new TextEncoder().encode(jsonString).buffer;
|
|
13449
|
+
return arrayBuffer;
|
|
13450
|
+
};
|
|
13451
|
+
|
|
13452
|
+
// src/mcp/index.ts
|
|
13453
|
+
var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
13454
|
+
var import_node_crypto6 = require("crypto");
|
|
13455
|
+
var import_streamableHttp = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
|
|
13456
|
+
var import_types2 = require("@modelcontextprotocol/sdk/types.js");
|
|
13457
|
+
var import_express6 = require("express");
|
|
13458
|
+
var import_api3 = require("@opentelemetry/api");
|
|
13459
|
+
var import_crypto_js8 = __toESM(require("crypto-js"), 1);
|
|
13460
|
+
var import_zod11 = require("zod");
|
|
12196
13461
|
var SESSION_ID_HEADER = "mcp-session-id";
|
|
12197
13462
|
var ExuluMCP = class {
|
|
12198
13463
|
server = {};
|
|
@@ -12257,34 +13522,34 @@ var ExuluMCP = class {
|
|
|
12257
13522
|
);
|
|
12258
13523
|
}
|
|
12259
13524
|
if (variable.encrypted) {
|
|
12260
|
-
const bytes =
|
|
12261
|
-
providerapikey = bytes.toString(
|
|
13525
|
+
const bytes = import_crypto_js8.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
13526
|
+
providerapikey = bytes.toString(import_crypto_js8.default.enc.Utf8);
|
|
12262
13527
|
}
|
|
12263
13528
|
}
|
|
12264
13529
|
console.log(
|
|
12265
13530
|
"[EXULU] Enabled tools",
|
|
12266
13531
|
enabledTools?.map((x) => x.name + " (" + x.id + ")")
|
|
12267
13532
|
);
|
|
12268
|
-
for (const
|
|
12269
|
-
if (server.tools[
|
|
13533
|
+
for (const tool5 of enabledTools || []) {
|
|
13534
|
+
if (server.tools[tool5.id]) {
|
|
12270
13535
|
continue;
|
|
12271
13536
|
}
|
|
12272
13537
|
server.mcp.registerTool(
|
|
12273
|
-
sanitizeToolName(
|
|
13538
|
+
sanitizeToolName(tool5.name + "_agent_" + tool5.id),
|
|
12274
13539
|
{
|
|
12275
|
-
title:
|
|
12276
|
-
description:
|
|
13540
|
+
title: tool5.name + " agent",
|
|
13541
|
+
description: tool5.description,
|
|
12277
13542
|
inputSchema: {
|
|
12278
|
-
inputs:
|
|
13543
|
+
inputs: tool5.inputSchema || import_zod11.z.object({})
|
|
12279
13544
|
}
|
|
12280
13545
|
},
|
|
12281
13546
|
async ({ inputs }, args) => {
|
|
12282
|
-
console.log("[EXULU] MCP tool name",
|
|
13547
|
+
console.log("[EXULU] MCP tool name", tool5.name);
|
|
12283
13548
|
console.log("[EXULU] MCP tool inputs", inputs);
|
|
12284
13549
|
console.log("[EXULU] MCP tool args", args);
|
|
12285
13550
|
const configValues = agent.tools;
|
|
12286
13551
|
const tools = await convertExuluToolsToAiSdkTools(
|
|
12287
|
-
[
|
|
13552
|
+
[tool5],
|
|
12288
13553
|
[],
|
|
12289
13554
|
allTools,
|
|
12290
13555
|
configValues,
|
|
@@ -12297,13 +13562,13 @@ var ExuluMCP = class {
|
|
|
12297
13562
|
void 0,
|
|
12298
13563
|
void 0
|
|
12299
13564
|
);
|
|
12300
|
-
const convertedTool = tools[sanitizeToolName(
|
|
13565
|
+
const convertedTool = tools[sanitizeToolName(tool5.name)];
|
|
12301
13566
|
if (!convertedTool?.execute) {
|
|
12302
13567
|
console.error("[EXULU] Tool not found in converted tools array.", tools);
|
|
12303
13568
|
throw new Error("Tool not found in converted tools array.");
|
|
12304
13569
|
}
|
|
12305
13570
|
const iterator = await convertedTool.execute(inputs, {
|
|
12306
|
-
toolCallId:
|
|
13571
|
+
toolCallId: tool5.id + "_" + (0, import_node_crypto6.randomUUID)(),
|
|
12307
13572
|
messages: []
|
|
12308
13573
|
});
|
|
12309
13574
|
let result;
|
|
@@ -12317,7 +13582,7 @@ var ExuluMCP = class {
|
|
|
12317
13582
|
};
|
|
12318
13583
|
}
|
|
12319
13584
|
);
|
|
12320
|
-
server.tools[
|
|
13585
|
+
server.tools[tool5.id] = tool5.name;
|
|
12321
13586
|
}
|
|
12322
13587
|
const getListOfPromptTemplatesName = "getListOfPromptTemplates";
|
|
12323
13588
|
if (!server.tools[getListOfPromptTemplatesName]) {
|
|
@@ -12327,7 +13592,7 @@ var ExuluMCP = class {
|
|
|
12327
13592
|
title: "Get List of Prompt Templates",
|
|
12328
13593
|
description: "Retrieves a list of prompt templates available for this agent. Returns the name, description, and ID of each template.",
|
|
12329
13594
|
inputSchema: {
|
|
12330
|
-
inputs:
|
|
13595
|
+
inputs: import_zod11.z.object({})
|
|
12331
13596
|
}
|
|
12332
13597
|
},
|
|
12333
13598
|
async ({ inputs }, args) => {
|
|
@@ -12373,8 +13638,8 @@ var ExuluMCP = class {
|
|
|
12373
13638
|
title: "Get Prompt Template Details",
|
|
12374
13639
|
description: "Retrieves the full details of a specific prompt template by ID, including the actual template content with variables.",
|
|
12375
13640
|
inputSchema: {
|
|
12376
|
-
inputs:
|
|
12377
|
-
id:
|
|
13641
|
+
inputs: import_zod11.z.object({
|
|
13642
|
+
id: import_zod11.z.string().describe("The ID of the prompt template to retrieve")
|
|
12378
13643
|
})
|
|
12379
13644
|
}
|
|
12380
13645
|
},
|
|
@@ -12437,20 +13702,20 @@ var ExuluMCP = class {
|
|
|
12437
13702
|
return server.mcp;
|
|
12438
13703
|
};
|
|
12439
13704
|
create = async ({
|
|
12440
|
-
express:
|
|
13705
|
+
express: express4,
|
|
12441
13706
|
allTools,
|
|
12442
13707
|
allProviders,
|
|
12443
13708
|
allContexts,
|
|
12444
13709
|
allRerankers,
|
|
12445
13710
|
config
|
|
12446
13711
|
}) => {
|
|
12447
|
-
if (!
|
|
13712
|
+
if (!express4) {
|
|
12448
13713
|
throw new Error("Express not initialized.");
|
|
12449
13714
|
}
|
|
12450
13715
|
if (!this.server) {
|
|
12451
13716
|
throw new Error("MCP server not initialized.");
|
|
12452
13717
|
}
|
|
12453
|
-
|
|
13718
|
+
express4.post("/mcp/:agent", async (req, res) => {
|
|
12454
13719
|
console.log("[EXULU] MCP request received.", req.params.agent);
|
|
12455
13720
|
if (!req.params.agent) {
|
|
12456
13721
|
res.status(400).json({
|
|
@@ -12497,7 +13762,7 @@ var ExuluMCP = class {
|
|
|
12497
13762
|
transport = this.transports[sessionId];
|
|
12498
13763
|
} else if (!sessionId && (0, import_types2.isInitializeRequest)(req.body)) {
|
|
12499
13764
|
transport = new import_streamableHttp.StreamableHTTPServerTransport({
|
|
12500
|
-
sessionIdGenerator: () => (0,
|
|
13765
|
+
sessionIdGenerator: () => (0, import_node_crypto6.randomUUID)(),
|
|
12501
13766
|
onsessioninitialized: (sessionId2) => {
|
|
12502
13767
|
this.transports[sessionId2] = transport;
|
|
12503
13768
|
}
|
|
@@ -12531,15 +13796,15 @@ var ExuluMCP = class {
|
|
|
12531
13796
|
const transport = this.transports[sessionId];
|
|
12532
13797
|
await transport.handleRequest(req, res);
|
|
12533
13798
|
};
|
|
12534
|
-
|
|
12535
|
-
|
|
13799
|
+
express4.get("/mcp/:agent", handleSessionRequest);
|
|
13800
|
+
express4.delete("/mcp/:agent", handleSessionRequest);
|
|
12536
13801
|
console.log("[EXULU] MCP server created.");
|
|
12537
|
-
return
|
|
13802
|
+
return express4;
|
|
12538
13803
|
};
|
|
12539
13804
|
};
|
|
12540
13805
|
|
|
12541
13806
|
// src/exulu/app/index.ts
|
|
12542
|
-
var
|
|
13807
|
+
var import_express8 = __toESM(require("express"), 1);
|
|
12543
13808
|
|
|
12544
13809
|
// src/templates/providers/anthropic/claude.ts
|
|
12545
13810
|
var import_anthropic = require("@ai-sdk/anthropic");
|
|
@@ -13245,7 +14510,7 @@ var import_winston2 = __toESM(require("winston"), 1);
|
|
|
13245
14510
|
var import_util = __toESM(require("util"), 1);
|
|
13246
14511
|
|
|
13247
14512
|
// src/exulu/evals.ts
|
|
13248
|
-
var
|
|
14513
|
+
var import_ai11 = require("ai");
|
|
13249
14514
|
var ExuluEval = class {
|
|
13250
14515
|
id;
|
|
13251
14516
|
name;
|
|
@@ -13282,7 +14547,7 @@ var ExuluEval = class {
|
|
|
13282
14547
|
};
|
|
13283
14548
|
|
|
13284
14549
|
// src/templates/evals/index.ts
|
|
13285
|
-
var
|
|
14550
|
+
var import_zod12 = require("zod");
|
|
13286
14551
|
var llmAsJudgeEval = () => {
|
|
13287
14552
|
if (process.env.REDIS_HOST?.length && process.env.REDIS_PORT?.length) {
|
|
13288
14553
|
return new ExuluEval({
|
|
@@ -13327,8 +14592,8 @@ var llmAsJudgeEval = () => {
|
|
|
13327
14592
|
contexts: [],
|
|
13328
14593
|
rerankers: [],
|
|
13329
14594
|
prompt,
|
|
13330
|
-
outputSchema:
|
|
13331
|
-
score:
|
|
14595
|
+
outputSchema: import_zod12.z.object({
|
|
14596
|
+
score: import_zod12.z.number().min(0).max(100).describe("The score between 0 and 100.")
|
|
13332
14597
|
}),
|
|
13333
14598
|
providerapikey
|
|
13334
14599
|
});
|
|
@@ -13556,12 +14821,12 @@ Usage:
|
|
|
13556
14821
|
- If no todos exist yet, an empty list will be returned`;
|
|
13557
14822
|
|
|
13558
14823
|
// src/templates/tools/todo/todo.ts
|
|
13559
|
-
var
|
|
13560
|
-
var TodoSchema =
|
|
13561
|
-
content:
|
|
13562
|
-
status:
|
|
13563
|
-
priority:
|
|
13564
|
-
id:
|
|
14824
|
+
var import_zod13 = __toESM(require("zod"), 1);
|
|
14825
|
+
var TodoSchema = import_zod13.default.object({
|
|
14826
|
+
content: import_zod13.default.string().describe("Brief description of the task"),
|
|
14827
|
+
status: import_zod13.default.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
|
|
14828
|
+
priority: import_zod13.default.string().describe("Priority level of the task: high, medium, low"),
|
|
14829
|
+
id: import_zod13.default.string().describe("Unique identifier for the todo item")
|
|
13565
14830
|
});
|
|
13566
14831
|
var TodoWriteTool = new ExuluTool({
|
|
13567
14832
|
id: "todo_write",
|
|
@@ -13577,8 +14842,8 @@ var TodoWriteTool = new ExuluTool({
|
|
|
13577
14842
|
default: todowrite_default
|
|
13578
14843
|
}
|
|
13579
14844
|
],
|
|
13580
|
-
inputSchema:
|
|
13581
|
-
todos:
|
|
14845
|
+
inputSchema: import_zod13.default.object({
|
|
14846
|
+
todos: import_zod13.default.array(TodoSchema).describe("The updated todo list")
|
|
13582
14847
|
}),
|
|
13583
14848
|
execute: async (inputs) => {
|
|
13584
14849
|
const { sessionID, todos, user } = inputs;
|
|
@@ -13613,7 +14878,7 @@ var TodoReadTool = new ExuluTool({
|
|
|
13613
14878
|
id: "todo_read",
|
|
13614
14879
|
name: "Todo Read",
|
|
13615
14880
|
description: "Use this tool to read your todo list",
|
|
13616
|
-
inputSchema:
|
|
14881
|
+
inputSchema: import_zod13.default.object({}),
|
|
13617
14882
|
type: "function",
|
|
13618
14883
|
category: "todo",
|
|
13619
14884
|
config: [
|
|
@@ -13739,18 +15004,18 @@ After asking a question, use the Question Read tool to check if the user has ans
|
|
|
13739
15004
|
var questionread_default = 'Use this tool to read questions you\'ve asked and check if they\'ve been answered by the user. This tool helps you track the status of questions and retrieve the user\'s selected answers.\n\n## When to Use This Tool\n\nUse this tool proactively in these situations:\n- After asking a question to check if the user has responded\n- To retrieve the user\'s answer before proceeding with implementation\n- To review all questions and answers in the current session\n- When you need to reference a previous answer\n\n## How It Works\n\n- This tool takes no parameters (leave the input blank or empty)\n- Returns an array of all questions in the session\n- Each question includes:\n - `id`: Unique identifier for the question\n - `question`: The question text\n - `answerOptions`: Array of answer options with their IDs and text\n - `status`: Either "pending" (not answered) or "answered"\n - `selectedAnswerId`: The ID of the chosen answer (only present if answered)\n\n## Usage Pattern\n\nTypically you\'ll:\n1. Use Question Ask to pose a question\n2. Wait for the user to respond\n3. Use Question Read to check the answer\n4. Find the selected answer by matching the `selectedAnswerId` with an option in `answerOptions`\n5. Proceed with implementation based on the user\'s choice\n\n## Example Response\n\n```json\n[\n {\n "id": "question123",\n "question": "Which authentication method would you like to implement?",\n "answerOptions": [\n { "id": "ans1", "text": "JWT tokens" },\n { "id": "ans2", "text": "OAuth 2.0" },\n { "id": "ans3", "text": "Session-based auth" },\n { "id": "ans4", "text": "None of the above..." }\n ],\n "status": "answered",\n "selectedAnswerId": "ans1"\n }\n]\n```\n\nIn this example, the user selected "JWT tokens" (id: ans1).\n\n## Important Notes\n\n- If no questions exist in the session, an empty array will be returned\n- Questions remain in the session even after being answered for reference\n- Use the `selectedAnswerId` to find which answer option the user chose by matching it against the `id` field in `answerOptions`\n';
|
|
13740
15005
|
|
|
13741
15006
|
// src/templates/tools/question/question.ts
|
|
13742
|
-
var
|
|
13743
|
-
var
|
|
13744
|
-
var AnswerOptionSchema =
|
|
13745
|
-
id:
|
|
13746
|
-
text:
|
|
15007
|
+
var import_zod14 = __toESM(require("zod"), 1);
|
|
15008
|
+
var import_node_crypto7 = require("crypto");
|
|
15009
|
+
var AnswerOptionSchema = import_zod14.default.object({
|
|
15010
|
+
id: import_zod14.default.string().describe("Unique identifier for the answer option"),
|
|
15011
|
+
text: import_zod14.default.string().describe("The text of the answer option")
|
|
13747
15012
|
});
|
|
13748
|
-
var _QuestionSchema =
|
|
13749
|
-
id:
|
|
13750
|
-
question:
|
|
13751
|
-
answerOptions:
|
|
13752
|
-
selectedAnswerId:
|
|
13753
|
-
status:
|
|
15013
|
+
var _QuestionSchema = import_zod14.default.object({
|
|
15014
|
+
id: import_zod14.default.string().describe("Unique identifier for the question"),
|
|
15015
|
+
question: import_zod14.default.string().describe("The question to ask the user"),
|
|
15016
|
+
answerOptions: import_zod14.default.array(AnswerOptionSchema).describe("Array of possible answer options"),
|
|
15017
|
+
selectedAnswerId: import_zod14.default.string().optional().describe("The ID of the answer option selected by the user"),
|
|
15018
|
+
status: import_zod14.default.enum(["pending", "answered"]).describe("Status of the question: pending or answered")
|
|
13754
15019
|
});
|
|
13755
15020
|
var QuestionAskTool = new ExuluTool({
|
|
13756
15021
|
id: "question_ask",
|
|
@@ -13758,6 +15023,7 @@ var QuestionAskTool = new ExuluTool({
|
|
|
13758
15023
|
description: "Use this tool to ask a question to the user with multiple choice answers",
|
|
13759
15024
|
type: "function",
|
|
13760
15025
|
category: "question",
|
|
15026
|
+
needsApproval: false,
|
|
13761
15027
|
config: [
|
|
13762
15028
|
{
|
|
13763
15029
|
name: "description",
|
|
@@ -13766,9 +15032,9 @@ var QuestionAskTool = new ExuluTool({
|
|
|
13766
15032
|
default: questionask_default
|
|
13767
15033
|
}
|
|
13768
15034
|
],
|
|
13769
|
-
inputSchema:
|
|
13770
|
-
question:
|
|
13771
|
-
answerOptions:
|
|
15035
|
+
inputSchema: import_zod14.default.object({
|
|
15036
|
+
question: import_zod14.default.string().describe("The question to ask the user"),
|
|
15037
|
+
answerOptions: import_zod14.default.array(import_zod14.default.string()).describe("Array of possible answer options (strings)")
|
|
13772
15038
|
}),
|
|
13773
15039
|
execute: async (inputs) => {
|
|
13774
15040
|
const { sessionID, question, answerOptions, user } = inputs;
|
|
@@ -13793,15 +15059,15 @@ var QuestionAskTool = new ExuluTool({
|
|
|
13793
15059
|
throw new Error("You don't have access to this session " + session.id + ".");
|
|
13794
15060
|
}
|
|
13795
15061
|
const answerOptionsWithIds = answerOptions.map((text) => ({
|
|
13796
|
-
id: (0,
|
|
15062
|
+
id: (0, import_node_crypto7.randomUUID)(),
|
|
13797
15063
|
text
|
|
13798
15064
|
}));
|
|
13799
15065
|
answerOptionsWithIds.push({
|
|
13800
|
-
id: (0,
|
|
15066
|
+
id: (0, import_node_crypto7.randomUUID)(),
|
|
13801
15067
|
text: "None of the above..."
|
|
13802
15068
|
});
|
|
13803
15069
|
const newQuestion = {
|
|
13804
|
-
id: (0,
|
|
15070
|
+
id: (0, import_node_crypto7.randomUUID)(),
|
|
13805
15071
|
question,
|
|
13806
15072
|
answerOptions: answerOptionsWithIds,
|
|
13807
15073
|
status: "pending"
|
|
@@ -13827,8 +15093,9 @@ var QuestionAskTool = new ExuluTool({
|
|
|
13827
15093
|
var QuestionReadTool = new ExuluTool({
|
|
13828
15094
|
id: "question_read",
|
|
13829
15095
|
name: "Question Read",
|
|
15096
|
+
needsApproval: false,
|
|
13830
15097
|
description: "Use this tool to read questions and their answers",
|
|
13831
|
-
inputSchema:
|
|
15098
|
+
inputSchema: import_zod14.default.object({}),
|
|
13832
15099
|
type: "function",
|
|
13833
15100
|
category: "question",
|
|
13834
15101
|
config: [
|
|
@@ -13868,15 +15135,15 @@ async function getQuestions(sessionID) {
|
|
|
13868
15135
|
var questionTools = [QuestionAskTool, QuestionReadTool];
|
|
13869
15136
|
|
|
13870
15137
|
// src/templates/tools/perplexity.ts
|
|
13871
|
-
var
|
|
15138
|
+
var import_zod15 = __toESM(require("zod"), 1);
|
|
13872
15139
|
var import_perplexity_ai = __toESM(require("@perplexity-ai/perplexity_ai"), 1);
|
|
13873
15140
|
var internetSearchTool = new ExuluTool({
|
|
13874
15141
|
id: "internet_search",
|
|
13875
|
-
name: "
|
|
15142
|
+
name: "Internet Search",
|
|
13876
15143
|
description: "Search the internet for information.",
|
|
13877
|
-
inputSchema:
|
|
13878
|
-
query:
|
|
13879
|
-
search_recency_filter:
|
|
15144
|
+
inputSchema: import_zod15.default.object({
|
|
15145
|
+
query: import_zod15.default.string().describe("The query to the tool."),
|
|
15146
|
+
search_recency_filter: import_zod15.default.enum(["day", "week", "month", "year"]).optional().describe("The recency filter for the search, can be day, week, month or year.")
|
|
13880
15147
|
}),
|
|
13881
15148
|
category: "internet_search",
|
|
13882
15149
|
type: "web_search",
|
|
@@ -13954,7 +15221,7 @@ var internetSearchTool = new ExuluTool({
|
|
|
13954
15221
|
} catch (error) {
|
|
13955
15222
|
if (error instanceof import_perplexity_ai.default.RateLimitError && attempt < maxRetries - 1) {
|
|
13956
15223
|
const delay = Math.pow(2, attempt) * 1e3 + Math.random() * 1e3;
|
|
13957
|
-
await new Promise((
|
|
15224
|
+
await new Promise((resolve3) => setTimeout(resolve3, delay));
|
|
13958
15225
|
continue;
|
|
13959
15226
|
}
|
|
13960
15227
|
throw error;
|
|
@@ -13967,6 +15234,101 @@ var internetSearchTool = new ExuluTool({
|
|
|
13967
15234
|
});
|
|
13968
15235
|
var perplexityTools = [internetSearchTool];
|
|
13969
15236
|
|
|
15237
|
+
// src/templates/tools/email.ts
|
|
15238
|
+
var nodemailer = __toESM(require("nodemailer"), 1);
|
|
15239
|
+
var import_zod16 = require("zod");
|
|
15240
|
+
var transporter = null;
|
|
15241
|
+
function getTransporter(config) {
|
|
15242
|
+
if (!transporter) {
|
|
15243
|
+
transporter = nodemailer.createTransport(config);
|
|
15244
|
+
}
|
|
15245
|
+
return transporter;
|
|
15246
|
+
}
|
|
15247
|
+
async function sendEmail(recipient, subject, html, text, config) {
|
|
15248
|
+
const transport = getTransporter(config);
|
|
15249
|
+
html = html.trim();
|
|
15250
|
+
text = text.trim();
|
|
15251
|
+
await transport.sendMail({
|
|
15252
|
+
from: config.from,
|
|
15253
|
+
to: recipient,
|
|
15254
|
+
subject,
|
|
15255
|
+
text,
|
|
15256
|
+
html
|
|
15257
|
+
});
|
|
15258
|
+
}
|
|
15259
|
+
var emailTool = new ExuluTool({
|
|
15260
|
+
id: "email",
|
|
15261
|
+
name: "Email",
|
|
15262
|
+
description: "Send an email.",
|
|
15263
|
+
inputSchema: import_zod16.z.object({
|
|
15264
|
+
recipient: import_zod16.z.string().describe("The recipient of the email."),
|
|
15265
|
+
subject: import_zod16.z.string().describe("The subject of the email."),
|
|
15266
|
+
html: import_zod16.z.string().describe("The HTML body of the email."),
|
|
15267
|
+
text: import_zod16.z.string().describe("The text body of the email.")
|
|
15268
|
+
}),
|
|
15269
|
+
type: "function",
|
|
15270
|
+
config: [{
|
|
15271
|
+
name: "smtp_host",
|
|
15272
|
+
description: "The SMTP host to send the email from.",
|
|
15273
|
+
type: "variable",
|
|
15274
|
+
default: void 0
|
|
15275
|
+
}, {
|
|
15276
|
+
name: "smtp_port",
|
|
15277
|
+
description: "The SMTP port to send the email from.",
|
|
15278
|
+
type: "variable",
|
|
15279
|
+
default: void 0
|
|
15280
|
+
}, {
|
|
15281
|
+
name: "smtp_user",
|
|
15282
|
+
description: "The SMTP user to send the email from.",
|
|
15283
|
+
type: "variable",
|
|
15284
|
+
default: void 0
|
|
15285
|
+
}, {
|
|
15286
|
+
name: "smtp_password",
|
|
15287
|
+
description: "The SMTP password to send the email from.",
|
|
15288
|
+
type: "variable",
|
|
15289
|
+
default: void 0
|
|
15290
|
+
}, {
|
|
15291
|
+
name: "smtp_from",
|
|
15292
|
+
description: "The SMTP from address to send the email from.",
|
|
15293
|
+
type: "variable",
|
|
15294
|
+
default: void 0
|
|
15295
|
+
}, {
|
|
15296
|
+
name: "allowed_recipient_domains",
|
|
15297
|
+
description: "A comma seperated list of allowed recipient domains to send emails to.",
|
|
15298
|
+
type: "string",
|
|
15299
|
+
default: void 0
|
|
15300
|
+
}],
|
|
15301
|
+
execute: async ({ recipient, subject, html, text, toolVariablesConfig }) => {
|
|
15302
|
+
const EMAIL_CONFIG = {
|
|
15303
|
+
host: toolVariablesConfig.smtp_host || process.env.SMTP_HOST || "",
|
|
15304
|
+
port: parseInt(toolVariablesConfig.smtp_port || process.env.SMTP_PORT || "587", 10),
|
|
15305
|
+
secure: toolVariablesConfig.smtp_secure === "true" || process.env.SMTP_SECURE === "true",
|
|
15306
|
+
// true for 465, false for other ports
|
|
15307
|
+
auth: {
|
|
15308
|
+
user: toolVariablesConfig.smtp_user || process.env.SMTP_USER || "",
|
|
15309
|
+
pass: toolVariablesConfig.smtp_password || process.env.SMTP_PASSWORD || ""
|
|
15310
|
+
},
|
|
15311
|
+
from: toolVariablesConfig.smtp_from || process.env.SMTP_FROM || "",
|
|
15312
|
+
// Allow self-signed certificates if SMTP_REJECT_UNAUTHORIZED is set to 'false'
|
|
15313
|
+
tls: {
|
|
15314
|
+
rejectUnauthorized: false
|
|
15315
|
+
}
|
|
15316
|
+
};
|
|
15317
|
+
if (toolVariablesConfig.allowed_recipient_domains) {
|
|
15318
|
+
const allowedRecipientDomains = toolVariablesConfig.allowed_recipient_domains.split(",");
|
|
15319
|
+
if (!allowedRecipientDomains.some((domain) => recipient.endsWith(`@${domain}`))) {
|
|
15320
|
+
return {
|
|
15321
|
+
result: "Recipient domain not allowed to send emails to."
|
|
15322
|
+
};
|
|
15323
|
+
}
|
|
15324
|
+
}
|
|
15325
|
+
await sendEmail(recipient, subject, html, text, EMAIL_CONFIG);
|
|
15326
|
+
return {
|
|
15327
|
+
result: "Email sent successfully to " + recipient + " with subject " + subject + "."
|
|
15328
|
+
};
|
|
15329
|
+
}
|
|
15330
|
+
});
|
|
15331
|
+
|
|
13970
15332
|
// src/validators/postgres-name.ts
|
|
13971
15333
|
var isValidPostgresName = (id) => {
|
|
13972
15334
|
const regex = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
@@ -14071,8 +15433,7 @@ var ExuluApp2 = class {
|
|
|
14071
15433
|
...todoTools,
|
|
14072
15434
|
...questionTools,
|
|
14073
15435
|
...perplexityTools,
|
|
14074
|
-
|
|
14075
|
-
...Object.values(contexts || {}).map((context) => context.tool()).filter(Boolean)
|
|
15436
|
+
emailTool
|
|
14076
15437
|
// Because agents are stored in the database, we add those as tools
|
|
14077
15438
|
// at request time, not during ExuluApp initialization. We add them
|
|
14078
15439
|
// in the grahql tools resolver.
|
|
@@ -14089,9 +15450,9 @@ var ExuluApp2 = class {
|
|
|
14089
15450
|
id: provider.id ?? "",
|
|
14090
15451
|
type: "agent"
|
|
14091
15452
|
})),
|
|
14092
|
-
...this._tools.map((
|
|
14093
|
-
name:
|
|
14094
|
-
id:
|
|
15453
|
+
...this._tools.map((tool5) => ({
|
|
15454
|
+
name: tool5.name ?? "",
|
|
15455
|
+
id: tool5.id ?? "",
|
|
14095
15456
|
type: "tool"
|
|
14096
15457
|
})),
|
|
14097
15458
|
...this._rerankers.map((reranker) => ({
|
|
@@ -14137,7 +15498,7 @@ var ExuluApp2 = class {
|
|
|
14137
15498
|
express = {
|
|
14138
15499
|
init: async () => {
|
|
14139
15500
|
if (!this._expressApp) {
|
|
14140
|
-
this._expressApp = (0,
|
|
15501
|
+
this._expressApp = (0, import_express8.default)();
|
|
14141
15502
|
await this.server.express.init();
|
|
14142
15503
|
console.log("[EXULU] Express app initialized.");
|
|
14143
15504
|
}
|
|
@@ -15353,7 +16714,7 @@ var RecursiveChunker = class _RecursiveChunker extends BaseChunker {
|
|
|
15353
16714
|
};
|
|
15354
16715
|
|
|
15355
16716
|
// src/exulu/embedder.ts
|
|
15356
|
-
var
|
|
16717
|
+
var import_crypto_js9 = __toESM(require("crypto-js"), 1);
|
|
15357
16718
|
var ExuluEmbedder = class {
|
|
15358
16719
|
id;
|
|
15359
16720
|
name;
|
|
@@ -15424,8 +16785,8 @@ var ExuluEmbedder = class {
|
|
|
15424
16785
|
);
|
|
15425
16786
|
}
|
|
15426
16787
|
try {
|
|
15427
|
-
const bytes =
|
|
15428
|
-
const decrypted = bytes.toString(
|
|
16788
|
+
const bytes = import_crypto_js9.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
16789
|
+
const decrypted = bytes.toString(import_crypto_js9.default.enc.Utf8);
|
|
15429
16790
|
if (!decrypted) {
|
|
15430
16791
|
throw new Error("Decryption returned empty string - invalid key or corrupted data");
|
|
15431
16792
|
}
|
|
@@ -15913,6 +17274,7 @@ var {
|
|
|
15913
17274
|
agentMessagesSchema: agentMessagesSchema3,
|
|
15914
17275
|
rolesSchema: rolesSchema3,
|
|
15915
17276
|
usersSchema: usersSchema3,
|
|
17277
|
+
skillsSchema: skillsSchema3,
|
|
15916
17278
|
statisticsSchema: statisticsSchema3,
|
|
15917
17279
|
variablesSchema: variablesSchema3,
|
|
15918
17280
|
workflowTemplatesSchema: workflowTemplatesSchema3,
|
|
@@ -15963,6 +17325,7 @@ var up = async function(knex) {
|
|
|
15963
17325
|
agentsSchema3(),
|
|
15964
17326
|
feedbackSchema3(),
|
|
15965
17327
|
variablesSchema3(),
|
|
17328
|
+
skillsSchema3(),
|
|
15966
17329
|
workflowTemplatesSchema3()
|
|
15967
17330
|
];
|
|
15968
17331
|
const createTable = async (schema) => {
|
|
@@ -16165,7 +17528,7 @@ var create = ({
|
|
|
16165
17528
|
};
|
|
16166
17529
|
|
|
16167
17530
|
// src/index.ts
|
|
16168
|
-
var
|
|
17531
|
+
var import_crypto_js10 = __toESM(require("crypto-js"), 1);
|
|
16169
17532
|
|
|
16170
17533
|
// ee/chunking/markdown.ts
|
|
16171
17534
|
var extractPageTag = (text) => {
|
|
@@ -16649,7 +18012,7 @@ var MarkdownChunker = class {
|
|
|
16649
18012
|
var import_child_process = require("child_process");
|
|
16650
18013
|
var import_util2 = require("util");
|
|
16651
18014
|
var import_path = require("path");
|
|
16652
|
-
var
|
|
18015
|
+
var import_fs3 = require("fs");
|
|
16653
18016
|
var import_url = require("url");
|
|
16654
18017
|
var execAsync = (0, import_util2.promisify)(import_child_process.exec);
|
|
16655
18018
|
function getPackageRoot() {
|
|
@@ -16659,9 +18022,9 @@ function getPackageRoot() {
|
|
|
16659
18022
|
const maxAttempts = 10;
|
|
16660
18023
|
while (attempts < maxAttempts) {
|
|
16661
18024
|
const packageJsonPath = (0, import_path.join)(currentDir, "package.json");
|
|
16662
|
-
if ((0,
|
|
18025
|
+
if ((0, import_fs3.existsSync)(packageJsonPath)) {
|
|
16663
18026
|
try {
|
|
16664
|
-
const packageJson = JSON.parse((0,
|
|
18027
|
+
const packageJson = JSON.parse((0, import_fs3.readFileSync)(packageJsonPath, "utf-8"));
|
|
16665
18028
|
if (packageJson.name === "@exulu/backend") {
|
|
16666
18029
|
return currentDir;
|
|
16667
18030
|
}
|
|
@@ -16688,7 +18051,7 @@ function isPythonEnvironmentSetup(packageRoot) {
|
|
|
16688
18051
|
const root = packageRoot ?? getPackageRoot();
|
|
16689
18052
|
const venvPath = getVenvPath(root);
|
|
16690
18053
|
const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
|
|
16691
|
-
return (0,
|
|
18054
|
+
return (0, import_fs3.existsSync)(venvPath) && (0, import_fs3.existsSync)(pythonPath);
|
|
16692
18055
|
}
|
|
16693
18056
|
async function setupPythonEnvironment(options = {}) {
|
|
16694
18057
|
const {
|
|
@@ -16709,7 +18072,7 @@ async function setupPythonEnvironment(options = {}) {
|
|
|
16709
18072
|
};
|
|
16710
18073
|
}
|
|
16711
18074
|
const setupScriptPath = getSetupScriptPath(packageRoot);
|
|
16712
|
-
if (!(0,
|
|
18075
|
+
if (!(0, import_fs3.existsSync)(setupScriptPath)) {
|
|
16713
18076
|
return {
|
|
16714
18077
|
success: false,
|
|
16715
18078
|
message: `Setup script not found at: ${setupScriptPath}`,
|
|
@@ -16791,13 +18154,13 @@ async function validatePythonEnvironment(packageRoot, checkPackages = true) {
|
|
|
16791
18154
|
const root = packageRoot ?? getPackageRoot();
|
|
16792
18155
|
const venvPath = getVenvPath(root);
|
|
16793
18156
|
const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
|
|
16794
|
-
if (!(0,
|
|
18157
|
+
if (!(0, import_fs3.existsSync)(venvPath)) {
|
|
16795
18158
|
return {
|
|
16796
18159
|
valid: false,
|
|
16797
18160
|
message: getPythonSetupInstructions()
|
|
16798
18161
|
};
|
|
16799
18162
|
}
|
|
16800
|
-
if (!(0,
|
|
18163
|
+
if (!(0, import_fs3.existsSync)(pythonPath)) {
|
|
16801
18164
|
return {
|
|
16802
18165
|
valid: false,
|
|
16803
18166
|
message: "Python virtual environment is corrupted. Please run:\n await setupPythonEnvironment({ force: true })"
|
|
@@ -16849,9 +18212,9 @@ Or manually run the setup script:
|
|
|
16849
18212
|
}
|
|
16850
18213
|
|
|
16851
18214
|
// ee/python/documents/processing/doc_processor.ts
|
|
16852
|
-
var
|
|
18215
|
+
var fs4 = __toESM(require("fs"), 1);
|
|
16853
18216
|
var path2 = __toESM(require("path"), 1);
|
|
16854
|
-
var
|
|
18217
|
+
var import_ai12 = require("ai");
|
|
16855
18218
|
var import_zod17 = require("zod");
|
|
16856
18219
|
var import_p_limit = __toESM(require("p-limit"), 1);
|
|
16857
18220
|
var import_crypto = require("crypto");
|
|
@@ -16864,7 +18227,7 @@ var import_officeparser2 = require("officeparser");
|
|
|
16864
18227
|
var import_child_process2 = require("child_process");
|
|
16865
18228
|
var import_util3 = require("util");
|
|
16866
18229
|
var import_path2 = require("path");
|
|
16867
|
-
var
|
|
18230
|
+
var import_fs4 = require("fs");
|
|
16868
18231
|
var import_url2 = require("url");
|
|
16869
18232
|
var execAsync2 = (0, import_util3.promisify)(import_child_process2.exec);
|
|
16870
18233
|
function getPackageRoot2() {
|
|
@@ -16874,9 +18237,9 @@ function getPackageRoot2() {
|
|
|
16874
18237
|
const maxAttempts = 10;
|
|
16875
18238
|
while (attempts < maxAttempts) {
|
|
16876
18239
|
const packageJsonPath = (0, import_path2.join)(currentDir, "package.json");
|
|
16877
|
-
if ((0,
|
|
18240
|
+
if ((0, import_fs4.existsSync)(packageJsonPath)) {
|
|
16878
18241
|
try {
|
|
16879
|
-
const packageJson = JSON.parse((0,
|
|
18242
|
+
const packageJson = JSON.parse((0, import_fs4.readFileSync)(packageJsonPath, "utf-8"));
|
|
16880
18243
|
if (packageJson.name === "@exulu/backend") {
|
|
16881
18244
|
return currentDir;
|
|
16882
18245
|
}
|
|
@@ -16938,7 +18301,7 @@ async function executePythonScript(config) {
|
|
|
16938
18301
|
await validatePythonEnvironmentForExecution(packageRoot);
|
|
16939
18302
|
}
|
|
16940
18303
|
const resolvedScriptPath = (0, import_path2.resolve)(packageRoot, scriptPath);
|
|
16941
|
-
if (!(0,
|
|
18304
|
+
if (!(0, import_fs4.existsSync)(resolvedScriptPath)) {
|
|
16942
18305
|
throw new PythonExecutionError(
|
|
16943
18306
|
`Python script not found: ${resolvedScriptPath}`,
|
|
16944
18307
|
"",
|
|
@@ -17031,6 +18394,55 @@ async function processWord(file) {
|
|
|
17031
18394
|
markdown: content
|
|
17032
18395
|
};
|
|
17033
18396
|
}
|
|
18397
|
+
async function processImage(buffer, paths, config, verbose = false) {
|
|
18398
|
+
try {
|
|
18399
|
+
await fs4.promises.mkdir(paths.images, { recursive: true });
|
|
18400
|
+
const imagePath = path2.join(paths.images, "1.png");
|
|
18401
|
+
await fs4.promises.writeFile(imagePath, buffer);
|
|
18402
|
+
console.log(`[EXULU] Image saved to: ${imagePath}`);
|
|
18403
|
+
let json = [{
|
|
18404
|
+
page: 1,
|
|
18405
|
+
content: "",
|
|
18406
|
+
// Empty initially, will be populated by VLM if enabled
|
|
18407
|
+
image: imagePath,
|
|
18408
|
+
headings: []
|
|
18409
|
+
}];
|
|
18410
|
+
if (config?.vlm?.model) {
|
|
18411
|
+
console.log("[EXULU] Extracting content from image using VLM...");
|
|
18412
|
+
json = await validateWithVLM(
|
|
18413
|
+
json,
|
|
18414
|
+
config.vlm.model,
|
|
18415
|
+
verbose,
|
|
18416
|
+
config.vlm.concurrency
|
|
18417
|
+
);
|
|
18418
|
+
await fs4.promises.writeFile(
|
|
18419
|
+
paths.json,
|
|
18420
|
+
JSON.stringify(json, null, 2),
|
|
18421
|
+
"utf-8"
|
|
18422
|
+
);
|
|
18423
|
+
console.log("[EXULU] VLM content extraction complete");
|
|
18424
|
+
const correctedCount = json.filter((p) => p.vlm_corrected_text).length;
|
|
18425
|
+
console.log(`[EXULU] Content extracted: ${correctedCount > 0 ? "Yes" : "No"}`);
|
|
18426
|
+
} else {
|
|
18427
|
+
console.log("[EXULU] No VLM configured, image saved without content extraction");
|
|
18428
|
+
console.log("[EXULU] Note: Enable VLM in config to extract text/content from images");
|
|
18429
|
+
await fs4.promises.writeFile(
|
|
18430
|
+
paths.json,
|
|
18431
|
+
JSON.stringify(json, null, 2),
|
|
18432
|
+
"utf-8"
|
|
18433
|
+
);
|
|
18434
|
+
}
|
|
18435
|
+
const markdown = json.map((p) => p.vlm_corrected_text ?? p.content).join("\n\n\n<!-- END_OF_PAGE -->\n\n\n");
|
|
18436
|
+
await fs4.promises.writeFile(paths.markdown, markdown, "utf-8");
|
|
18437
|
+
return {
|
|
18438
|
+
markdown,
|
|
18439
|
+
json
|
|
18440
|
+
};
|
|
18441
|
+
} catch (error) {
|
|
18442
|
+
console.error("[EXULU] Error processing image:", error);
|
|
18443
|
+
throw error;
|
|
18444
|
+
}
|
|
18445
|
+
}
|
|
17034
18446
|
function normalizeMarkdownContent(content) {
|
|
17035
18447
|
const lines = content.split("\n");
|
|
17036
18448
|
const normalizedLines = [];
|
|
@@ -17074,7 +18486,7 @@ function reconstructHeadings(correctedText, headingsHierarchy) {
|
|
|
17074
18486
|
return result;
|
|
17075
18487
|
}
|
|
17076
18488
|
async function validatePageWithVLM(page, imagePath, model) {
|
|
17077
|
-
const imageBuffer = await
|
|
18489
|
+
const imageBuffer = await fs4.promises.readFile(imagePath);
|
|
17078
18490
|
const imageBase64 = imageBuffer.toString("base64");
|
|
17079
18491
|
const mimeType = "image/png";
|
|
17080
18492
|
const prompt = `You are a document validation assistant. Your task is to analyze a page image and correct the output of an OCR/parsing pipeline. The content may include tables, technical diagrams, schematics, and structured text.
|
|
@@ -17152,9 +18564,9 @@ If the page contains a flow-chart, schematic, technical drawing or control board
|
|
|
17152
18564
|
|
|
17153
18565
|
### 7. Only populate \`corrected_text\` when \`needs_correction\` is true. If the OCR output is accurate, return \`needs_correction: false\` and \`corrected_content: null\`.
|
|
17154
18566
|
`;
|
|
17155
|
-
const result = await (0,
|
|
18567
|
+
const result = await (0, import_ai12.generateText)({
|
|
17156
18568
|
model,
|
|
17157
|
-
output:
|
|
18569
|
+
output: import_ai12.Output.object({
|
|
17158
18570
|
schema: import_zod17.z.object({
|
|
17159
18571
|
needs_correction: import_zod17.z.boolean(),
|
|
17160
18572
|
corrected_text: import_zod17.z.string().nullable(),
|
|
@@ -17187,6 +18599,7 @@ If the page contains a flow-chart, schematic, technical drawing or control board
|
|
|
17187
18599
|
current_page_table: parsedOutput.current_page_table || void 0,
|
|
17188
18600
|
reasoning: parsedOutput.reasoning
|
|
17189
18601
|
};
|
|
18602
|
+
console.log(`[EXULU] VLM validation result: ${JSON.stringify(validation)}`);
|
|
17190
18603
|
return validation;
|
|
17191
18604
|
}
|
|
17192
18605
|
function reconstructTableHeaders(document2, validationResults, verbose = false) {
|
|
@@ -17242,26 +18655,29 @@ async function validateWithVLM(document2, model, verbose = false, concurrency =
|
|
|
17242
18655
|
let correctedCount = 0;
|
|
17243
18656
|
const validationTasks = document2.map(
|
|
17244
18657
|
(page) => limit(async () => {
|
|
17245
|
-
await new Promise((
|
|
18658
|
+
await new Promise((resolve3) => setImmediate(resolve3));
|
|
17246
18659
|
const imagePath = page.image;
|
|
17247
|
-
if (!page.content) {
|
|
17248
|
-
console.warn(`[EXULU] Page ${page.page}: No content found, skipping validation`);
|
|
17249
|
-
return;
|
|
17250
|
-
}
|
|
17251
18660
|
if (!imagePath) {
|
|
17252
18661
|
console.warn(`[EXULU] Page ${page.page}: No image found, skipping validation`);
|
|
17253
18662
|
return;
|
|
17254
18663
|
}
|
|
17255
|
-
|
|
17256
|
-
|
|
17257
|
-
|
|
18664
|
+
if (page.content) {
|
|
18665
|
+
const hasImage = page.content.match(/\.(jpeg|jpg|png|gif|webp)/i);
|
|
18666
|
+
const hasTable = (page.content.match(/\|/g)?.length || 0) > 1;
|
|
18667
|
+
if (!hasImage && !hasTable) {
|
|
18668
|
+
if (verbose) {
|
|
18669
|
+
console.log(`[EXULU] Page ${page.page}: No image or table found, SKIPPING VLM validation`);
|
|
18670
|
+
}
|
|
18671
|
+
return;
|
|
18672
|
+
}
|
|
18673
|
+
} else {
|
|
17258
18674
|
if (verbose) {
|
|
17259
|
-
console.log(`[EXULU] Page ${page.page}:
|
|
18675
|
+
console.log(`[EXULU] Page ${page.page}: Standalone image, proceeding with VLM content extraction`);
|
|
17260
18676
|
}
|
|
17261
|
-
return;
|
|
17262
18677
|
}
|
|
17263
18678
|
let validation;
|
|
17264
18679
|
try {
|
|
18680
|
+
console.log(`[EXULU] Validating page ${page.page} with VLM`);
|
|
17265
18681
|
validation = await withRetry(async () => {
|
|
17266
18682
|
return await validatePageWithVLM(page, imagePath, model);
|
|
17267
18683
|
}, 3);
|
|
@@ -17344,6 +18760,13 @@ async function processDocument(filePath, fileType, buffer, tempDir, config, verb
|
|
|
17344
18760
|
case "doc":
|
|
17345
18761
|
result = await processWord(buffer);
|
|
17346
18762
|
break;
|
|
18763
|
+
case "jpg":
|
|
18764
|
+
case "jpeg":
|
|
18765
|
+
case "png":
|
|
18766
|
+
case "gif":
|
|
18767
|
+
case "webp":
|
|
18768
|
+
result = await processImage(buffer, paths, config, verbose);
|
|
18769
|
+
break;
|
|
17347
18770
|
// Todo other file types with docx and officeparser
|
|
17348
18771
|
default:
|
|
17349
18772
|
throw new Error(`[EXULU] Unsupported file type: ${fileType}`);
|
|
@@ -17406,7 +18829,7 @@ ${setupResult.output || ""}`);
|
|
|
17406
18829
|
if (!result.success) {
|
|
17407
18830
|
throw new Error(`Document processing failed: ${result.stderr}`);
|
|
17408
18831
|
}
|
|
17409
|
-
const jsonContent = await
|
|
18832
|
+
const jsonContent = await fs4.promises.readFile(paths.json, "utf-8");
|
|
17410
18833
|
json = JSON.parse(jsonContent);
|
|
17411
18834
|
} else if (config?.processor.name === "officeparser") {
|
|
17412
18835
|
const text = await (0, import_officeparser2.parseOfficeAsync)(buffer, {
|
|
@@ -17423,7 +18846,7 @@ ${setupResult.output || ""}`);
|
|
|
17423
18846
|
if (!MISTRAL_API_KEY) {
|
|
17424
18847
|
throw new Error('[EXULU] MISTRAL_API_KEY is not set, please set it in the environment variable via process.env or via an Exulu variable named "MISTRAL_API_KEY".');
|
|
17425
18848
|
}
|
|
17426
|
-
await new Promise((
|
|
18849
|
+
await new Promise((resolve3) => setTimeout(resolve3, Math.floor(Math.random() * 4e3) + 1e3));
|
|
17427
18850
|
const base64Pdf = buffer.toString("base64");
|
|
17428
18851
|
const client2 = new import_mistralai.Mistral({ apiKey: MISTRAL_API_KEY });
|
|
17429
18852
|
const ocrResponse = await withRetry(async () => {
|
|
@@ -17439,9 +18862,9 @@ ${setupResult.output || ""}`);
|
|
|
17439
18862
|
}, 10);
|
|
17440
18863
|
const parser = new import_liteparse.LiteParse();
|
|
17441
18864
|
const screenshots = await parser.screenshot(paths.source, void 0);
|
|
17442
|
-
await
|
|
18865
|
+
await fs4.promises.mkdir(paths.images, { recursive: true });
|
|
17443
18866
|
for (const screenshot of screenshots) {
|
|
17444
|
-
await
|
|
18867
|
+
await fs4.promises.writeFile(
|
|
17445
18868
|
path2.join(
|
|
17446
18869
|
paths.images,
|
|
17447
18870
|
`${screenshot.pageNum}.png`
|
|
@@ -17456,15 +18879,15 @@ ${setupResult.output || ""}`);
|
|
|
17456
18879
|
image: screenshots.find((s) => s.pageNum === page.index + 1)?.imagePath,
|
|
17457
18880
|
headings: []
|
|
17458
18881
|
}));
|
|
17459
|
-
|
|
18882
|
+
fs4.writeFileSync(paths.json, JSON.stringify(json, null, 2));
|
|
17460
18883
|
} else if (config?.processor.name === "liteparse") {
|
|
17461
18884
|
const parser = new import_liteparse.LiteParse();
|
|
17462
18885
|
const result = await parser.parse(paths.source);
|
|
17463
18886
|
const screenshots = await parser.screenshot(paths.source, void 0);
|
|
17464
18887
|
console.log(`[EXULU] Liteparse screenshots: ${JSON.stringify(screenshots)}`);
|
|
17465
|
-
await
|
|
18888
|
+
await fs4.promises.mkdir(paths.images, { recursive: true });
|
|
17466
18889
|
for (const screenshot of screenshots) {
|
|
17467
|
-
await
|
|
18890
|
+
await fs4.promises.writeFile(path2.join(paths.images, `${screenshot.pageNum}.png`), screenshot.imageBuffer);
|
|
17468
18891
|
screenshot.imagePath = path2.join(paths.images, `${screenshot.pageNum}.png`);
|
|
17469
18892
|
}
|
|
17470
18893
|
json = result.pages.map((page) => ({
|
|
@@ -17472,7 +18895,7 @@ ${setupResult.output || ""}`);
|
|
|
17472
18895
|
content: page.text,
|
|
17473
18896
|
image: screenshots.find((s) => s.pageNum === page.pageNum)?.imagePath
|
|
17474
18897
|
}));
|
|
17475
|
-
|
|
18898
|
+
fs4.writeFileSync(paths.json, JSON.stringify(json, null, 2));
|
|
17476
18899
|
}
|
|
17477
18900
|
console.log(`[EXULU]
|
|
17478
18901
|
\u2713 Document processing completed successfully`);
|
|
@@ -17503,13 +18926,13 @@ ${setupResult.output || ""}`);
|
|
|
17503
18926
|
console.log(`[EXULU] Corrected: ${page.vlm_corrected_text.substring(0, 150)}...`);
|
|
17504
18927
|
});
|
|
17505
18928
|
}
|
|
17506
|
-
await
|
|
18929
|
+
await fs4.promises.writeFile(
|
|
17507
18930
|
paths.json,
|
|
17508
18931
|
JSON.stringify(json, null, 2),
|
|
17509
18932
|
"utf-8"
|
|
17510
18933
|
);
|
|
17511
18934
|
}
|
|
17512
|
-
const markdownStream =
|
|
18935
|
+
const markdownStream = fs4.createWriteStream(paths.markdown, { encoding: "utf-8" });
|
|
17513
18936
|
for (let i = 0; i < json.length; i++) {
|
|
17514
18937
|
const p = json[i];
|
|
17515
18938
|
if (!p) continue;
|
|
@@ -17519,13 +18942,13 @@ ${setupResult.output || ""}`);
|
|
|
17519
18942
|
markdownStream.write("\n\n\n<!-- END_OF_PAGE -->\n\n\n");
|
|
17520
18943
|
}
|
|
17521
18944
|
}
|
|
17522
|
-
await new Promise((
|
|
17523
|
-
markdownStream.end(() =>
|
|
18945
|
+
await new Promise((resolve3, reject) => {
|
|
18946
|
+
markdownStream.end(() => resolve3());
|
|
17524
18947
|
markdownStream.on("error", reject);
|
|
17525
18948
|
});
|
|
17526
18949
|
console.log(`[EXULU] Validated output saved to: ${paths.json}`);
|
|
17527
18950
|
console.log(`[EXULU] Validated markdown saved to: ${paths.markdown}`);
|
|
17528
|
-
const markdown = await
|
|
18951
|
+
const markdown = await fs4.promises.readFile(paths.markdown, "utf-8");
|
|
17529
18952
|
const processedJson = json.map((e) => {
|
|
17530
18953
|
const finalContent = e.vlm_corrected_text ?? e.content;
|
|
17531
18954
|
return {
|
|
@@ -17556,7 +18979,7 @@ var loadFile = async (file, name, tempDir) => {
|
|
|
17556
18979
|
let buffer;
|
|
17557
18980
|
if (Buffer.isBuffer(file)) {
|
|
17558
18981
|
filePath = path2.join(tempDir, `${UUID}.${fileType}`);
|
|
17559
|
-
await
|
|
18982
|
+
await fs4.promises.writeFile(filePath, file);
|
|
17560
18983
|
buffer = file;
|
|
17561
18984
|
} else {
|
|
17562
18985
|
filePath = filePath.trim();
|
|
@@ -17564,11 +18987,11 @@ var loadFile = async (file, name, tempDir) => {
|
|
|
17564
18987
|
const response = await fetch(filePath);
|
|
17565
18988
|
const array = await response.arrayBuffer();
|
|
17566
18989
|
const tempFilePath = path2.join(tempDir, `${UUID}.${fileType}`);
|
|
17567
|
-
await
|
|
18990
|
+
await fs4.promises.writeFile(tempFilePath, Buffer.from(array));
|
|
17568
18991
|
buffer = Buffer.from(array);
|
|
17569
18992
|
filePath = tempFilePath;
|
|
17570
18993
|
} else {
|
|
17571
|
-
buffer = await
|
|
18994
|
+
buffer = await fs4.promises.readFile(file);
|
|
17572
18995
|
}
|
|
17573
18996
|
}
|
|
17574
18997
|
return { filePath, fileType, buffer };
|
|
@@ -17586,9 +19009,9 @@ async function documentProcessor({
|
|
|
17586
19009
|
const tempDir = path2.join(process.cwd(), "temp", uuid);
|
|
17587
19010
|
const localFilesAndFoldersToDelete = [tempDir];
|
|
17588
19011
|
console.log(`[EXULU] Temporary directory for processing document ${name}: ${tempDir}`);
|
|
17589
|
-
await
|
|
19012
|
+
await fs4.promises.mkdir(tempDir, { recursive: true });
|
|
17590
19013
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
17591
|
-
await
|
|
19014
|
+
await fs4.promises.writeFile(path2.join(tempDir, "created_at.txt"), timestamp);
|
|
17592
19015
|
try {
|
|
17593
19016
|
const {
|
|
17594
19017
|
filePath,
|
|
@@ -17598,7 +19021,7 @@ async function documentProcessor({
|
|
|
17598
19021
|
let supportedTypes = [];
|
|
17599
19022
|
switch (config?.processor.name) {
|
|
17600
19023
|
case "docling":
|
|
17601
|
-
supportedTypes = ["pdf", "docx", "doc", "txt", "md"];
|
|
19024
|
+
supportedTypes = ["pdf", "docx", "doc", "txt", "md", "jpg", "jpeg", "png", "gif", "webp"];
|
|
17602
19025
|
break;
|
|
17603
19026
|
case "officeparser":
|
|
17604
19027
|
supportedTypes = [];
|
|
@@ -17607,7 +19030,7 @@ async function documentProcessor({
|
|
|
17607
19030
|
supportedTypes = ["pdf", "doc", "docx", "docm", "odt", "rtf", "ppt", "pptx", "pptm", "odp", "xls", "xlsx", "xlsm", "ods", "csv", "tsv"];
|
|
17608
19031
|
break;
|
|
17609
19032
|
case "mistral":
|
|
17610
|
-
supportedTypes = ["pdf", "docx", "doc", "txt", "md"];
|
|
19033
|
+
supportedTypes = ["pdf", "docx", "doc", "txt", "md", "jpg", "jpeg", "png", "gif", "webp"];
|
|
17611
19034
|
break;
|
|
17612
19035
|
}
|
|
17613
19036
|
if (!supportedTypes.includes(fileType)) {
|
|
@@ -17629,7 +19052,7 @@ async function documentProcessor({
|
|
|
17629
19052
|
if (config?.debugging?.deleteTempFiles !== false) {
|
|
17630
19053
|
for (const file2 of localFilesAndFoldersToDelete) {
|
|
17631
19054
|
try {
|
|
17632
|
-
await
|
|
19055
|
+
await fs4.promises.rm(file2, { recursive: true });
|
|
17633
19056
|
console.log(`[EXULU] Deleted file or folder: ${file2}`);
|
|
17634
19057
|
} catch (error) {
|
|
17635
19058
|
console.error(`[EXULU] Error deleting file or folder: ${file2}`, error);
|
|
@@ -17640,634 +19063,6 @@ async function documentProcessor({
|
|
|
17640
19063
|
}
|
|
17641
19064
|
}
|
|
17642
19065
|
|
|
17643
|
-
// ee/agentic-retrieval/v4/index.ts
|
|
17644
|
-
var os = __toESM(require("os"), 1);
|
|
17645
|
-
var path4 = __toESM(require("path"), 1);
|
|
17646
|
-
var fs7 = __toESM(require("fs/promises"), 1);
|
|
17647
|
-
var import_zod19 = require("zod");
|
|
17648
|
-
var import_crypto2 = require("crypto");
|
|
17649
|
-
|
|
17650
|
-
// ee/agentic-retrieval/v4/tools.ts
|
|
17651
|
-
var fs6 = __toESM(require("fs/promises"), 1);
|
|
17652
|
-
var path3 = __toESM(require("path"), 1);
|
|
17653
|
-
var import_child_process3 = require("child_process");
|
|
17654
|
-
var import_util4 = require("util");
|
|
17655
|
-
var import_zod18 = require("zod");
|
|
17656
|
-
var import_ai12 = require("ai");
|
|
17657
|
-
|
|
17658
|
-
// ee/agentic-retrieval/v4/embed-preprocessor.ts
|
|
17659
|
-
async function preprocessEmbedCalls(sql, contexts, user, role) {
|
|
17660
|
-
const EMBED_RE = /embed\('((?:[^'\\]|\\.)*)'\s*(?:,\s*'((?:[^'\\]|\\.)*)')?\)/gi;
|
|
17661
|
-
const matches = [];
|
|
17662
|
-
let m;
|
|
17663
|
-
while ((m = EMBED_RE.exec(sql)) !== null) {
|
|
17664
|
-
matches.push({
|
|
17665
|
-
fullMatch: m[0],
|
|
17666
|
-
text: m[1],
|
|
17667
|
-
contextId: m[2] || void 0,
|
|
17668
|
-
index: m.index
|
|
17669
|
-
});
|
|
17670
|
-
}
|
|
17671
|
-
if (matches.length === 0) return sql;
|
|
17672
|
-
const substitutions = await Promise.all(
|
|
17673
|
-
matches.map(async ({ text, contextId }) => {
|
|
17674
|
-
const context = contextId ? contexts.find((c) => c.id === contextId) : contexts.find((c) => c.embedder != null);
|
|
17675
|
-
if (!context?.embedder) {
|
|
17676
|
-
throw new Error(
|
|
17677
|
-
`No embedder available${contextId ? ` for context "${contextId}"` : ""}. Available contexts with embedders: [${contexts.filter((c) => c.embedder).map((c) => c.id).join(", ")}]`
|
|
17678
|
-
);
|
|
17679
|
-
}
|
|
17680
|
-
const result2 = await context.embedder.generateFromQuery(
|
|
17681
|
-
context.id,
|
|
17682
|
-
text,
|
|
17683
|
-
void 0,
|
|
17684
|
-
user?.id,
|
|
17685
|
-
role
|
|
17686
|
-
);
|
|
17687
|
-
const vector = result2?.chunks?.[0]?.vector;
|
|
17688
|
-
if (!vector?.length) {
|
|
17689
|
-
throw new Error(`Embedder returned no vector for text: "${text}"`);
|
|
17690
|
-
}
|
|
17691
|
-
return `ARRAY[${vector.join(",")}]::vector`;
|
|
17692
|
-
})
|
|
17693
|
-
);
|
|
17694
|
-
let result = sql;
|
|
17695
|
-
for (let i = matches.length - 1; i >= 0; i--) {
|
|
17696
|
-
const { fullMatch, index } = matches[i];
|
|
17697
|
-
result = result.slice(0, index) + substitutions[i] + result.slice(index + fullMatch.length);
|
|
17698
|
-
}
|
|
17699
|
-
return result;
|
|
17700
|
-
}
|
|
17701
|
-
|
|
17702
|
-
// ee/agentic-retrieval/v4/tools.ts
|
|
17703
|
-
var execAsync3 = (0, import_util4.promisify)(import_child_process3.exec);
|
|
17704
|
-
var MAX_INLINE_CHARS = 2e4;
|
|
17705
|
-
var MAX_GREP_OUTPUT_CHARS = 5e3;
|
|
17706
|
-
var WRITE_PATTERN = /^\s*(INSERT|UPDATE|DELETE|DROP|CREATE|ALTER|TRUNCATE|GRANT|REVOKE|VACUUM|ANALYZE|EXPLAIN\s+ANALYZE)\b/i;
|
|
17707
|
-
function assertReadOnly(sql) {
|
|
17708
|
-
if (WRITE_PATTERN.test(sql)) {
|
|
17709
|
-
throw new Error(
|
|
17710
|
-
"Only SELECT queries are allowed. Write operations (INSERT, UPDATE, DELETE, DROP, etc.) are not permitted."
|
|
17711
|
-
);
|
|
17712
|
-
}
|
|
17713
|
-
}
|
|
17714
|
-
function rowToChunkResult(row) {
|
|
17715
|
-
const chunkId = row.chunk_id ?? row.id;
|
|
17716
|
-
const chunkContent = row.chunk_content ?? row.content;
|
|
17717
|
-
const itemId = row.item_id ?? row.source;
|
|
17718
|
-
const context = row.context ?? row.context_id;
|
|
17719
|
-
const itemName = row.item_name ?? row.name;
|
|
17720
|
-
if (!chunkId || !chunkContent && !itemId) return null;
|
|
17721
|
-
return {
|
|
17722
|
-
item_name: itemName ?? "",
|
|
17723
|
-
item_id: itemId ?? "",
|
|
17724
|
-
context: context ?? "",
|
|
17725
|
-
chunk_id: chunkId,
|
|
17726
|
-
chunk_index: row.chunk_index ?? void 0,
|
|
17727
|
-
chunk_content: chunkContent ?? void 0,
|
|
17728
|
-
metadata: row.metadata ?? row.chunk_metadata ?? void 0
|
|
17729
|
-
};
|
|
17730
|
-
}
|
|
17731
|
-
function createTools(params) {
|
|
17732
|
-
const { contexts, user, role, sessionDir } = params;
|
|
17733
|
-
let queryCount = 0;
|
|
17734
|
-
const execute_query = (0, import_ai12.tool)({
|
|
17735
|
-
description: `Execute a read-only PostgreSQL SELECT query against the knowledge base.
|
|
17736
|
-
|
|
17737
|
-
Use this to search, filter, aggregate, and explore content. The database contains items
|
|
17738
|
-
and chunks tables for each knowledge base (see schema in the system prompt).
|
|
17739
|
-
|
|
17740
|
-
Use embed('your text') anywhere in the query to generate a semantic search vector:
|
|
17741
|
-
embedding <=> embed('machine learning') AS distance
|
|
17742
|
-
|
|
17743
|
-
If the result exceeds ${(MAX_INLINE_CHARS / 1e3).toFixed(0)}k characters it is saved to a file.
|
|
17744
|
-
Use the grep tool to iteratively search the file for relevant information.`,
|
|
17745
|
-
inputSchema: import_zod18.z.object({
|
|
17746
|
-
sql: import_zod18.z.string().describe("A read-only SELECT (or WITH ... SELECT) PostgreSQL query")
|
|
17747
|
-
}),
|
|
17748
|
-
execute: async ({ sql }) => {
|
|
17749
|
-
assertReadOnly(sql);
|
|
17750
|
-
let processedSql;
|
|
17751
|
-
try {
|
|
17752
|
-
processedSql = await preprocessEmbedCalls(sql, contexts, user, role);
|
|
17753
|
-
} catch (err) {
|
|
17754
|
-
return JSON.stringify({ error: `embed() preprocessing failed: ${err.message}` });
|
|
17755
|
-
}
|
|
17756
|
-
let rows;
|
|
17757
|
-
try {
|
|
17758
|
-
const { db: db2 } = await postgresClient();
|
|
17759
|
-
const result = await db2.raw(processedSql);
|
|
17760
|
-
rows = result.rows ?? [];
|
|
17761
|
-
} catch (err) {
|
|
17762
|
-
return JSON.stringify({ error: `Query failed: ${err.message}` });
|
|
17763
|
-
}
|
|
17764
|
-
const json = JSON.stringify(rows, null, 2);
|
|
17765
|
-
if (json.length <= MAX_INLINE_CHARS) {
|
|
17766
|
-
return json;
|
|
17767
|
-
}
|
|
17768
|
-
await fs6.mkdir(sessionDir, { recursive: true });
|
|
17769
|
-
const filename = `query_${++queryCount}.json`;
|
|
17770
|
-
const filePath = path3.join(sessionDir, filename);
|
|
17771
|
-
await fs6.writeFile(filePath, json, "utf-8");
|
|
17772
|
-
return JSON.stringify({
|
|
17773
|
-
stored: true,
|
|
17774
|
-
file: filePath,
|
|
17775
|
-
row_count: rows.length,
|
|
17776
|
-
message: `Results too large to display (${rows.length} rows, ${(json.length / 1e3).toFixed(1)}k chars). Stored at ${filePath}. Use the grep tool to search for relevant information.`,
|
|
17777
|
-
grep_hint: `grep -i "keyword" ${filePath}`
|
|
17778
|
-
});
|
|
17779
|
-
}
|
|
17780
|
-
});
|
|
17781
|
-
const grep = (0, import_ai12.tool)({
|
|
17782
|
-
description: `Search a stored query result file using grep.
|
|
17783
|
-
|
|
17784
|
-
Use this after execute_query returns a file path because results were too large.
|
|
17785
|
-
Iteratively narrow down the results with multiple grep calls.`,
|
|
17786
|
-
inputSchema: import_zod18.z.object({
|
|
17787
|
-
pattern: import_zod18.z.string().describe("Regular expression or literal string to search for"),
|
|
17788
|
-
file: import_zod18.z.string().describe("Absolute path to the file returned by execute_query"),
|
|
17789
|
-
context_lines: import_zod18.z.number().int().min(0).max(10).default(2).describe("Number of lines of context to show around each match (default 2)"),
|
|
17790
|
-
case_insensitive: import_zod18.z.boolean().default(true).describe("Case-insensitive matching (default true)")
|
|
17791
|
-
}),
|
|
17792
|
-
execute: async ({ pattern, file, context_lines, case_insensitive }) => {
|
|
17793
|
-
const resolvedFile = path3.resolve(file);
|
|
17794
|
-
const resolvedSession = path3.resolve(sessionDir);
|
|
17795
|
-
if (!resolvedFile.startsWith(resolvedSession)) {
|
|
17796
|
-
return JSON.stringify({
|
|
17797
|
-
error: `Access denied. Only files within the session directory (${sessionDir}) can be searched.`
|
|
17798
|
-
});
|
|
17799
|
-
}
|
|
17800
|
-
try {
|
|
17801
|
-
await fs6.access(resolvedFile);
|
|
17802
|
-
} catch {
|
|
17803
|
-
return JSON.stringify({ error: `File not found: ${file}` });
|
|
17804
|
-
}
|
|
17805
|
-
const flags = [
|
|
17806
|
-
"-n",
|
|
17807
|
-
context_lines > 0 ? `-C${context_lines}` : "",
|
|
17808
|
-
case_insensitive ? "-i" : ""
|
|
17809
|
-
].filter(Boolean).join(" ");
|
|
17810
|
-
const escapedPattern = pattern.replace(/'/g, `'\\''`);
|
|
17811
|
-
const cmd = `grep ${flags} '${escapedPattern}' '${resolvedFile}'`;
|
|
17812
|
-
let output;
|
|
17813
|
-
try {
|
|
17814
|
-
const { stdout } = await execAsync3(cmd, { maxBuffer: 10 * 1024 * 1024 });
|
|
17815
|
-
output = stdout;
|
|
17816
|
-
} catch (err) {
|
|
17817
|
-
if (err.code === 1) {
|
|
17818
|
-
return JSON.stringify({ matches: 0, output: "No matches found." });
|
|
17819
|
-
}
|
|
17820
|
-
return JSON.stringify({ error: `grep failed: ${err.message}` });
|
|
17821
|
-
}
|
|
17822
|
-
if (output.length > MAX_GREP_OUTPUT_CHARS) {
|
|
17823
|
-
output = output.slice(0, MAX_GREP_OUTPUT_CHARS) + `
|
|
17824
|
-
... (output truncated at ${MAX_GREP_OUTPUT_CHARS} chars \u2014 refine your pattern to narrow results)`;
|
|
17825
|
-
}
|
|
17826
|
-
const lineCount = output.split("\n").filter(Boolean).length;
|
|
17827
|
-
return JSON.stringify({ matches: lineCount, output });
|
|
17828
|
-
}
|
|
17829
|
-
});
|
|
17830
|
-
return { execute_query, grep };
|
|
17831
|
-
}
|
|
17832
|
-
function harvestChunks(toolResults) {
|
|
17833
|
-
const chunks = [];
|
|
17834
|
-
for (const result of toolResults ?? []) {
|
|
17835
|
-
const rawOutput = result.output ?? result.result;
|
|
17836
|
-
let parsed;
|
|
17837
|
-
try {
|
|
17838
|
-
parsed = typeof rawOutput === "string" ? JSON.parse(rawOutput) : rawOutput;
|
|
17839
|
-
} catch {
|
|
17840
|
-
continue;
|
|
17841
|
-
}
|
|
17842
|
-
if (Array.isArray(parsed)) {
|
|
17843
|
-
for (const row of parsed) {
|
|
17844
|
-
if (row && typeof row === "object") {
|
|
17845
|
-
const chunk = rowToChunkResult(row);
|
|
17846
|
-
if (chunk) chunks.push(chunk);
|
|
17847
|
-
}
|
|
17848
|
-
}
|
|
17849
|
-
}
|
|
17850
|
-
}
|
|
17851
|
-
return chunks;
|
|
17852
|
-
}
|
|
17853
|
-
|
|
17854
|
-
// ee/agentic-retrieval/v4/system-prompt.ts
|
|
17855
|
-
function buildSystemPrompt(contexts, customInstructions) {
|
|
17856
|
-
const schemaBlock = buildSchemaBlock(contexts);
|
|
17857
|
-
const hasEmbedder = contexts.some((c) => c.embedder != null);
|
|
17858
|
-
return `You are a knowledge base retrieval agent. Your job is to find all information relevant to the user's query.
|
|
17859
|
-
|
|
17860
|
-
## Approach: Observe \u2192 Infer \u2192 Act
|
|
17861
|
-
|
|
17862
|
-
Work iteratively:
|
|
17863
|
-
1. **Observe** \u2014 examine what data you have and what the query asks for
|
|
17864
|
-
2. **Infer** \u2014 decide what SQL query will best surface relevant information
|
|
17865
|
-
3. **Act** \u2014 execute the query and study the results
|
|
17866
|
-
4. Repeat until you have found sufficient information, then write your final answer.
|
|
17867
|
-
|
|
17868
|
-
Do NOT guess or hallucinate. If results are empty, try alternative queries (different keywords,
|
|
17869
|
-
broader filters, semantic search). Exhaust the available search strategies before concluding
|
|
17870
|
-
that no relevant data exists.
|
|
17871
|
-
|
|
17872
|
-
---
|
|
17873
|
-
|
|
17874
|
-
## Database Schema
|
|
17875
|
-
|
|
17876
|
-
${schemaBlock}
|
|
17877
|
-
|
|
17878
|
-
---
|
|
17879
|
-
|
|
17880
|
-
## Query Patterns
|
|
17881
|
-
|
|
17882
|
-
### Keyword / Full-Text Search
|
|
17883
|
-
\`\`\`sql
|
|
17884
|
-
SELECT
|
|
17885
|
-
c.id AS chunk_id,
|
|
17886
|
-
c.chunk_index,
|
|
17887
|
-
c.content AS chunk_content,
|
|
17888
|
-
c.metadata,
|
|
17889
|
-
c.source AS item_id,
|
|
17890
|
-
i.name AS item_name,
|
|
17891
|
-
'<context_id>' AS context
|
|
17892
|
-
FROM <context_id>_chunks c
|
|
17893
|
-
JOIN <context_id>_items i ON c.source = i.id
|
|
17894
|
-
WHERE c.fts @@ plainto_tsquery('english', 'your search terms')
|
|
17895
|
-
AND (i.archived IS FALSE OR i.archived IS NULL)
|
|
17896
|
-
ORDER BY ts_rank(c.fts, plainto_tsquery('english', 'your search terms')) DESC
|
|
17897
|
-
LIMIT 20;
|
|
17898
|
-
\`\`\`
|
|
17899
|
-
|
|
17900
|
-
For German text use \`'german'\` instead of \`'english'\`.
|
|
17901
|
-
For multi-language, use \`websearch_to_tsquery\` or UNION both languages.
|
|
17902
|
-
${hasEmbedder ? `
|
|
17903
|
-
### Semantic Search (use embed() helper)
|
|
17904
|
-
\`\`\`sql
|
|
17905
|
-
SELECT
|
|
17906
|
-
c.id AS chunk_id,
|
|
17907
|
-
c.chunk_index,
|
|
17908
|
-
c.content AS chunk_content,
|
|
17909
|
-
c.metadata,
|
|
17910
|
-
c.source AS item_id,
|
|
17911
|
-
i.name AS item_name,
|
|
17912
|
-
'<context_id>' AS context,
|
|
17913
|
-
c.embedding <=> embed('your concept here') AS distance
|
|
17914
|
-
FROM <context_id>_chunks c
|
|
17915
|
-
JOIN <context_id>_items i ON c.source = i.id
|
|
17916
|
-
WHERE (i.archived IS FALSE OR i.archived IS NULL)
|
|
17917
|
-
ORDER BY distance ASC
|
|
17918
|
-
LIMIT 20;
|
|
17919
|
-
\`\`\`
|
|
17920
|
-
|
|
17921
|
-
### Hybrid Search (keyword + semantic combined via RRF)
|
|
17922
|
-
\`\`\`sql
|
|
17923
|
-
WITH fts AS (
|
|
17924
|
-
SELECT id, ROW_NUMBER() OVER (ORDER BY ts_rank(fts, q) DESC) AS rank
|
|
17925
|
-
FROM <context_id>_chunks, plainto_tsquery('english', 'your query') q
|
|
17926
|
-
WHERE fts @@ q
|
|
17927
|
-
LIMIT 500
|
|
17928
|
-
),
|
|
17929
|
-
sem AS (
|
|
17930
|
-
SELECT id, ROW_NUMBER() OVER (ORDER BY embedding <=> embed('your query') ASC) AS rank
|
|
17931
|
-
FROM <context_id>_chunks
|
|
17932
|
-
LIMIT 500
|
|
17933
|
-
),
|
|
17934
|
-
rrf AS (
|
|
17935
|
-
SELECT
|
|
17936
|
-
COALESCE(fts.id, sem.id) AS id,
|
|
17937
|
-
(COALESCE(1.0 / (50 + fts.rank), 0) * 2 + COALESCE(1.0 / (50 + sem.rank), 0)) AS score
|
|
17938
|
-
FROM fts FULL OUTER JOIN sem ON fts.id = sem.id
|
|
17939
|
-
)
|
|
17940
|
-
SELECT
|
|
17941
|
-
c.id AS chunk_id,
|
|
17942
|
-
c.chunk_index,
|
|
17943
|
-
c.content AS chunk_content,
|
|
17944
|
-
c.metadata,
|
|
17945
|
-
c.source AS item_id,
|
|
17946
|
-
i.name AS item_name,
|
|
17947
|
-
'<context_id>' AS context,
|
|
17948
|
-
rrf.score
|
|
17949
|
-
FROM rrf
|
|
17950
|
-
JOIN <context_id>_chunks c ON c.id = rrf.id
|
|
17951
|
-
JOIN <context_id>_items i ON c.source = i.id
|
|
17952
|
-
WHERE (i.archived IS FALSE OR i.archived IS NULL)
|
|
17953
|
-
ORDER BY rrf.score DESC
|
|
17954
|
-
LIMIT 20;
|
|
17955
|
-
\`\`\`
|
|
17956
|
-
` : `
|
|
17957
|
-
Note: No embedder is configured for these contexts. Use keyword/full-text search only.
|
|
17958
|
-
`}
|
|
17959
|
-
### Browse all chunks of a specific document (in order)
|
|
17960
|
-
\`\`\`sql
|
|
17961
|
-
SELECT
|
|
17962
|
-
c.id AS chunk_id,
|
|
17963
|
-
c.chunk_index,
|
|
17964
|
-
c.content AS chunk_content,
|
|
17965
|
-
c.metadata,
|
|
17966
|
-
c.source AS item_id,
|
|
17967
|
-
i.name AS item_name,
|
|
17968
|
-
'<context_id>' AS context
|
|
17969
|
-
FROM <context_id>_chunks c
|
|
17970
|
-
JOIN <context_id>_items i ON c.source = i.id
|
|
17971
|
-
WHERE c.source = '<item_id>'
|
|
17972
|
-
ORDER BY c.chunk_index;
|
|
17973
|
-
\`\`\`
|
|
17974
|
-
|
|
17975
|
-
### Count / aggregate
|
|
17976
|
-
\`\`\`sql
|
|
17977
|
-
SELECT COUNT(*) FROM <context_id>_items WHERE archived IS FALSE;
|
|
17978
|
-
SELECT COUNT(*) FROM <context_id>_chunks;
|
|
17979
|
-
\`\`\`
|
|
17980
|
-
|
|
17981
|
-
### Explore item names (when query is about a specific document)
|
|
17982
|
-
\`\`\`sql
|
|
17983
|
-
SELECT id, name, external_id, "createdAt"
|
|
17984
|
-
FROM <context_id>_items
|
|
17985
|
-
WHERE (archived IS FALSE OR archived IS NULL)
|
|
17986
|
-
AND LOWER(name) LIKE '%keyword%'
|
|
17987
|
-
LIMIT 50;
|
|
17988
|
-
\`\`\`
|
|
17989
|
-
|
|
17990
|
-
### Filter by custom metadata on chunks
|
|
17991
|
-
\`\`\`sql
|
|
17992
|
-
SELECT chunk_id, chunk_content, item_name, context
|
|
17993
|
-
FROM ...
|
|
17994
|
-
WHERE c.metadata->>'page' = '5'
|
|
17995
|
-
OR c.metadata @> '{"category": "finance"}'
|
|
17996
|
-
\`\`\`
|
|
17997
|
-
|
|
17998
|
-
---
|
|
17999
|
-
|
|
18000
|
-
## Column Alias Convention
|
|
18001
|
-
|
|
18002
|
-
**Always use these aliases** in queries that return chunks so results are collected correctly:
|
|
18003
|
-
|
|
18004
|
-
| Alias | Source column |
|
|
18005
|
-
|----------------|-------------------------|
|
|
18006
|
-
| \`chunk_id\` | \`c.id\` |
|
|
18007
|
-
| \`chunk_index\` | \`c.chunk_index\` |
|
|
18008
|
-
| \`chunk_content\`| \`c.content\` |
|
|
18009
|
-
| \`item_id\` | \`c.source\` |
|
|
18010
|
-
| \`item_name\` | \`i.name\` |
|
|
18011
|
-
| \`context\` | literal context id string |
|
|
18012
|
-
| \`metadata\` | \`c.metadata\` |
|
|
18013
|
-
|
|
18014
|
-
---
|
|
18015
|
-
|
|
18016
|
-
## Handling Large Results
|
|
18017
|
-
|
|
18018
|
-
When execute_query returns a file path (results > 20k chars):
|
|
18019
|
-
1. Use \`grep\` with a specific pattern to find relevant sections
|
|
18020
|
-
2. Multiple grep calls are fine \u2014 narrow down iteratively
|
|
18021
|
-
3. Once you know specific \`item_id\` or \`chunk_id\` values, run a targeted SELECT to get full content
|
|
18022
|
-
|
|
18023
|
-
---
|
|
18024
|
-
|
|
18025
|
-
## Search Strategy
|
|
18026
|
-
|
|
18027
|
-
- **Start broad**: use keyword or hybrid search with your main terms, LIMIT 30\u201350
|
|
18028
|
-
- **Go deeper**: if results are sparse, try alternative phrasings, synonyms, or semantic search
|
|
18029
|
-
- **Drill into documents**: once you find a relevant item, fetch its chunks in order to get full context
|
|
18030
|
-
- **Cross-context**: search multiple contexts when the query could span knowledge bases
|
|
18031
|
-
- **Aggregate last**: use COUNT queries only for "how many" questions
|
|
18032
|
-
|
|
18033
|
-
---
|
|
18034
|
-
${customInstructions ? `## Additional Instructions
|
|
18035
|
-
|
|
18036
|
-
${customInstructions}
|
|
18037
|
-
|
|
18038
|
-
---
|
|
18039
|
-
` : ""}
|
|
18040
|
-
When you have gathered sufficient information, write a clear answer. Do not call any more tools once you have what you need.`;
|
|
18041
|
-
}
|
|
18042
|
-
function buildSchemaBlock(contexts) {
|
|
18043
|
-
return contexts.map((ctx) => {
|
|
18044
|
-
const itemsTable = getTableName(ctx.id);
|
|
18045
|
-
const chunksTable = getChunksTableName(ctx.id);
|
|
18046
|
-
const customFields = ctx.fields.length > 0 ? ctx.fields.map((f) => ` ${f.name} (${f.type})`).join("\n") : " (no custom fields)";
|
|
18047
|
-
const embedderNote = ctx.embedder ? `Embedder: ${ctx.embedder.name} \u2014 semantic search and embed() are available` : "No embedder \u2014 use keyword search only";
|
|
18048
|
-
return `### Context: "${ctx.name}" (id: \`${ctx.id}\`)
|
|
18049
|
-
${ctx.description || ""}
|
|
18050
|
-
${embedderNote}
|
|
18051
|
-
|
|
18052
|
-
**${itemsTable}** \u2014 documents / items
|
|
18053
|
-
id (uuid, primary key)
|
|
18054
|
-
name (text)
|
|
18055
|
-
external_id (text, nullable)
|
|
18056
|
-
archived (boolean, nullable)
|
|
18057
|
-
created_by (integer, nullable)
|
|
18058
|
-
rights_mode (text, nullable)
|
|
18059
|
-
"createdAt" (timestamp)
|
|
18060
|
-
"updatedAt" (timestamp)
|
|
18061
|
-
-- Custom fields:
|
|
18062
|
-
${customFields}
|
|
18063
|
-
|
|
18064
|
-
**${chunksTable}** \u2014 text chunks (source FK \u2192 ${itemsTable}.id)
|
|
18065
|
-
id (uuid, primary key)
|
|
18066
|
-
source (uuid, FK \u2192 ${itemsTable}.id)
|
|
18067
|
-
content (text)
|
|
18068
|
-
chunk_index (integer)
|
|
18069
|
-
fts (tsvector \u2014 full-text search index)
|
|
18070
|
-
embedding (vector \u2014 pgvector, nullable)
|
|
18071
|
-
metadata (jsonb, nullable)
|
|
18072
|
-
"createdAt" (timestamp)
|
|
18073
|
-
"updatedAt" (timestamp)`;
|
|
18074
|
-
}).join("\n\n");
|
|
18075
|
-
}
|
|
18076
|
-
|
|
18077
|
-
// ee/agentic-retrieval/v4/agent-loop.ts
|
|
18078
|
-
var import_ai13 = require("ai");
|
|
18079
|
-
var MAX_STEPS = 10;
|
|
18080
|
-
async function* runAgentLoop2(params) {
|
|
18081
|
-
const { query, systemPrompt, tools, model, onStepComplete } = params;
|
|
18082
|
-
const output = {
|
|
18083
|
-
steps: [],
|
|
18084
|
-
reasoning: [],
|
|
18085
|
-
chunks: [],
|
|
18086
|
-
usage: [],
|
|
18087
|
-
totalTokens: 0
|
|
18088
|
-
};
|
|
18089
|
-
const seenChunkIds = /* @__PURE__ */ new Set();
|
|
18090
|
-
const messages = [{ role: "user", content: query }];
|
|
18091
|
-
for (let step = 0; step < MAX_STEPS; step++) {
|
|
18092
|
-
console.log(`[EXULU] v4 agent loop \u2014 step ${step + 1}/${MAX_STEPS}`);
|
|
18093
|
-
let result;
|
|
18094
|
-
try {
|
|
18095
|
-
result = await withRetry(
|
|
18096
|
-
() => (0, import_ai13.generateText)({
|
|
18097
|
-
model,
|
|
18098
|
-
temperature: 0,
|
|
18099
|
-
system: systemPrompt,
|
|
18100
|
-
messages,
|
|
18101
|
-
tools,
|
|
18102
|
-
toolChoice: "auto",
|
|
18103
|
-
stopWhen: (0, import_ai13.stepCountIs)(1)
|
|
18104
|
-
})
|
|
18105
|
-
);
|
|
18106
|
-
} catch (err) {
|
|
18107
|
-
console.error("[EXULU] v4 generateText failed:", err);
|
|
18108
|
-
throw err;
|
|
18109
|
-
}
|
|
18110
|
-
messages.push(...result.response.messages);
|
|
18111
|
-
const rawToolResults = result.toolResults ?? [];
|
|
18112
|
-
const stepChunks = [];
|
|
18113
|
-
for (const chunk of harvestChunks(rawToolResults)) {
|
|
18114
|
-
if (!chunk.chunk_id || !seenChunkIds.has(chunk.chunk_id)) {
|
|
18115
|
-
if (chunk.chunk_id) seenChunkIds.add(chunk.chunk_id);
|
|
18116
|
-
stepChunks.push(chunk);
|
|
18117
|
-
}
|
|
18118
|
-
}
|
|
18119
|
-
const stepRecord = {
|
|
18120
|
-
stepNumber: step + 1,
|
|
18121
|
-
text: result.text ?? "",
|
|
18122
|
-
toolCalls: result.toolCalls?.map((tc) => ({
|
|
18123
|
-
name: tc.toolName,
|
|
18124
|
-
id: tc.toolCallId,
|
|
18125
|
-
input: tc.input
|
|
18126
|
-
})) ?? [],
|
|
18127
|
-
chunks: stepChunks,
|
|
18128
|
-
tokens: result.usage?.totalTokens ?? 0
|
|
18129
|
-
};
|
|
18130
|
-
output.steps.push(stepRecord);
|
|
18131
|
-
output.reasoning.push({
|
|
18132
|
-
text: result.text ?? "",
|
|
18133
|
-
tools: result.toolCalls?.map((tc) => ({
|
|
18134
|
-
name: tc.toolName,
|
|
18135
|
-
id: tc.toolCallId,
|
|
18136
|
-
input: tc.input,
|
|
18137
|
-
output: rawToolResults.find(
|
|
18138
|
-
(r) => (r.toolCallId ?? r.id) === tc.toolCallId
|
|
18139
|
-
)?.output
|
|
18140
|
-
})) ?? []
|
|
18141
|
-
});
|
|
18142
|
-
output.chunks.push(...stepChunks);
|
|
18143
|
-
output.usage.push(result.usage);
|
|
18144
|
-
onStepComplete?.(stepRecord);
|
|
18145
|
-
yield { ...output };
|
|
18146
|
-
const calledTools = result.toolCalls?.length > 0;
|
|
18147
|
-
if (!calledTools) {
|
|
18148
|
-
console.log(`[EXULU] v4 \u2014 model finished after step ${step + 1} (no tool calls)`);
|
|
18149
|
-
break;
|
|
18150
|
-
}
|
|
18151
|
-
}
|
|
18152
|
-
output.totalTokens = output.usage.reduce((sum, u) => sum + (u?.totalTokens ?? 0), 0);
|
|
18153
|
-
}
|
|
18154
|
-
|
|
18155
|
-
// ee/agentic-retrieval/v4/index.ts
|
|
18156
|
-
async function* executeV4({
|
|
18157
|
-
query,
|
|
18158
|
-
contexts,
|
|
18159
|
-
model,
|
|
18160
|
-
user,
|
|
18161
|
-
role,
|
|
18162
|
-
customInstructions
|
|
18163
|
-
}) {
|
|
18164
|
-
const sessionId = (0, import_crypto2.randomUUID)();
|
|
18165
|
-
const sessionDir = path4.join(os.tmpdir(), `exulu-v4-${sessionId}`);
|
|
18166
|
-
console.log("[EXULU] v4 \u2014 starting observe-infer-act retrieval");
|
|
18167
|
-
const tools = createTools({ contexts, user, role, sessionDir });
|
|
18168
|
-
const systemPrompt = buildSystemPrompt(contexts, customInstructions);
|
|
18169
|
-
let finalOutput;
|
|
18170
|
-
try {
|
|
18171
|
-
for await (const output of runAgentLoop2({
|
|
18172
|
-
query,
|
|
18173
|
-
systemPrompt,
|
|
18174
|
-
tools,
|
|
18175
|
-
model
|
|
18176
|
-
})) {
|
|
18177
|
-
finalOutput = output;
|
|
18178
|
-
yield output;
|
|
18179
|
-
}
|
|
18180
|
-
} finally {
|
|
18181
|
-
fs7.rm(sessionDir, { recursive: true, force: true }).catch(() => {
|
|
18182
|
-
});
|
|
18183
|
-
}
|
|
18184
|
-
if (finalOutput) {
|
|
18185
|
-
console.log(
|
|
18186
|
-
`[EXULU] v4 \u2014 done. steps=${finalOutput.steps.length} chunks=${finalOutput.chunks.length} tokens=${finalOutput.totalTokens}`
|
|
18187
|
-
);
|
|
18188
|
-
}
|
|
18189
|
-
}
|
|
18190
|
-
function createAgenticRetrievalToolV4({
|
|
18191
|
-
contexts,
|
|
18192
|
-
instructions: adminInstructions,
|
|
18193
|
-
rerankers,
|
|
18194
|
-
user,
|
|
18195
|
-
role,
|
|
18196
|
-
model
|
|
18197
|
-
}) {
|
|
18198
|
-
const license = checkLicense();
|
|
18199
|
-
if (!license["agentic-retrieval"]) {
|
|
18200
|
-
console.warn("[EXULU] Not licensed for agentic retrieval");
|
|
18201
|
-
return void 0;
|
|
18202
|
-
}
|
|
18203
|
-
const contextNames = contexts.map((c) => c.id).join(", ");
|
|
18204
|
-
return new ExuluTool({
|
|
18205
|
-
id: "agentic_context_search_v4",
|
|
18206
|
-
name: "Agentic Context Search (V4)",
|
|
18207
|
-
description: `Observe-infer-act retrieval using raw SQL. Searches: ${contextNames}`,
|
|
18208
|
-
category: "contexts",
|
|
18209
|
-
needsApproval: false,
|
|
18210
|
-
type: "context",
|
|
18211
|
-
config: [
|
|
18212
|
-
{
|
|
18213
|
-
name: "instructions",
|
|
18214
|
-
description: "Custom instructions for the retrieval agent",
|
|
18215
|
-
type: "string",
|
|
18216
|
-
default: ""
|
|
18217
|
-
},
|
|
18218
|
-
{
|
|
18219
|
-
name: "reasoning_model",
|
|
18220
|
-
description: "Override the model used by the retrieval agent (default: inherits from calling agent)",
|
|
18221
|
-
type: "string",
|
|
18222
|
-
default: ""
|
|
18223
|
-
},
|
|
18224
|
-
...contexts.map((ctx) => ({
|
|
18225
|
-
name: ctx.id,
|
|
18226
|
-
description: `Enable search in "${ctx.name}". ${ctx.description}`,
|
|
18227
|
-
type: "boolean",
|
|
18228
|
-
default: true
|
|
18229
|
-
}))
|
|
18230
|
-
],
|
|
18231
|
-
inputSchema: import_zod19.z.object({
|
|
18232
|
-
query: import_zod19.z.string().describe("The question or query to answer"),
|
|
18233
|
-
userInstructions: import_zod19.z.string().optional().describe("Additional instructions from the user to guide retrieval")
|
|
18234
|
-
}),
|
|
18235
|
-
execute: async function* ({
|
|
18236
|
-
query,
|
|
18237
|
-
userInstructions,
|
|
18238
|
-
toolVariablesConfig
|
|
18239
|
-
}) {
|
|
18240
|
-
if (!model) {
|
|
18241
|
-
throw new Error("Model is required for executing the agentic retrieval tool");
|
|
18242
|
-
}
|
|
18243
|
-
let activeContexts = contexts;
|
|
18244
|
-
let configInstructions = "";
|
|
18245
|
-
if (toolVariablesConfig) {
|
|
18246
|
-
configInstructions = toolVariablesConfig["instructions"] ?? "";
|
|
18247
|
-
activeContexts = contexts.filter(
|
|
18248
|
-
(ctx) => toolVariablesConfig[ctx.id] === true || toolVariablesConfig[ctx.id] === "true" || toolVariablesConfig[ctx.id] === 1
|
|
18249
|
-
);
|
|
18250
|
-
if (activeContexts.length === 0) activeContexts = contexts;
|
|
18251
|
-
}
|
|
18252
|
-
const combinedInstructions = [
|
|
18253
|
-
configInstructions ? `Configuration instructions: ${configInstructions}` : "",
|
|
18254
|
-
adminInstructions ? `Admin instructions: ${adminInstructions}` : "",
|
|
18255
|
-
userInstructions ? `User instructions: ${userInstructions}` : ""
|
|
18256
|
-
].filter(Boolean).join("\n");
|
|
18257
|
-
for await (const output of executeV4({
|
|
18258
|
-
query,
|
|
18259
|
-
contexts: activeContexts,
|
|
18260
|
-
model,
|
|
18261
|
-
user,
|
|
18262
|
-
role,
|
|
18263
|
-
customInstructions: combinedInstructions || void 0
|
|
18264
|
-
})) {
|
|
18265
|
-
yield { result: JSON.stringify(output) };
|
|
18266
|
-
}
|
|
18267
|
-
}
|
|
18268
|
-
});
|
|
18269
|
-
}
|
|
18270
|
-
|
|
18271
19066
|
// src/index.ts
|
|
18272
19067
|
var ExuluJobs = {
|
|
18273
19068
|
redis: redisClient
|
|
@@ -18276,8 +19071,7 @@ var ExuluDefaultTools = {
|
|
|
18276
19071
|
agentic: {
|
|
18277
19072
|
retrieval: {
|
|
18278
19073
|
create: {
|
|
18279
|
-
v3: createAgenticRetrievalToolV3
|
|
18280
|
-
v4: createAgenticRetrievalToolV4
|
|
19074
|
+
v3: createAgenticRetrievalToolV3
|
|
18281
19075
|
}
|
|
18282
19076
|
}
|
|
18283
19077
|
}
|
|
@@ -18318,8 +19112,8 @@ var ExuluVariables = {
|
|
|
18318
19112
|
throw new Error(`Variable ${name} not found.`);
|
|
18319
19113
|
}
|
|
18320
19114
|
if (variable.encrypted) {
|
|
18321
|
-
const bytes =
|
|
18322
|
-
variable.value = bytes.toString(
|
|
19115
|
+
const bytes = import_crypto_js10.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
19116
|
+
variable.value = bytes.toString(import_crypto_js10.default.enc.Utf8);
|
|
18323
19117
|
}
|
|
18324
19118
|
return variable.value;
|
|
18325
19119
|
}
|