@exulu/backend 1.51.0 → 1.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +60 -37
- package/dist/index.js +60 -37
- package/ee/chunking/markdown.ts +1 -0
- package/ee/python/documents/processing/doc_processor.ts +15 -3
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -10863,6 +10863,7 @@ var providerRateLimiter = async (key, windowSeconds, limit, points) => {
|
|
|
10863
10863
|
|
|
10864
10864
|
// src/exulu/routes.ts
|
|
10865
10865
|
var import_zod_from_json_schema = require("zod-from-json-schema");
|
|
10866
|
+
var import_zod8 = require("zod");
|
|
10866
10867
|
var REQUEST_SIZE_LIMIT = "50mb";
|
|
10867
10868
|
var getExuluVersionNumber = async () => {
|
|
10868
10869
|
try {
|
|
@@ -11330,12 +11331,16 @@ Mood: friendly and intelligent.
|
|
|
11330
11331
|
message = req.body.message;
|
|
11331
11332
|
}
|
|
11332
11333
|
const approvedTools = req.body.approvedTools ? typeof req.body.approvedTools === "string" ? JSON.parse(req.body.approvedTools) : req.body.approvedTools : [];
|
|
11334
|
+
const customInstructions = req.body.customInstructions ? typeof req.body.customInstructions === "string" ? req.body.customInstructions : JSON.stringify(req.body.customInstructions) : "";
|
|
11335
|
+
const instructions = customInstructions ? `${agent.instructions}
|
|
11336
|
+
|
|
11337
|
+
${customInstructions}` : agent.instructions;
|
|
11333
11338
|
const result = await provider.generateStream({
|
|
11334
11339
|
contexts,
|
|
11335
11340
|
rerankers: rerankers || [],
|
|
11336
11341
|
agent,
|
|
11337
11342
|
user,
|
|
11338
|
-
instructions
|
|
11343
|
+
instructions,
|
|
11339
11344
|
session: headers.session,
|
|
11340
11345
|
message,
|
|
11341
11346
|
previousMessages,
|
|
@@ -11436,6 +11441,10 @@ Mood: friendly and intelligent.
|
|
|
11436
11441
|
});
|
|
11437
11442
|
return;
|
|
11438
11443
|
} else {
|
|
11444
|
+
const customInstructions = req.body.customInstructions ? typeof req.body.customInstructions === "string" ? req.body.customInstructions : JSON.stringify(req.body.customInstructions) : "";
|
|
11445
|
+
const instructions = customInstructions ? `${agent.instructions}
|
|
11446
|
+
|
|
11447
|
+
${customInstructions}` : agent.instructions;
|
|
11439
11448
|
const response = await provider.generateSync({
|
|
11440
11449
|
contexts,
|
|
11441
11450
|
rerankers: rerankers || [],
|
|
@@ -11443,7 +11452,7 @@ Mood: friendly and intelligent.
|
|
|
11443
11452
|
agent,
|
|
11444
11453
|
user,
|
|
11445
11454
|
req,
|
|
11446
|
-
instructions
|
|
11455
|
+
instructions,
|
|
11447
11456
|
session: headers.session,
|
|
11448
11457
|
inputMessages: [req.body.message],
|
|
11449
11458
|
currentTools: enabledTools,
|
|
@@ -11588,6 +11597,10 @@ Mood: friendly and intelligent.
|
|
|
11588
11597
|
providers,
|
|
11589
11598
|
user
|
|
11590
11599
|
);
|
|
11600
|
+
const customInstructions = req.body.customInstructions ? typeof req.body.customInstructions === "string" ? req.body.customInstructions : JSON.stringify(req.body.customInstructions) : "";
|
|
11601
|
+
const agentInstructions = customInstructions ? `${agent?.instructions}
|
|
11602
|
+
|
|
11603
|
+
${customInstructions}` : agent?.instructions;
|
|
11591
11604
|
let system = req.body.system;
|
|
11592
11605
|
if (Array.isArray(req.body.system)) {
|
|
11593
11606
|
system = [
|
|
@@ -11597,7 +11610,7 @@ Mood: friendly and intelligent.
|
|
|
11597
11610
|
type: "text",
|
|
11598
11611
|
text: `
|
|
11599
11612
|
You are an agent named: ${agent?.name}
|
|
11600
|
-
Here are some additional instructions for you: ${
|
|
11613
|
+
Here are some additional instructions for you: ${agentInstructions}`
|
|
11601
11614
|
}
|
|
11602
11615
|
] : [],
|
|
11603
11616
|
...project ? [
|
|
@@ -11615,7 +11628,7 @@ Mood: friendly and intelligent.
|
|
|
11615
11628
|
|
|
11616
11629
|
|
|
11617
11630
|
${agent ? `You are an agent named: ${agent?.name}
|
|
11618
|
-
Here are some additional instructions for you: ${
|
|
11631
|
+
Here are some additional instructions for you: ${agentInstructions}` : ""}
|
|
11619
11632
|
|
|
11620
11633
|
${project?.id ? `Additional information:
|
|
11621
11634
|
|
|
@@ -11765,7 +11778,7 @@ var import_types2 = require("@modelcontextprotocol/sdk/types.js");
|
|
|
11765
11778
|
var import_express4 = require("express");
|
|
11766
11779
|
var import_api3 = require("@opentelemetry/api");
|
|
11767
11780
|
var import_crypto_js7 = __toESM(require("crypto-js"), 1);
|
|
11768
|
-
var
|
|
11781
|
+
var import_zod9 = require("zod");
|
|
11769
11782
|
var SESSION_ID_HEADER = "mcp-session-id";
|
|
11770
11783
|
var ExuluMCP = class {
|
|
11771
11784
|
server = {};
|
|
@@ -11848,7 +11861,7 @@ var ExuluMCP = class {
|
|
|
11848
11861
|
title: tool3.name + " agent",
|
|
11849
11862
|
description: tool3.description,
|
|
11850
11863
|
inputSchema: {
|
|
11851
|
-
inputs: tool3.inputSchema ||
|
|
11864
|
+
inputs: tool3.inputSchema || import_zod9.z.object({})
|
|
11852
11865
|
}
|
|
11853
11866
|
},
|
|
11854
11867
|
async ({ inputs }, args) => {
|
|
@@ -11900,7 +11913,7 @@ var ExuluMCP = class {
|
|
|
11900
11913
|
title: "Get List of Prompt Templates",
|
|
11901
11914
|
description: "Retrieves a list of prompt templates available for this agent. Returns the name, description, and ID of each template.",
|
|
11902
11915
|
inputSchema: {
|
|
11903
|
-
inputs:
|
|
11916
|
+
inputs: import_zod9.z.object({})
|
|
11904
11917
|
}
|
|
11905
11918
|
},
|
|
11906
11919
|
async ({ inputs }, args) => {
|
|
@@ -11946,8 +11959,8 @@ var ExuluMCP = class {
|
|
|
11946
11959
|
title: "Get Prompt Template Details",
|
|
11947
11960
|
description: "Retrieves the full details of a specific prompt template by ID, including the actual template content with variables.",
|
|
11948
11961
|
inputSchema: {
|
|
11949
|
-
inputs:
|
|
11950
|
-
id:
|
|
11962
|
+
inputs: import_zod9.z.object({
|
|
11963
|
+
id: import_zod9.z.string().describe("The ID of the prompt template to retrieve")
|
|
11951
11964
|
})
|
|
11952
11965
|
}
|
|
11953
11966
|
},
|
|
@@ -12855,7 +12868,7 @@ var ExuluEval = class {
|
|
|
12855
12868
|
};
|
|
12856
12869
|
|
|
12857
12870
|
// src/templates/evals/index.ts
|
|
12858
|
-
var
|
|
12871
|
+
var import_zod10 = require("zod");
|
|
12859
12872
|
var llmAsJudgeEval = () => {
|
|
12860
12873
|
if (process.env.REDIS_HOST?.length && process.env.REDIS_PORT?.length) {
|
|
12861
12874
|
return new ExuluEval({
|
|
@@ -12900,8 +12913,8 @@ var llmAsJudgeEval = () => {
|
|
|
12900
12913
|
contexts: [],
|
|
12901
12914
|
rerankers: [],
|
|
12902
12915
|
prompt,
|
|
12903
|
-
outputSchema:
|
|
12904
|
-
score:
|
|
12916
|
+
outputSchema: import_zod10.z.object({
|
|
12917
|
+
score: import_zod10.z.number().min(0).max(100).describe("The score between 0 and 100.")
|
|
12905
12918
|
}),
|
|
12906
12919
|
providerapikey
|
|
12907
12920
|
});
|
|
@@ -13129,12 +13142,12 @@ Usage:
|
|
|
13129
13142
|
- If no todos exist yet, an empty list will be returned`;
|
|
13130
13143
|
|
|
13131
13144
|
// src/templates/tools/todo/todo.ts
|
|
13132
|
-
var
|
|
13133
|
-
var TodoSchema =
|
|
13134
|
-
content:
|
|
13135
|
-
status:
|
|
13136
|
-
priority:
|
|
13137
|
-
id:
|
|
13145
|
+
var import_zod11 = __toESM(require("zod"), 1);
|
|
13146
|
+
var TodoSchema = import_zod11.default.object({
|
|
13147
|
+
content: import_zod11.default.string().describe("Brief description of the task"),
|
|
13148
|
+
status: import_zod11.default.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
|
|
13149
|
+
priority: import_zod11.default.string().describe("Priority level of the task: high, medium, low"),
|
|
13150
|
+
id: import_zod11.default.string().describe("Unique identifier for the todo item")
|
|
13138
13151
|
});
|
|
13139
13152
|
var TodoWriteTool = new ExuluTool({
|
|
13140
13153
|
id: "todo_write",
|
|
@@ -13150,8 +13163,8 @@ var TodoWriteTool = new ExuluTool({
|
|
|
13150
13163
|
default: todowrite_default
|
|
13151
13164
|
}
|
|
13152
13165
|
],
|
|
13153
|
-
inputSchema:
|
|
13154
|
-
todos:
|
|
13166
|
+
inputSchema: import_zod11.default.object({
|
|
13167
|
+
todos: import_zod11.default.array(TodoSchema).describe("The updated todo list")
|
|
13155
13168
|
}),
|
|
13156
13169
|
execute: async (inputs) => {
|
|
13157
13170
|
const { sessionID, todos, user } = inputs;
|
|
@@ -13186,7 +13199,7 @@ var TodoReadTool = new ExuluTool({
|
|
|
13186
13199
|
id: "todo_read",
|
|
13187
13200
|
name: "Todo Read",
|
|
13188
13201
|
description: "Use this tool to read your todo list",
|
|
13189
|
-
inputSchema:
|
|
13202
|
+
inputSchema: import_zod11.default.object({}),
|
|
13190
13203
|
type: "function",
|
|
13191
13204
|
category: "todo",
|
|
13192
13205
|
config: [
|
|
@@ -13225,15 +13238,15 @@ async function getTodos(sessionID) {
|
|
|
13225
13238
|
var todoTools = [TodoWriteTool, TodoReadTool];
|
|
13226
13239
|
|
|
13227
13240
|
// src/templates/tools/perplexity.ts
|
|
13228
|
-
var
|
|
13241
|
+
var import_zod12 = __toESM(require("zod"), 1);
|
|
13229
13242
|
var import_perplexity_ai = __toESM(require("@perplexity-ai/perplexity_ai"), 1);
|
|
13230
13243
|
var internetSearchTool = new ExuluTool({
|
|
13231
13244
|
id: "internet_search",
|
|
13232
13245
|
name: "Perplexity Live Internet Search",
|
|
13233
13246
|
description: "Search the internet for information.",
|
|
13234
|
-
inputSchema:
|
|
13235
|
-
query:
|
|
13236
|
-
search_recency_filter:
|
|
13247
|
+
inputSchema: import_zod12.default.object({
|
|
13248
|
+
query: import_zod12.default.string().describe("The query to the tool."),
|
|
13249
|
+
search_recency_filter: import_zod12.default.enum(["day", "week", "month", "year"]).optional().describe("The recency filter for the search, can be day, week, month or year.")
|
|
13237
13250
|
}),
|
|
13238
13251
|
category: "internet_search",
|
|
13239
13252
|
type: "web_search",
|
|
@@ -15953,6 +15966,7 @@ var MarkdownChunker = class {
|
|
|
15953
15966
|
if (currentSlice.length === 0) {
|
|
15954
15967
|
currentPosition++;
|
|
15955
15968
|
targetPosition = currentPosition + chunkSize * this._CHARS_PER_TOKEN;
|
|
15969
|
+
contentLeft = text.length - currentPosition;
|
|
15956
15970
|
}
|
|
15957
15971
|
}
|
|
15958
15972
|
const mergedChunks = [];
|
|
@@ -16192,7 +16206,7 @@ Or manually run the setup script:
|
|
|
16192
16206
|
var fs2 = __toESM(require("fs"), 1);
|
|
16193
16207
|
var path = __toESM(require("path"), 1);
|
|
16194
16208
|
var import_ai7 = require("ai");
|
|
16195
|
-
var
|
|
16209
|
+
var import_zod13 = require("zod");
|
|
16196
16210
|
var import_p_limit = __toESM(require("p-limit"), 1);
|
|
16197
16211
|
var import_crypto = require("crypto");
|
|
16198
16212
|
var mammoth = __toESM(require("mammoth"), 1);
|
|
@@ -16495,15 +16509,15 @@ If the page contains a flow-chart, schematic, technical drawing or control board
|
|
|
16495
16509
|
const result = await (0, import_ai7.generateText)({
|
|
16496
16510
|
model,
|
|
16497
16511
|
output: import_ai7.Output.object({
|
|
16498
|
-
schema:
|
|
16499
|
-
needs_correction:
|
|
16500
|
-
corrected_text:
|
|
16501
|
-
current_page_table:
|
|
16502
|
-
headers:
|
|
16503
|
-
is_continuation:
|
|
16512
|
+
schema: import_zod13.z.object({
|
|
16513
|
+
needs_correction: import_zod13.z.boolean(),
|
|
16514
|
+
corrected_text: import_zod13.z.string().nullable(),
|
|
16515
|
+
current_page_table: import_zod13.z.object({
|
|
16516
|
+
headers: import_zod13.z.array(import_zod13.z.string()),
|
|
16517
|
+
is_continuation: import_zod13.z.boolean()
|
|
16504
16518
|
}).nullable(),
|
|
16505
|
-
confidence:
|
|
16506
|
-
reasoning:
|
|
16519
|
+
confidence: import_zod13.z.enum(["high", "medium", "low"]),
|
|
16520
|
+
reasoning: import_zod13.z.string()
|
|
16507
16521
|
})
|
|
16508
16522
|
}),
|
|
16509
16523
|
messages: [
|
|
@@ -16696,6 +16710,14 @@ async function processDocument(filePath, fileType, buffer, tempDir, config, verb
|
|
|
16696
16710
|
}
|
|
16697
16711
|
};
|
|
16698
16712
|
}
|
|
16713
|
+
var getMistralApiKey = async () => {
|
|
16714
|
+
if (process.env.MISTRAL_API_KEY) {
|
|
16715
|
+
return process.env.MISTRAL_API_KEY;
|
|
16716
|
+
} else {
|
|
16717
|
+
const variable = await ExuluVariables.get("MISTRAL_API_KEY");
|
|
16718
|
+
return variable;
|
|
16719
|
+
}
|
|
16720
|
+
};
|
|
16699
16721
|
async function processPdf(buffer, paths, config, verbose = false) {
|
|
16700
16722
|
try {
|
|
16701
16723
|
let json = [];
|
|
@@ -16751,12 +16773,13 @@ ${setupResult.output || ""}`);
|
|
|
16751
16773
|
headings: []
|
|
16752
16774
|
}];
|
|
16753
16775
|
} else if (config?.processor.name === "mistral") {
|
|
16754
|
-
|
|
16755
|
-
|
|
16776
|
+
const MISTRAL_API_KEY = await getMistralApiKey();
|
|
16777
|
+
if (!MISTRAL_API_KEY) {
|
|
16778
|
+
throw new Error('[EXULU] MISTRAL_API_KEY is not set, please set it in the environment variable via process.env or via an Exulu variable named "MISTRAL_API_KEY".');
|
|
16756
16779
|
}
|
|
16757
16780
|
await new Promise((resolve3) => setTimeout(resolve3, Math.floor(Math.random() * 4e3) + 1e3));
|
|
16758
16781
|
const base64Pdf = buffer.toString("base64");
|
|
16759
|
-
const client2 = new import_mistralai.Mistral({ apiKey:
|
|
16782
|
+
const client2 = new import_mistralai.Mistral({ apiKey: MISTRAL_API_KEY });
|
|
16760
16783
|
const ocrResponse = await withRetry(async () => {
|
|
16761
16784
|
const ocrResponse2 = await client2.ocr.process({
|
|
16762
16785
|
document: {
|
package/dist/index.js
CHANGED
|
@@ -10829,6 +10829,7 @@ var providerRateLimiter = async (key, windowSeconds, limit, points) => {
|
|
|
10829
10829
|
|
|
10830
10830
|
// src/exulu/routes.ts
|
|
10831
10831
|
import { convertJsonSchemaToZod } from "zod-from-json-schema";
|
|
10832
|
+
import "zod";
|
|
10832
10833
|
var REQUEST_SIZE_LIMIT = "50mb";
|
|
10833
10834
|
var getExuluVersionNumber = async () => {
|
|
10834
10835
|
try {
|
|
@@ -11296,12 +11297,16 @@ Mood: friendly and intelligent.
|
|
|
11296
11297
|
message = req.body.message;
|
|
11297
11298
|
}
|
|
11298
11299
|
const approvedTools = req.body.approvedTools ? typeof req.body.approvedTools === "string" ? JSON.parse(req.body.approvedTools) : req.body.approvedTools : [];
|
|
11300
|
+
const customInstructions = req.body.customInstructions ? typeof req.body.customInstructions === "string" ? req.body.customInstructions : JSON.stringify(req.body.customInstructions) : "";
|
|
11301
|
+
const instructions = customInstructions ? `${agent.instructions}
|
|
11302
|
+
|
|
11303
|
+
${customInstructions}` : agent.instructions;
|
|
11299
11304
|
const result = await provider.generateStream({
|
|
11300
11305
|
contexts,
|
|
11301
11306
|
rerankers: rerankers || [],
|
|
11302
11307
|
agent,
|
|
11303
11308
|
user,
|
|
11304
|
-
instructions
|
|
11309
|
+
instructions,
|
|
11305
11310
|
session: headers.session,
|
|
11306
11311
|
message,
|
|
11307
11312
|
previousMessages,
|
|
@@ -11402,6 +11407,10 @@ Mood: friendly and intelligent.
|
|
|
11402
11407
|
});
|
|
11403
11408
|
return;
|
|
11404
11409
|
} else {
|
|
11410
|
+
const customInstructions = req.body.customInstructions ? typeof req.body.customInstructions === "string" ? req.body.customInstructions : JSON.stringify(req.body.customInstructions) : "";
|
|
11411
|
+
const instructions = customInstructions ? `${agent.instructions}
|
|
11412
|
+
|
|
11413
|
+
${customInstructions}` : agent.instructions;
|
|
11405
11414
|
const response = await provider.generateSync({
|
|
11406
11415
|
contexts,
|
|
11407
11416
|
rerankers: rerankers || [],
|
|
@@ -11409,7 +11418,7 @@ Mood: friendly and intelligent.
|
|
|
11409
11418
|
agent,
|
|
11410
11419
|
user,
|
|
11411
11420
|
req,
|
|
11412
|
-
instructions
|
|
11421
|
+
instructions,
|
|
11413
11422
|
session: headers.session,
|
|
11414
11423
|
inputMessages: [req.body.message],
|
|
11415
11424
|
currentTools: enabledTools,
|
|
@@ -11554,6 +11563,10 @@ Mood: friendly and intelligent.
|
|
|
11554
11563
|
providers,
|
|
11555
11564
|
user
|
|
11556
11565
|
);
|
|
11566
|
+
const customInstructions = req.body.customInstructions ? typeof req.body.customInstructions === "string" ? req.body.customInstructions : JSON.stringify(req.body.customInstructions) : "";
|
|
11567
|
+
const agentInstructions = customInstructions ? `${agent?.instructions}
|
|
11568
|
+
|
|
11569
|
+
${customInstructions}` : agent?.instructions;
|
|
11557
11570
|
let system = req.body.system;
|
|
11558
11571
|
if (Array.isArray(req.body.system)) {
|
|
11559
11572
|
system = [
|
|
@@ -11563,7 +11576,7 @@ Mood: friendly and intelligent.
|
|
|
11563
11576
|
type: "text",
|
|
11564
11577
|
text: `
|
|
11565
11578
|
You are an agent named: ${agent?.name}
|
|
11566
|
-
Here are some additional instructions for you: ${
|
|
11579
|
+
Here are some additional instructions for you: ${agentInstructions}`
|
|
11567
11580
|
}
|
|
11568
11581
|
] : [],
|
|
11569
11582
|
...project ? [
|
|
@@ -11581,7 +11594,7 @@ Mood: friendly and intelligent.
|
|
|
11581
11594
|
|
|
11582
11595
|
|
|
11583
11596
|
${agent ? `You are an agent named: ${agent?.name}
|
|
11584
|
-
Here are some additional instructions for you: ${
|
|
11597
|
+
Here are some additional instructions for you: ${agentInstructions}` : ""}
|
|
11585
11598
|
|
|
11586
11599
|
${project?.id ? `Additional information:
|
|
11587
11600
|
|
|
@@ -11731,7 +11744,7 @@ import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
|
|
|
11731
11744
|
import "express";
|
|
11732
11745
|
import "@opentelemetry/api";
|
|
11733
11746
|
import CryptoJS7 from "crypto-js";
|
|
11734
|
-
import { z as
|
|
11747
|
+
import { z as z9 } from "zod";
|
|
11735
11748
|
var SESSION_ID_HEADER = "mcp-session-id";
|
|
11736
11749
|
var ExuluMCP = class {
|
|
11737
11750
|
server = {};
|
|
@@ -11814,7 +11827,7 @@ var ExuluMCP = class {
|
|
|
11814
11827
|
title: tool3.name + " agent",
|
|
11815
11828
|
description: tool3.description,
|
|
11816
11829
|
inputSchema: {
|
|
11817
|
-
inputs: tool3.inputSchema ||
|
|
11830
|
+
inputs: tool3.inputSchema || z9.object({})
|
|
11818
11831
|
}
|
|
11819
11832
|
},
|
|
11820
11833
|
async ({ inputs }, args) => {
|
|
@@ -11866,7 +11879,7 @@ var ExuluMCP = class {
|
|
|
11866
11879
|
title: "Get List of Prompt Templates",
|
|
11867
11880
|
description: "Retrieves a list of prompt templates available for this agent. Returns the name, description, and ID of each template.",
|
|
11868
11881
|
inputSchema: {
|
|
11869
|
-
inputs:
|
|
11882
|
+
inputs: z9.object({})
|
|
11870
11883
|
}
|
|
11871
11884
|
},
|
|
11872
11885
|
async ({ inputs }, args) => {
|
|
@@ -11912,8 +11925,8 @@ var ExuluMCP = class {
|
|
|
11912
11925
|
title: "Get Prompt Template Details",
|
|
11913
11926
|
description: "Retrieves the full details of a specific prompt template by ID, including the actual template content with variables.",
|
|
11914
11927
|
inputSchema: {
|
|
11915
|
-
inputs:
|
|
11916
|
-
id:
|
|
11928
|
+
inputs: z9.object({
|
|
11929
|
+
id: z9.string().describe("The ID of the prompt template to retrieve")
|
|
11917
11930
|
})
|
|
11918
11931
|
}
|
|
11919
11932
|
},
|
|
@@ -12821,7 +12834,7 @@ var ExuluEval = class {
|
|
|
12821
12834
|
};
|
|
12822
12835
|
|
|
12823
12836
|
// src/templates/evals/index.ts
|
|
12824
|
-
import { z as
|
|
12837
|
+
import { z as z10 } from "zod";
|
|
12825
12838
|
var llmAsJudgeEval = () => {
|
|
12826
12839
|
if (process.env.REDIS_HOST?.length && process.env.REDIS_PORT?.length) {
|
|
12827
12840
|
return new ExuluEval({
|
|
@@ -12866,8 +12879,8 @@ var llmAsJudgeEval = () => {
|
|
|
12866
12879
|
contexts: [],
|
|
12867
12880
|
rerankers: [],
|
|
12868
12881
|
prompt,
|
|
12869
|
-
outputSchema:
|
|
12870
|
-
score:
|
|
12882
|
+
outputSchema: z10.object({
|
|
12883
|
+
score: z10.number().min(0).max(100).describe("The score between 0 and 100.")
|
|
12871
12884
|
}),
|
|
12872
12885
|
providerapikey
|
|
12873
12886
|
});
|
|
@@ -13095,12 +13108,12 @@ Usage:
|
|
|
13095
13108
|
- If no todos exist yet, an empty list will be returned`;
|
|
13096
13109
|
|
|
13097
13110
|
// src/templates/tools/todo/todo.ts
|
|
13098
|
-
import
|
|
13099
|
-
var TodoSchema =
|
|
13100
|
-
content:
|
|
13101
|
-
status:
|
|
13102
|
-
priority:
|
|
13103
|
-
id:
|
|
13111
|
+
import z11 from "zod";
|
|
13112
|
+
var TodoSchema = z11.object({
|
|
13113
|
+
content: z11.string().describe("Brief description of the task"),
|
|
13114
|
+
status: z11.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
|
|
13115
|
+
priority: z11.string().describe("Priority level of the task: high, medium, low"),
|
|
13116
|
+
id: z11.string().describe("Unique identifier for the todo item")
|
|
13104
13117
|
});
|
|
13105
13118
|
var TodoWriteTool = new ExuluTool({
|
|
13106
13119
|
id: "todo_write",
|
|
@@ -13116,8 +13129,8 @@ var TodoWriteTool = new ExuluTool({
|
|
|
13116
13129
|
default: todowrite_default
|
|
13117
13130
|
}
|
|
13118
13131
|
],
|
|
13119
|
-
inputSchema:
|
|
13120
|
-
todos:
|
|
13132
|
+
inputSchema: z11.object({
|
|
13133
|
+
todos: z11.array(TodoSchema).describe("The updated todo list")
|
|
13121
13134
|
}),
|
|
13122
13135
|
execute: async (inputs) => {
|
|
13123
13136
|
const { sessionID, todos, user } = inputs;
|
|
@@ -13152,7 +13165,7 @@ var TodoReadTool = new ExuluTool({
|
|
|
13152
13165
|
id: "todo_read",
|
|
13153
13166
|
name: "Todo Read",
|
|
13154
13167
|
description: "Use this tool to read your todo list",
|
|
13155
|
-
inputSchema:
|
|
13168
|
+
inputSchema: z11.object({}),
|
|
13156
13169
|
type: "function",
|
|
13157
13170
|
category: "todo",
|
|
13158
13171
|
config: [
|
|
@@ -13191,15 +13204,15 @@ async function getTodos(sessionID) {
|
|
|
13191
13204
|
var todoTools = [TodoWriteTool, TodoReadTool];
|
|
13192
13205
|
|
|
13193
13206
|
// src/templates/tools/perplexity.ts
|
|
13194
|
-
import
|
|
13207
|
+
import z12 from "zod";
|
|
13195
13208
|
import Perplexity from "@perplexity-ai/perplexity_ai";
|
|
13196
13209
|
var internetSearchTool = new ExuluTool({
|
|
13197
13210
|
id: "internet_search",
|
|
13198
13211
|
name: "Perplexity Live Internet Search",
|
|
13199
13212
|
description: "Search the internet for information.",
|
|
13200
|
-
inputSchema:
|
|
13201
|
-
query:
|
|
13202
|
-
search_recency_filter:
|
|
13213
|
+
inputSchema: z12.object({
|
|
13214
|
+
query: z12.string().describe("The query to the tool."),
|
|
13215
|
+
search_recency_filter: z12.enum(["day", "week", "month", "year"]).optional().describe("The recency filter for the search, can be day, week, month or year.")
|
|
13203
13216
|
}),
|
|
13204
13217
|
category: "internet_search",
|
|
13205
13218
|
type: "web_search",
|
|
@@ -15919,6 +15932,7 @@ var MarkdownChunker = class {
|
|
|
15919
15932
|
if (currentSlice.length === 0) {
|
|
15920
15933
|
currentPosition++;
|
|
15921
15934
|
targetPosition = currentPosition + chunkSize * this._CHARS_PER_TOKEN;
|
|
15935
|
+
contentLeft = text.length - currentPosition;
|
|
15922
15936
|
}
|
|
15923
15937
|
}
|
|
15924
15938
|
const mergedChunks = [];
|
|
@@ -16158,7 +16172,7 @@ Or manually run the setup script:
|
|
|
16158
16172
|
import * as fs2 from "fs";
|
|
16159
16173
|
import * as path from "path";
|
|
16160
16174
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
16161
|
-
import { z as
|
|
16175
|
+
import { z as z13 } from "zod";
|
|
16162
16176
|
import pLimit from "p-limit";
|
|
16163
16177
|
import { randomUUID as randomUUID6 } from "crypto";
|
|
16164
16178
|
import * as mammoth from "mammoth";
|
|
@@ -16461,15 +16475,15 @@ If the page contains a flow-chart, schematic, technical drawing or control board
|
|
|
16461
16475
|
const result = await generateText3({
|
|
16462
16476
|
model,
|
|
16463
16477
|
output: Output3.object({
|
|
16464
|
-
schema:
|
|
16465
|
-
needs_correction:
|
|
16466
|
-
corrected_text:
|
|
16467
|
-
current_page_table:
|
|
16468
|
-
headers:
|
|
16469
|
-
is_continuation:
|
|
16478
|
+
schema: z13.object({
|
|
16479
|
+
needs_correction: z13.boolean(),
|
|
16480
|
+
corrected_text: z13.string().nullable(),
|
|
16481
|
+
current_page_table: z13.object({
|
|
16482
|
+
headers: z13.array(z13.string()),
|
|
16483
|
+
is_continuation: z13.boolean()
|
|
16470
16484
|
}).nullable(),
|
|
16471
|
-
confidence:
|
|
16472
|
-
reasoning:
|
|
16485
|
+
confidence: z13.enum(["high", "medium", "low"]),
|
|
16486
|
+
reasoning: z13.string()
|
|
16473
16487
|
})
|
|
16474
16488
|
}),
|
|
16475
16489
|
messages: [
|
|
@@ -16662,6 +16676,14 @@ async function processDocument(filePath, fileType, buffer, tempDir, config, verb
|
|
|
16662
16676
|
}
|
|
16663
16677
|
};
|
|
16664
16678
|
}
|
|
16679
|
+
var getMistralApiKey = async () => {
|
|
16680
|
+
if (process.env.MISTRAL_API_KEY) {
|
|
16681
|
+
return process.env.MISTRAL_API_KEY;
|
|
16682
|
+
} else {
|
|
16683
|
+
const variable = await ExuluVariables.get("MISTRAL_API_KEY");
|
|
16684
|
+
return variable;
|
|
16685
|
+
}
|
|
16686
|
+
};
|
|
16665
16687
|
async function processPdf(buffer, paths, config, verbose = false) {
|
|
16666
16688
|
try {
|
|
16667
16689
|
let json = [];
|
|
@@ -16717,12 +16739,13 @@ ${setupResult.output || ""}`);
|
|
|
16717
16739
|
headings: []
|
|
16718
16740
|
}];
|
|
16719
16741
|
} else if (config?.processor.name === "mistral") {
|
|
16720
|
-
|
|
16721
|
-
|
|
16742
|
+
const MISTRAL_API_KEY = await getMistralApiKey();
|
|
16743
|
+
if (!MISTRAL_API_KEY) {
|
|
16744
|
+
throw new Error('[EXULU] MISTRAL_API_KEY is not set, please set it in the environment variable via process.env or via an Exulu variable named "MISTRAL_API_KEY".');
|
|
16722
16745
|
}
|
|
16723
16746
|
await new Promise((resolve3) => setTimeout(resolve3, Math.floor(Math.random() * 4e3) + 1e3));
|
|
16724
16747
|
const base64Pdf = buffer.toString("base64");
|
|
16725
|
-
const client2 = new Mistral({ apiKey:
|
|
16748
|
+
const client2 = new Mistral({ apiKey: MISTRAL_API_KEY });
|
|
16726
16749
|
const ocrResponse = await withRetry(async () => {
|
|
16727
16750
|
const ocrResponse2 = await client2.ocr.process({
|
|
16728
16751
|
document: {
|
package/ee/chunking/markdown.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { executePythonScript } from '@SRC/utils/python-executor';
|
|
|
15
15
|
import { setupPythonEnvironment, validatePythonEnvironment } from '@SRC/utils/python-setup';
|
|
16
16
|
import { LiteParse } from '@llamaindex/liteparse';
|
|
17
17
|
import { Mistral } from '@mistralai/mistralai';
|
|
18
|
+
import { ExuluVariables } from '@SRC/index';
|
|
18
19
|
|
|
19
20
|
type DocumentProcessorConfig = {
|
|
20
21
|
vlm?: {
|
|
@@ -586,6 +587,15 @@ async function processDocument(
|
|
|
586
587
|
};
|
|
587
588
|
}
|
|
588
589
|
|
|
590
|
+
const getMistralApiKey = async () => {
|
|
591
|
+
if (process.env.MISTRAL_API_KEY) {
|
|
592
|
+
return process.env.MISTRAL_API_KEY;
|
|
593
|
+
} else {
|
|
594
|
+
const variable = await ExuluVariables.get("MISTRAL_API_KEY");
|
|
595
|
+
return variable;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
589
599
|
async function processPdf(
|
|
590
600
|
buffer: Buffer,
|
|
591
601
|
paths: ProcessingPaths,
|
|
@@ -656,15 +666,17 @@ async function processPdf(
|
|
|
656
666
|
}];
|
|
657
667
|
|
|
658
668
|
} else if (config?.processor.name === "mistral") {
|
|
659
|
-
|
|
660
|
-
|
|
669
|
+
|
|
670
|
+
const MISTRAL_API_KEY = await getMistralApiKey();
|
|
671
|
+
if (!MISTRAL_API_KEY) {
|
|
672
|
+
throw new Error('[EXULU] MISTRAL_API_KEY is not set, please set it in the environment variable via process.env or via an Exulu variable named "MISTRAL_API_KEY".');
|
|
661
673
|
}
|
|
662
674
|
|
|
663
675
|
// Wait a randomn time between 1 and 5 seconds to prevent rate limiting
|
|
664
676
|
await new Promise(resolve => setTimeout(resolve, Math.floor(Math.random() * 4000) + 1000));
|
|
665
677
|
|
|
666
678
|
const base64Pdf = buffer.toString('base64');
|
|
667
|
-
const client = new Mistral({ apiKey:
|
|
679
|
+
const client = new Mistral({ apiKey: MISTRAL_API_KEY });
|
|
668
680
|
|
|
669
681
|
const ocrResponse = await withRetry(async () => {
|
|
670
682
|
type MistralOCRResponse = Awaited<ReturnType<typeof client.ocr.process>>;
|