@exulu/backend 1.28.0 → 1.28.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -3
- package/dist/index.cjs +143 -94
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +138 -89
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
|
|
1
|
+
## [1.28.1](https://github.com/Qventu/exulu-backend/compare/v1.28.0...v1.28.1) (2025-10-24)
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
###
|
|
4
|
+
### Bug Fixes
|
|
5
5
|
|
|
6
|
-
*
|
|
6
|
+
* made fileUploads config in ExuluApp optional ([078bc9b](https://github.com/Qventu/exulu-backend/commit/078bc9ba3e294387e5cae5a25ee7144353aa6482))
|
package/dist/index.cjs
CHANGED
|
@@ -3247,7 +3247,7 @@ var contextToTableDefinition = (context) => {
|
|
|
3247
3247
|
});
|
|
3248
3248
|
return addCoreFields(definition);
|
|
3249
3249
|
};
|
|
3250
|
-
function createSDL(tables, contexts, agents, tools, config, evals,
|
|
3250
|
+
function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
|
|
3251
3251
|
const contextSchemas = contexts.map((context) => contextToTableDefinition(context));
|
|
3252
3252
|
tables.forEach((table) => {
|
|
3253
3253
|
if (!table.fields.some((field) => field.name === "createdAt")) {
|
|
@@ -4053,6 +4053,9 @@ var import_node_crypto = require("crypto");
|
|
|
4053
4053
|
var expiresIn = 60 * 60 * 24 * 1;
|
|
4054
4054
|
var s3Client;
|
|
4055
4055
|
function getS3Client(config) {
|
|
4056
|
+
if (!config.fileUploads) {
|
|
4057
|
+
throw new Error("File uploads are not configured");
|
|
4058
|
+
}
|
|
4056
4059
|
s3Client ??= new import_client_s3.S3Client({
|
|
4057
4060
|
region: config.fileUploads.s3region,
|
|
4058
4061
|
...config.fileUploads.s3endpoint && {
|
|
@@ -4067,6 +4070,9 @@ function getS3Client(config) {
|
|
|
4067
4070
|
return s3Client;
|
|
4068
4071
|
}
|
|
4069
4072
|
var getPresignedUrl = async (key, config) => {
|
|
4073
|
+
if (!config.fileUploads) {
|
|
4074
|
+
throw new Error("File uploads are not configured");
|
|
4075
|
+
}
|
|
4070
4076
|
const url = await (0, import_s3_request_presigner.getSignedUrl)(
|
|
4071
4077
|
getS3Client(config),
|
|
4072
4078
|
new import_client_s3.GetObjectCommand({
|
|
@@ -4078,6 +4084,9 @@ var getPresignedUrl = async (key, config) => {
|
|
|
4078
4084
|
return url;
|
|
4079
4085
|
};
|
|
4080
4086
|
var addPrefixToKey = (keyPath, config) => {
|
|
4087
|
+
if (!config.fileUploads) {
|
|
4088
|
+
throw new Error("File uploads are not configured");
|
|
4089
|
+
}
|
|
4081
4090
|
if (!config.fileUploads.s3prefix) {
|
|
4082
4091
|
return keyPath;
|
|
4083
4092
|
}
|
|
@@ -4089,6 +4098,9 @@ var addPrefixToKey = (keyPath, config) => {
|
|
|
4089
4098
|
};
|
|
4090
4099
|
var uploadFile = async (user, file, key, config, options = {}) => {
|
|
4091
4100
|
console.log("[EXULU] Uploading file to S3", key);
|
|
4101
|
+
if (!config.fileUploads) {
|
|
4102
|
+
throw new Error("File uploads are not configured");
|
|
4103
|
+
}
|
|
4092
4104
|
const client2 = getS3Client(config);
|
|
4093
4105
|
let folder = `${user}/`;
|
|
4094
4106
|
const fullKey = addPrefixToKey(!key.includes(folder) ? folder + key : key, config);
|
|
@@ -4104,7 +4116,13 @@ var uploadFile = async (user, file, key, config, options = {}) => {
|
|
|
4104
4116
|
return key;
|
|
4105
4117
|
};
|
|
4106
4118
|
var createUppyRoutes = async (app, config) => {
|
|
4119
|
+
if (!config.fileUploads) {
|
|
4120
|
+
throw new Error("File uploads are not configured");
|
|
4121
|
+
}
|
|
4107
4122
|
const extractUserPrefix = (key) => {
|
|
4123
|
+
if (!config.fileUploads) {
|
|
4124
|
+
throw new Error("File uploads are not configured");
|
|
4125
|
+
}
|
|
4108
4126
|
if (!config.fileUploads.s3prefix) {
|
|
4109
4127
|
return key.split("/")[0];
|
|
4110
4128
|
}
|
|
@@ -4132,6 +4150,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4132
4150
|
};
|
|
4133
4151
|
let stsClient;
|
|
4134
4152
|
function getSTSClient() {
|
|
4153
|
+
if (!config.fileUploads) {
|
|
4154
|
+
throw new Error("File uploads are not configured");
|
|
4155
|
+
}
|
|
4135
4156
|
stsClient ??= new import_client_sts.STSClient({
|
|
4136
4157
|
region: config.fileUploads.s3region,
|
|
4137
4158
|
...config.fileUploads.s3endpoint && { endpoint: config.fileUploads.s3endpoint },
|
|
@@ -4143,6 +4164,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4143
4164
|
return stsClient;
|
|
4144
4165
|
}
|
|
4145
4166
|
app.delete("/s3/delete", async (req, res, next) => {
|
|
4167
|
+
if (!config.fileUploads) {
|
|
4168
|
+
throw new Error("File uploads are not configured");
|
|
4169
|
+
}
|
|
4146
4170
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4147
4171
|
const internalkey = req.headers["internal-key"] || null;
|
|
4148
4172
|
const { db: db3 } = await postgresClient();
|
|
@@ -4233,6 +4257,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4233
4257
|
}
|
|
4234
4258
|
});
|
|
4235
4259
|
app.post("/s3/object", async (req, res, next) => {
|
|
4260
|
+
if (!config.fileUploads) {
|
|
4261
|
+
throw new Error("File uploads are not configured");
|
|
4262
|
+
}
|
|
4236
4263
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4237
4264
|
const internalkey = req.headers["internal-key"] || null;
|
|
4238
4265
|
const { db: db3 } = await postgresClient();
|
|
@@ -4263,6 +4290,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4263
4290
|
res.end();
|
|
4264
4291
|
});
|
|
4265
4292
|
app.get("/s3/list", async (req, res, next) => {
|
|
4293
|
+
if (!config.fileUploads) {
|
|
4294
|
+
throw new Error("File uploads are not configured");
|
|
4295
|
+
}
|
|
4266
4296
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4267
4297
|
const internalkey = req.headers["internal-key"] || null;
|
|
4268
4298
|
const { db: db3 } = await postgresClient();
|
|
@@ -4299,6 +4329,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4299
4329
|
res.end();
|
|
4300
4330
|
});
|
|
4301
4331
|
app.get("/s3/sts", (req, res, next) => {
|
|
4332
|
+
if (!config.fileUploads) {
|
|
4333
|
+
throw new Error("File uploads are not configured");
|
|
4334
|
+
}
|
|
4302
4335
|
getSTSClient().send(new import_client_sts.GetFederationTokenCommand({
|
|
4303
4336
|
Name: "Exulu",
|
|
4304
4337
|
// The duration, in seconds, of the role session. The value specified
|
|
@@ -4311,8 +4344,8 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4311
4344
|
res.setHeader("Cache-Control", `public,max-age=${expiresIn}`);
|
|
4312
4345
|
res.json({
|
|
4313
4346
|
credentials: response.Credentials,
|
|
4314
|
-
bucket: config.fileUploads
|
|
4315
|
-
region: config.fileUploads
|
|
4347
|
+
bucket: config.fileUploads?.s3Bucket,
|
|
4348
|
+
region: config.fileUploads?.s3region
|
|
4316
4349
|
});
|
|
4317
4350
|
}, next);
|
|
4318
4351
|
});
|
|
@@ -4331,6 +4364,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4331
4364
|
};
|
|
4332
4365
|
const generateS3Key2 = (filename) => `${(0, import_node_crypto.randomUUID)()}-_EXULU_${filename}`;
|
|
4333
4366
|
const signOnServer = async (req, res, next) => {
|
|
4367
|
+
if (!config.fileUploads) {
|
|
4368
|
+
throw new Error("File uploads are not configured");
|
|
4369
|
+
}
|
|
4334
4370
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4335
4371
|
const { db: db3 } = await postgresClient();
|
|
4336
4372
|
let authtoken = null;
|
|
@@ -4376,6 +4412,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4376
4412
|
return await signOnServer(req, res, next);
|
|
4377
4413
|
});
|
|
4378
4414
|
app.post("/s3/multipart", async (req, res, next) => {
|
|
4415
|
+
if (!config.fileUploads) {
|
|
4416
|
+
throw new Error("File uploads are not configured");
|
|
4417
|
+
}
|
|
4379
4418
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4380
4419
|
const { db: db3 } = await postgresClient();
|
|
4381
4420
|
let authtoken = null;
|
|
@@ -4431,6 +4470,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4431
4470
|
return Number.isInteger(partNumber) && partNumber >= 1 && partNumber <= 1e4;
|
|
4432
4471
|
}
|
|
4433
4472
|
app.get("/s3/multipart/:uploadId/:partNumber", (req, res, next) => {
|
|
4473
|
+
if (!config.fileUploads) {
|
|
4474
|
+
throw new Error("File uploads are not configured");
|
|
4475
|
+
}
|
|
4434
4476
|
const { uploadId, partNumber } = req.params;
|
|
4435
4477
|
const { key } = req.query;
|
|
4436
4478
|
if (!validatePartNumber(partNumber)) {
|
|
@@ -4460,6 +4502,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4460
4502
|
}
|
|
4461
4503
|
const parts = [];
|
|
4462
4504
|
function listPartsPage(startAt) {
|
|
4505
|
+
if (!config.fileUploads) {
|
|
4506
|
+
throw new Error("File uploads are not configured");
|
|
4507
|
+
}
|
|
4463
4508
|
client2.send(new import_client_s3.ListPartsCommand({
|
|
4464
4509
|
Bucket: config.fileUploads.s3Bucket,
|
|
4465
4510
|
Key: key,
|
|
@@ -4484,6 +4529,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4484
4529
|
return part && typeof part === "object" && Number(part.PartNumber) && typeof part.ETag === "string";
|
|
4485
4530
|
}
|
|
4486
4531
|
app.post("/s3/multipart/:uploadId/complete", (req, res, next) => {
|
|
4532
|
+
if (!config.fileUploads) {
|
|
4533
|
+
throw new Error("File uploads are not configured");
|
|
4534
|
+
}
|
|
4487
4535
|
const client2 = getS3Client(config);
|
|
4488
4536
|
const { uploadId } = req.params;
|
|
4489
4537
|
const { key } = req.query;
|
|
@@ -4514,6 +4562,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4514
4562
|
});
|
|
4515
4563
|
});
|
|
4516
4564
|
app.delete("/s3/multipart/:uploadId", (req, res, next) => {
|
|
4565
|
+
if (!config.fileUploads) {
|
|
4566
|
+
throw new Error("File uploads are not configured");
|
|
4567
|
+
}
|
|
4517
4568
|
const client2 = getS3Client(config);
|
|
4518
4569
|
const { uploadId } = req.params;
|
|
4519
4570
|
const { key } = req.query;
|
|
@@ -5781,84 +5832,8 @@ var import_express7 = require("express");
|
|
|
5781
5832
|
|
|
5782
5833
|
// src/registry/routes.ts
|
|
5783
5834
|
var import_express3 = require("express");
|
|
5784
|
-
|
|
5785
|
-
// src/bullmq/queues.ts
|
|
5786
|
-
var import_bullmq4 = require("bullmq");
|
|
5787
|
-
var import_bullmq_otel = require("bullmq-otel");
|
|
5788
|
-
var ExuluQueues = class {
|
|
5789
|
-
queues;
|
|
5790
|
-
constructor() {
|
|
5791
|
-
this.queues = [];
|
|
5792
|
-
}
|
|
5793
|
-
list = /* @__PURE__ */ new Map();
|
|
5794
|
-
// list of queue names
|
|
5795
|
-
queue(name) {
|
|
5796
|
-
return this.queues.find((x) => x.queue?.name === name);
|
|
5797
|
-
}
|
|
5798
|
-
// name: string
|
|
5799
|
-
// concurrency: global concurrency for the queue
|
|
5800
|
-
// ratelimit: maximum number of jobs per second
|
|
5801
|
-
// Rate limit is set on workers (see workers.ts), even global rate limits,
|
|
5802
|
-
// that is a bit counter-intuitive. Since queues are registered using .user
|
|
5803
|
-
// method of ExuluQueues we need to store the desired rate limit on the queue
|
|
5804
|
-
// here so we can use the value when creating workers for the queue instance
|
|
5805
|
-
// as there is no way to store a rate limit value natively on a bullm queue.
|
|
5806
|
-
register = (name, concurrency = 1, ratelimit = 1) => {
|
|
5807
|
-
const use = async () => {
|
|
5808
|
-
const existing = this.queues.find((x) => x.queue?.name === name);
|
|
5809
|
-
if (existing) {
|
|
5810
|
-
const globalConcurrency = await existing.queue.getGlobalConcurrency();
|
|
5811
|
-
if (globalConcurrency !== concurrency) {
|
|
5812
|
-
await existing.queue.setGlobalConcurrency(concurrency);
|
|
5813
|
-
}
|
|
5814
|
-
return {
|
|
5815
|
-
queue: existing.queue,
|
|
5816
|
-
ratelimit,
|
|
5817
|
-
concurrency
|
|
5818
|
-
};
|
|
5819
|
-
}
|
|
5820
|
-
if (!redisServer.host?.length || !redisServer.port?.length) {
|
|
5821
|
-
console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or embedder (look for ExuluQueues.register().use() ).`);
|
|
5822
|
-
throw new Error(`[EXULU] no redis server configured.`);
|
|
5823
|
-
}
|
|
5824
|
-
const newQueue = new import_bullmq4.Queue(
|
|
5825
|
-
`${name}`,
|
|
5826
|
-
{
|
|
5827
|
-
connection: {
|
|
5828
|
-
...redisServer,
|
|
5829
|
-
enableOfflineQueue: false
|
|
5830
|
-
},
|
|
5831
|
-
telemetry: new import_bullmq_otel.BullMQOtel("simple-guide")
|
|
5832
|
-
}
|
|
5833
|
-
);
|
|
5834
|
-
await newQueue.setGlobalConcurrency(concurrency);
|
|
5835
|
-
this.queues.push({
|
|
5836
|
-
queue: newQueue,
|
|
5837
|
-
ratelimit,
|
|
5838
|
-
concurrency
|
|
5839
|
-
});
|
|
5840
|
-
return {
|
|
5841
|
-
queue: newQueue,
|
|
5842
|
-
ratelimit,
|
|
5843
|
-
concurrency
|
|
5844
|
-
};
|
|
5845
|
-
};
|
|
5846
|
-
this.list.set(name, {
|
|
5847
|
-
name,
|
|
5848
|
-
concurrency,
|
|
5849
|
-
ratelimit,
|
|
5850
|
-
use
|
|
5851
|
-
});
|
|
5852
|
-
return {
|
|
5853
|
-
use
|
|
5854
|
-
};
|
|
5855
|
-
};
|
|
5856
|
-
};
|
|
5857
|
-
var queues = new ExuluQueues();
|
|
5858
|
-
|
|
5859
|
-
// src/registry/routes.ts
|
|
5860
5835
|
var import_express4 = __toESM(require("express"), 1);
|
|
5861
|
-
var
|
|
5836
|
+
var import_server2 = require("@apollo/server");
|
|
5862
5837
|
var import_cors = __toESM(require("cors"), 1);
|
|
5863
5838
|
var import_reflect_metadata = require("reflect-metadata");
|
|
5864
5839
|
var import_express5 = require("@as-integrations/express5");
|
|
@@ -5916,7 +5891,7 @@ var {
|
|
|
5916
5891
|
rbacSchema: rbacSchema2,
|
|
5917
5892
|
statisticsSchema: statisticsSchema2
|
|
5918
5893
|
} = coreSchemas.get();
|
|
5919
|
-
var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer,
|
|
5894
|
+
var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer, queues2) => {
|
|
5920
5895
|
var corsOptions = {
|
|
5921
5896
|
origin: "*",
|
|
5922
5897
|
exposedHeaders: "*",
|
|
@@ -5954,8 +5929,8 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tr
|
|
|
5954
5929
|
workflowTemplatesSchema2(),
|
|
5955
5930
|
statisticsSchema2(),
|
|
5956
5931
|
rbacSchema2()
|
|
5957
|
-
], contexts ?? [], agents, tools, config, evals,
|
|
5958
|
-
const server = new
|
|
5932
|
+
], contexts ?? [], agents, tools, config, evals, queues2 || []);
|
|
5933
|
+
const server = new import_server2.ApolloServer({
|
|
5959
5934
|
cache: new import_utils3.InMemoryLRUCache(),
|
|
5960
5935
|
schema,
|
|
5961
5936
|
introspection: true
|
|
@@ -6325,7 +6300,7 @@ Mood: friendly and intelligent.
|
|
|
6325
6300
|
}
|
|
6326
6301
|
});
|
|
6327
6302
|
});
|
|
6328
|
-
if (config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
|
|
6303
|
+
if (config?.fileUploads && config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
|
|
6329
6304
|
await createUppyRoutes(app, config);
|
|
6330
6305
|
} else {
|
|
6331
6306
|
console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in ExuluApp instance.");
|
|
@@ -6501,7 +6476,7 @@ var createCustomAnthropicStreamingMessage = (message) => {
|
|
|
6501
6476
|
|
|
6502
6477
|
// src/registry/workers.ts
|
|
6503
6478
|
var import_ioredis = __toESM(require("ioredis"), 1);
|
|
6504
|
-
var
|
|
6479
|
+
var import_bullmq4 = require("bullmq");
|
|
6505
6480
|
var import_api2 = require("@opentelemetry/api");
|
|
6506
6481
|
var import_uuid3 = require("uuid");
|
|
6507
6482
|
var import_ai3 = require("ai");
|
|
@@ -6518,7 +6493,7 @@ function logMetadata(id, additionalMetadata) {
|
|
|
6518
6493
|
|
|
6519
6494
|
// src/registry/workers.ts
|
|
6520
6495
|
var redisConnection;
|
|
6521
|
-
var createWorkers = async (agents,
|
|
6496
|
+
var createWorkers = async (agents, queues2, config, contexts, evals, tools, tracer) => {
|
|
6522
6497
|
console.log(`
|
|
6523
6498
|
\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557
|
|
6524
6499
|
\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u255A\u2588\u2588\u2557\u2588\u2588\u2554\u255D\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551
|
|
@@ -6529,8 +6504,8 @@ var createWorkers = async (agents, queues3, config, contexts, evals, tools, trac
|
|
|
6529
6504
|
Intelligence Management Platform - Workers
|
|
6530
6505
|
|
|
6531
6506
|
`);
|
|
6532
|
-
console.log("[EXULU] creating workers for " +
|
|
6533
|
-
console.log("[EXULU] queues",
|
|
6507
|
+
console.log("[EXULU] creating workers for " + queues2?.length + " queues.");
|
|
6508
|
+
console.log("[EXULU] queues", queues2.map((q) => q.queue.name));
|
|
6534
6509
|
if (!redisServer.host || !redisServer.port) {
|
|
6535
6510
|
console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
|
|
6536
6511
|
throw new Error("No redis server configured in the environment, so cannot start worker.");
|
|
@@ -6545,9 +6520,9 @@ var createWorkers = async (agents, queues3, config, contexts, evals, tools, trac
|
|
|
6545
6520
|
maxRetriesPerRequest: null
|
|
6546
6521
|
});
|
|
6547
6522
|
}
|
|
6548
|
-
const workers =
|
|
6523
|
+
const workers = queues2.map((queue) => {
|
|
6549
6524
|
console.log(`[EXULU] creating worker for queue ${queue.queue.name}.`);
|
|
6550
|
-
const worker = new
|
|
6525
|
+
const worker = new import_bullmq4.Worker(
|
|
6551
6526
|
`${queue.queue.name}`,
|
|
6552
6527
|
async (bullmqJob) => {
|
|
6553
6528
|
console.log("[EXULU] starting execution for job", logMetadata(bullmqJob.name, {
|
|
@@ -6973,7 +6948,7 @@ var pollJobResult = async ({ queue, jobId }) => {
|
|
|
6973
6948
|
let result;
|
|
6974
6949
|
while (true) {
|
|
6975
6950
|
attempts++;
|
|
6976
|
-
const job = await
|
|
6951
|
+
const job = await import_bullmq4.Job.fromId(queue.queue, jobId);
|
|
6977
6952
|
if (!job) {
|
|
6978
6953
|
await new Promise((resolve) => setTimeout(resolve, 2e3));
|
|
6979
6954
|
continue;
|
|
@@ -7970,7 +7945,7 @@ var ExuluApp = class {
|
|
|
7970
7945
|
};
|
|
7971
7946
|
bullmq = {
|
|
7972
7947
|
workers: {
|
|
7973
|
-
create: async (
|
|
7948
|
+
create: async (queues2) => {
|
|
7974
7949
|
if (!this._config) {
|
|
7975
7950
|
throw new Error("Config not initialized, make sure to call await ExuluApp.create() first when starting your server.");
|
|
7976
7951
|
}
|
|
@@ -7989,8 +7964,8 @@ var ExuluApp = class {
|
|
|
7989
7964
|
console.error = createLogMethod(logger, "error");
|
|
7990
7965
|
console.debug = createLogMethod(logger, "debug");
|
|
7991
7966
|
let filteredQueues = this._queues;
|
|
7992
|
-
if (
|
|
7993
|
-
filteredQueues = filteredQueues.filter((q) =>
|
|
7967
|
+
if (queues2) {
|
|
7968
|
+
filteredQueues = filteredQueues.filter((q) => queues2.includes(q.queue.name));
|
|
7994
7969
|
}
|
|
7995
7970
|
return await createWorkers(
|
|
7996
7971
|
this._agents,
|
|
@@ -8056,6 +8031,80 @@ var ExuluApp = class {
|
|
|
8056
8031
|
};
|
|
8057
8032
|
};
|
|
8058
8033
|
|
|
8034
|
+
// src/bullmq/queues.ts
|
|
8035
|
+
var import_bullmq5 = require("bullmq");
|
|
8036
|
+
var import_bullmq_otel = require("bullmq-otel");
|
|
8037
|
+
var ExuluQueues = class {
|
|
8038
|
+
queues;
|
|
8039
|
+
constructor() {
|
|
8040
|
+
this.queues = [];
|
|
8041
|
+
}
|
|
8042
|
+
list = /* @__PURE__ */ new Map();
|
|
8043
|
+
// list of queue names
|
|
8044
|
+
queue(name) {
|
|
8045
|
+
return this.queues.find((x) => x.queue?.name === name);
|
|
8046
|
+
}
|
|
8047
|
+
// name: string
|
|
8048
|
+
// concurrency: global concurrency for the queue
|
|
8049
|
+
// ratelimit: maximum number of jobs per second
|
|
8050
|
+
// Rate limit is set on workers (see workers.ts), even global rate limits,
|
|
8051
|
+
// that is a bit counter-intuitive. Since queues are registered using .user
|
|
8052
|
+
// method of ExuluQueues we need to store the desired rate limit on the queue
|
|
8053
|
+
// here so we can use the value when creating workers for the queue instance
|
|
8054
|
+
// as there is no way to store a rate limit value natively on a bullm queue.
|
|
8055
|
+
register = (name, concurrency = 1, ratelimit = 1) => {
|
|
8056
|
+
const use = async () => {
|
|
8057
|
+
const existing = this.queues.find((x) => x.queue?.name === name);
|
|
8058
|
+
if (existing) {
|
|
8059
|
+
const globalConcurrency = await existing.queue.getGlobalConcurrency();
|
|
8060
|
+
if (globalConcurrency !== concurrency) {
|
|
8061
|
+
await existing.queue.setGlobalConcurrency(concurrency);
|
|
8062
|
+
}
|
|
8063
|
+
return {
|
|
8064
|
+
queue: existing.queue,
|
|
8065
|
+
ratelimit,
|
|
8066
|
+
concurrency
|
|
8067
|
+
};
|
|
8068
|
+
}
|
|
8069
|
+
if (!redisServer.host?.length || !redisServer.port?.length) {
|
|
8070
|
+
console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or embedder (look for ExuluQueues.register().use() ).`);
|
|
8071
|
+
throw new Error(`[EXULU] no redis server configured.`);
|
|
8072
|
+
}
|
|
8073
|
+
const newQueue = new import_bullmq5.Queue(
|
|
8074
|
+
`${name}`,
|
|
8075
|
+
{
|
|
8076
|
+
connection: {
|
|
8077
|
+
...redisServer,
|
|
8078
|
+
enableOfflineQueue: false
|
|
8079
|
+
},
|
|
8080
|
+
telemetry: new import_bullmq_otel.BullMQOtel("simple-guide")
|
|
8081
|
+
}
|
|
8082
|
+
);
|
|
8083
|
+
await newQueue.setGlobalConcurrency(concurrency);
|
|
8084
|
+
this.queues.push({
|
|
8085
|
+
queue: newQueue,
|
|
8086
|
+
ratelimit,
|
|
8087
|
+
concurrency
|
|
8088
|
+
});
|
|
8089
|
+
return {
|
|
8090
|
+
queue: newQueue,
|
|
8091
|
+
ratelimit,
|
|
8092
|
+
concurrency
|
|
8093
|
+
};
|
|
8094
|
+
};
|
|
8095
|
+
this.list.set(name, {
|
|
8096
|
+
name,
|
|
8097
|
+
concurrency,
|
|
8098
|
+
ratelimit,
|
|
8099
|
+
use
|
|
8100
|
+
});
|
|
8101
|
+
return {
|
|
8102
|
+
use
|
|
8103
|
+
};
|
|
8104
|
+
};
|
|
8105
|
+
};
|
|
8106
|
+
var queues = new ExuluQueues();
|
|
8107
|
+
|
|
8059
8108
|
// src/chunking/types/base.ts
|
|
8060
8109
|
var Chunk = class _Chunk {
|
|
8061
8110
|
/** The text of the chunk. */
|
package/dist/index.d.cts
CHANGED
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -3195,7 +3195,7 @@ var contextToTableDefinition = (context) => {
|
|
|
3195
3195
|
});
|
|
3196
3196
|
return addCoreFields(definition);
|
|
3197
3197
|
};
|
|
3198
|
-
function createSDL(tables, contexts, agents, tools, config, evals,
|
|
3198
|
+
function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
|
|
3199
3199
|
const contextSchemas = contexts.map((context) => contextToTableDefinition(context));
|
|
3200
3200
|
tables.forEach((table) => {
|
|
3201
3201
|
if (!table.fields.some((field) => field.name === "createdAt")) {
|
|
@@ -4020,6 +4020,9 @@ import { randomUUID } from "crypto";
|
|
|
4020
4020
|
var expiresIn = 60 * 60 * 24 * 1;
|
|
4021
4021
|
var s3Client;
|
|
4022
4022
|
function getS3Client(config) {
|
|
4023
|
+
if (!config.fileUploads) {
|
|
4024
|
+
throw new Error("File uploads are not configured");
|
|
4025
|
+
}
|
|
4023
4026
|
s3Client ??= new S3Client({
|
|
4024
4027
|
region: config.fileUploads.s3region,
|
|
4025
4028
|
...config.fileUploads.s3endpoint && {
|
|
@@ -4034,6 +4037,9 @@ function getS3Client(config) {
|
|
|
4034
4037
|
return s3Client;
|
|
4035
4038
|
}
|
|
4036
4039
|
var getPresignedUrl = async (key, config) => {
|
|
4040
|
+
if (!config.fileUploads) {
|
|
4041
|
+
throw new Error("File uploads are not configured");
|
|
4042
|
+
}
|
|
4037
4043
|
const url = await getSignedUrl(
|
|
4038
4044
|
getS3Client(config),
|
|
4039
4045
|
new GetObjectCommand({
|
|
@@ -4045,6 +4051,9 @@ var getPresignedUrl = async (key, config) => {
|
|
|
4045
4051
|
return url;
|
|
4046
4052
|
};
|
|
4047
4053
|
var addPrefixToKey = (keyPath, config) => {
|
|
4054
|
+
if (!config.fileUploads) {
|
|
4055
|
+
throw new Error("File uploads are not configured");
|
|
4056
|
+
}
|
|
4048
4057
|
if (!config.fileUploads.s3prefix) {
|
|
4049
4058
|
return keyPath;
|
|
4050
4059
|
}
|
|
@@ -4056,6 +4065,9 @@ var addPrefixToKey = (keyPath, config) => {
|
|
|
4056
4065
|
};
|
|
4057
4066
|
var uploadFile = async (user, file, key, config, options = {}) => {
|
|
4058
4067
|
console.log("[EXULU] Uploading file to S3", key);
|
|
4068
|
+
if (!config.fileUploads) {
|
|
4069
|
+
throw new Error("File uploads are not configured");
|
|
4070
|
+
}
|
|
4059
4071
|
const client2 = getS3Client(config);
|
|
4060
4072
|
let folder = `${user}/`;
|
|
4061
4073
|
const fullKey = addPrefixToKey(!key.includes(folder) ? folder + key : key, config);
|
|
@@ -4071,7 +4083,13 @@ var uploadFile = async (user, file, key, config, options = {}) => {
|
|
|
4071
4083
|
return key;
|
|
4072
4084
|
};
|
|
4073
4085
|
var createUppyRoutes = async (app, config) => {
|
|
4086
|
+
if (!config.fileUploads) {
|
|
4087
|
+
throw new Error("File uploads are not configured");
|
|
4088
|
+
}
|
|
4074
4089
|
const extractUserPrefix = (key) => {
|
|
4090
|
+
if (!config.fileUploads) {
|
|
4091
|
+
throw new Error("File uploads are not configured");
|
|
4092
|
+
}
|
|
4075
4093
|
if (!config.fileUploads.s3prefix) {
|
|
4076
4094
|
return key.split("/")[0];
|
|
4077
4095
|
}
|
|
@@ -4099,6 +4117,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4099
4117
|
};
|
|
4100
4118
|
let stsClient;
|
|
4101
4119
|
function getSTSClient() {
|
|
4120
|
+
if (!config.fileUploads) {
|
|
4121
|
+
throw new Error("File uploads are not configured");
|
|
4122
|
+
}
|
|
4102
4123
|
stsClient ??= new STSClient({
|
|
4103
4124
|
region: config.fileUploads.s3region,
|
|
4104
4125
|
...config.fileUploads.s3endpoint && { endpoint: config.fileUploads.s3endpoint },
|
|
@@ -4110,6 +4131,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4110
4131
|
return stsClient;
|
|
4111
4132
|
}
|
|
4112
4133
|
app.delete("/s3/delete", async (req, res, next) => {
|
|
4134
|
+
if (!config.fileUploads) {
|
|
4135
|
+
throw new Error("File uploads are not configured");
|
|
4136
|
+
}
|
|
4113
4137
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4114
4138
|
const internalkey = req.headers["internal-key"] || null;
|
|
4115
4139
|
const { db: db3 } = await postgresClient();
|
|
@@ -4200,6 +4224,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4200
4224
|
}
|
|
4201
4225
|
});
|
|
4202
4226
|
app.post("/s3/object", async (req, res, next) => {
|
|
4227
|
+
if (!config.fileUploads) {
|
|
4228
|
+
throw new Error("File uploads are not configured");
|
|
4229
|
+
}
|
|
4203
4230
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4204
4231
|
const internalkey = req.headers["internal-key"] || null;
|
|
4205
4232
|
const { db: db3 } = await postgresClient();
|
|
@@ -4230,6 +4257,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4230
4257
|
res.end();
|
|
4231
4258
|
});
|
|
4232
4259
|
app.get("/s3/list", async (req, res, next) => {
|
|
4260
|
+
if (!config.fileUploads) {
|
|
4261
|
+
throw new Error("File uploads are not configured");
|
|
4262
|
+
}
|
|
4233
4263
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4234
4264
|
const internalkey = req.headers["internal-key"] || null;
|
|
4235
4265
|
const { db: db3 } = await postgresClient();
|
|
@@ -4266,6 +4296,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4266
4296
|
res.end();
|
|
4267
4297
|
});
|
|
4268
4298
|
app.get("/s3/sts", (req, res, next) => {
|
|
4299
|
+
if (!config.fileUploads) {
|
|
4300
|
+
throw new Error("File uploads are not configured");
|
|
4301
|
+
}
|
|
4269
4302
|
getSTSClient().send(new GetFederationTokenCommand({
|
|
4270
4303
|
Name: "Exulu",
|
|
4271
4304
|
// The duration, in seconds, of the role session. The value specified
|
|
@@ -4278,8 +4311,8 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4278
4311
|
res.setHeader("Cache-Control", `public,max-age=${expiresIn}`);
|
|
4279
4312
|
res.json({
|
|
4280
4313
|
credentials: response.Credentials,
|
|
4281
|
-
bucket: config.fileUploads
|
|
4282
|
-
region: config.fileUploads
|
|
4314
|
+
bucket: config.fileUploads?.s3Bucket,
|
|
4315
|
+
region: config.fileUploads?.s3region
|
|
4283
4316
|
});
|
|
4284
4317
|
}, next);
|
|
4285
4318
|
});
|
|
@@ -4298,6 +4331,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4298
4331
|
};
|
|
4299
4332
|
const generateS3Key2 = (filename) => `${randomUUID()}-_EXULU_${filename}`;
|
|
4300
4333
|
const signOnServer = async (req, res, next) => {
|
|
4334
|
+
if (!config.fileUploads) {
|
|
4335
|
+
throw new Error("File uploads are not configured");
|
|
4336
|
+
}
|
|
4301
4337
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4302
4338
|
const { db: db3 } = await postgresClient();
|
|
4303
4339
|
let authtoken = null;
|
|
@@ -4343,6 +4379,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4343
4379
|
return await signOnServer(req, res, next);
|
|
4344
4380
|
});
|
|
4345
4381
|
app.post("/s3/multipart", async (req, res, next) => {
|
|
4382
|
+
if (!config.fileUploads) {
|
|
4383
|
+
throw new Error("File uploads are not configured");
|
|
4384
|
+
}
|
|
4346
4385
|
const apikey = req.headers["exulu-api-key"] || null;
|
|
4347
4386
|
const { db: db3 } = await postgresClient();
|
|
4348
4387
|
let authtoken = null;
|
|
@@ -4398,6 +4437,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4398
4437
|
return Number.isInteger(partNumber) && partNumber >= 1 && partNumber <= 1e4;
|
|
4399
4438
|
}
|
|
4400
4439
|
app.get("/s3/multipart/:uploadId/:partNumber", (req, res, next) => {
|
|
4440
|
+
if (!config.fileUploads) {
|
|
4441
|
+
throw new Error("File uploads are not configured");
|
|
4442
|
+
}
|
|
4401
4443
|
const { uploadId, partNumber } = req.params;
|
|
4402
4444
|
const { key } = req.query;
|
|
4403
4445
|
if (!validatePartNumber(partNumber)) {
|
|
@@ -4427,6 +4469,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4427
4469
|
}
|
|
4428
4470
|
const parts = [];
|
|
4429
4471
|
function listPartsPage(startAt) {
|
|
4472
|
+
if (!config.fileUploads) {
|
|
4473
|
+
throw new Error("File uploads are not configured");
|
|
4474
|
+
}
|
|
4430
4475
|
client2.send(new ListPartsCommand({
|
|
4431
4476
|
Bucket: config.fileUploads.s3Bucket,
|
|
4432
4477
|
Key: key,
|
|
@@ -4451,6 +4496,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4451
4496
|
return part && typeof part === "object" && Number(part.PartNumber) && typeof part.ETag === "string";
|
|
4452
4497
|
}
|
|
4453
4498
|
app.post("/s3/multipart/:uploadId/complete", (req, res, next) => {
|
|
4499
|
+
if (!config.fileUploads) {
|
|
4500
|
+
throw new Error("File uploads are not configured");
|
|
4501
|
+
}
|
|
4454
4502
|
const client2 = getS3Client(config);
|
|
4455
4503
|
const { uploadId } = req.params;
|
|
4456
4504
|
const { key } = req.query;
|
|
@@ -4481,6 +4529,9 @@ var createUppyRoutes = async (app, config) => {
|
|
|
4481
4529
|
});
|
|
4482
4530
|
});
|
|
4483
4531
|
app.delete("/s3/multipart/:uploadId", (req, res, next) => {
|
|
4532
|
+
if (!config.fileUploads) {
|
|
4533
|
+
throw new Error("File uploads are not configured");
|
|
4534
|
+
}
|
|
4484
4535
|
const client2 = getS3Client(config);
|
|
4485
4536
|
const { uploadId } = req.params;
|
|
4486
4537
|
const { key } = req.query;
|
|
@@ -5748,82 +5799,6 @@ import "express";
|
|
|
5748
5799
|
|
|
5749
5800
|
// src/registry/routes.ts
|
|
5750
5801
|
import "express";
|
|
5751
|
-
|
|
5752
|
-
// src/bullmq/queues.ts
|
|
5753
|
-
import { Queue as Queue3 } from "bullmq";
|
|
5754
|
-
import { BullMQOtel } from "bullmq-otel";
|
|
5755
|
-
var ExuluQueues = class {
|
|
5756
|
-
queues;
|
|
5757
|
-
constructor() {
|
|
5758
|
-
this.queues = [];
|
|
5759
|
-
}
|
|
5760
|
-
list = /* @__PURE__ */ new Map();
|
|
5761
|
-
// list of queue names
|
|
5762
|
-
queue(name) {
|
|
5763
|
-
return this.queues.find((x) => x.queue?.name === name);
|
|
5764
|
-
}
|
|
5765
|
-
// name: string
|
|
5766
|
-
// concurrency: global concurrency for the queue
|
|
5767
|
-
// ratelimit: maximum number of jobs per second
|
|
5768
|
-
// Rate limit is set on workers (see workers.ts), even global rate limits,
|
|
5769
|
-
// that is a bit counter-intuitive. Since queues are registered using .user
|
|
5770
|
-
// method of ExuluQueues we need to store the desired rate limit on the queue
|
|
5771
|
-
// here so we can use the value when creating workers for the queue instance
|
|
5772
|
-
// as there is no way to store a rate limit value natively on a bullm queue.
|
|
5773
|
-
register = (name, concurrency = 1, ratelimit = 1) => {
|
|
5774
|
-
const use = async () => {
|
|
5775
|
-
const existing = this.queues.find((x) => x.queue?.name === name);
|
|
5776
|
-
if (existing) {
|
|
5777
|
-
const globalConcurrency = await existing.queue.getGlobalConcurrency();
|
|
5778
|
-
if (globalConcurrency !== concurrency) {
|
|
5779
|
-
await existing.queue.setGlobalConcurrency(concurrency);
|
|
5780
|
-
}
|
|
5781
|
-
return {
|
|
5782
|
-
queue: existing.queue,
|
|
5783
|
-
ratelimit,
|
|
5784
|
-
concurrency
|
|
5785
|
-
};
|
|
5786
|
-
}
|
|
5787
|
-
if (!redisServer.host?.length || !redisServer.port?.length) {
|
|
5788
|
-
console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or embedder (look for ExuluQueues.register().use() ).`);
|
|
5789
|
-
throw new Error(`[EXULU] no redis server configured.`);
|
|
5790
|
-
}
|
|
5791
|
-
const newQueue = new Queue3(
|
|
5792
|
-
`${name}`,
|
|
5793
|
-
{
|
|
5794
|
-
connection: {
|
|
5795
|
-
...redisServer,
|
|
5796
|
-
enableOfflineQueue: false
|
|
5797
|
-
},
|
|
5798
|
-
telemetry: new BullMQOtel("simple-guide")
|
|
5799
|
-
}
|
|
5800
|
-
);
|
|
5801
|
-
await newQueue.setGlobalConcurrency(concurrency);
|
|
5802
|
-
this.queues.push({
|
|
5803
|
-
queue: newQueue,
|
|
5804
|
-
ratelimit,
|
|
5805
|
-
concurrency
|
|
5806
|
-
});
|
|
5807
|
-
return {
|
|
5808
|
-
queue: newQueue,
|
|
5809
|
-
ratelimit,
|
|
5810
|
-
concurrency
|
|
5811
|
-
};
|
|
5812
|
-
};
|
|
5813
|
-
this.list.set(name, {
|
|
5814
|
-
name,
|
|
5815
|
-
concurrency,
|
|
5816
|
-
ratelimit,
|
|
5817
|
-
use
|
|
5818
|
-
});
|
|
5819
|
-
return {
|
|
5820
|
-
use
|
|
5821
|
-
};
|
|
5822
|
-
};
|
|
5823
|
-
};
|
|
5824
|
-
var queues = new ExuluQueues();
|
|
5825
|
-
|
|
5826
|
-
// src/registry/routes.ts
|
|
5827
5802
|
import express from "express";
|
|
5828
5803
|
import { ApolloServer } from "@apollo/server";
|
|
5829
5804
|
import cors from "cors";
|
|
@@ -5883,7 +5858,7 @@ var {
|
|
|
5883
5858
|
rbacSchema: rbacSchema2,
|
|
5884
5859
|
statisticsSchema: statisticsSchema2
|
|
5885
5860
|
} = coreSchemas.get();
|
|
5886
|
-
var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer,
|
|
5861
|
+
var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer, queues2) => {
|
|
5887
5862
|
var corsOptions = {
|
|
5888
5863
|
origin: "*",
|
|
5889
5864
|
exposedHeaders: "*",
|
|
@@ -5921,7 +5896,7 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tr
|
|
|
5921
5896
|
workflowTemplatesSchema2(),
|
|
5922
5897
|
statisticsSchema2(),
|
|
5923
5898
|
rbacSchema2()
|
|
5924
|
-
], contexts ?? [], agents, tools, config, evals,
|
|
5899
|
+
], contexts ?? [], agents, tools, config, evals, queues2 || []);
|
|
5925
5900
|
const server = new ApolloServer({
|
|
5926
5901
|
cache: new InMemoryLRUCache(),
|
|
5927
5902
|
schema,
|
|
@@ -6292,7 +6267,7 @@ Mood: friendly and intelligent.
|
|
|
6292
6267
|
}
|
|
6293
6268
|
});
|
|
6294
6269
|
});
|
|
6295
|
-
if (config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
|
|
6270
|
+
if (config?.fileUploads && config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
|
|
6296
6271
|
await createUppyRoutes(app, config);
|
|
6297
6272
|
} else {
|
|
6298
6273
|
console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in ExuluApp instance.");
|
|
@@ -6485,7 +6460,7 @@ function logMetadata(id, additionalMetadata) {
|
|
|
6485
6460
|
|
|
6486
6461
|
// src/registry/workers.ts
|
|
6487
6462
|
var redisConnection;
|
|
6488
|
-
var createWorkers = async (agents,
|
|
6463
|
+
var createWorkers = async (agents, queues2, config, contexts, evals, tools, tracer) => {
|
|
6489
6464
|
console.log(`
|
|
6490
6465
|
\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557
|
|
6491
6466
|
\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u255A\u2588\u2588\u2557\u2588\u2588\u2554\u255D\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551
|
|
@@ -6496,8 +6471,8 @@ var createWorkers = async (agents, queues3, config, contexts, evals, tools, trac
|
|
|
6496
6471
|
Intelligence Management Platform - Workers
|
|
6497
6472
|
|
|
6498
6473
|
`);
|
|
6499
|
-
console.log("[EXULU] creating workers for " +
|
|
6500
|
-
console.log("[EXULU] queues",
|
|
6474
|
+
console.log("[EXULU] creating workers for " + queues2?.length + " queues.");
|
|
6475
|
+
console.log("[EXULU] queues", queues2.map((q) => q.queue.name));
|
|
6501
6476
|
if (!redisServer.host || !redisServer.port) {
|
|
6502
6477
|
console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
|
|
6503
6478
|
throw new Error("No redis server configured in the environment, so cannot start worker.");
|
|
@@ -6512,7 +6487,7 @@ var createWorkers = async (agents, queues3, config, contexts, evals, tools, trac
|
|
|
6512
6487
|
maxRetriesPerRequest: null
|
|
6513
6488
|
});
|
|
6514
6489
|
}
|
|
6515
|
-
const workers =
|
|
6490
|
+
const workers = queues2.map((queue) => {
|
|
6516
6491
|
console.log(`[EXULU] creating worker for queue ${queue.queue.name}.`);
|
|
6517
6492
|
const worker = new Worker(
|
|
6518
6493
|
`${queue.queue.name}`,
|
|
@@ -7937,7 +7912,7 @@ var ExuluApp = class {
|
|
|
7937
7912
|
};
|
|
7938
7913
|
bullmq = {
|
|
7939
7914
|
workers: {
|
|
7940
|
-
create: async (
|
|
7915
|
+
create: async (queues2) => {
|
|
7941
7916
|
if (!this._config) {
|
|
7942
7917
|
throw new Error("Config not initialized, make sure to call await ExuluApp.create() first when starting your server.");
|
|
7943
7918
|
}
|
|
@@ -7956,8 +7931,8 @@ var ExuluApp = class {
|
|
|
7956
7931
|
console.error = createLogMethod(logger, "error");
|
|
7957
7932
|
console.debug = createLogMethod(logger, "debug");
|
|
7958
7933
|
let filteredQueues = this._queues;
|
|
7959
|
-
if (
|
|
7960
|
-
filteredQueues = filteredQueues.filter((q) =>
|
|
7934
|
+
if (queues2) {
|
|
7935
|
+
filteredQueues = filteredQueues.filter((q) => queues2.includes(q.queue.name));
|
|
7961
7936
|
}
|
|
7962
7937
|
return await createWorkers(
|
|
7963
7938
|
this._agents,
|
|
@@ -8023,6 +7998,80 @@ var ExuluApp = class {
|
|
|
8023
7998
|
};
|
|
8024
7999
|
};
|
|
8025
8000
|
|
|
8001
|
+
// src/bullmq/queues.ts
|
|
8002
|
+
import { Queue as Queue3 } from "bullmq";
|
|
8003
|
+
import { BullMQOtel } from "bullmq-otel";
|
|
8004
|
+
var ExuluQueues = class {
|
|
8005
|
+
queues;
|
|
8006
|
+
constructor() {
|
|
8007
|
+
this.queues = [];
|
|
8008
|
+
}
|
|
8009
|
+
list = /* @__PURE__ */ new Map();
|
|
8010
|
+
// list of queue names
|
|
8011
|
+
queue(name) {
|
|
8012
|
+
return this.queues.find((x) => x.queue?.name === name);
|
|
8013
|
+
}
|
|
8014
|
+
// name: string
|
|
8015
|
+
// concurrency: global concurrency for the queue
|
|
8016
|
+
// ratelimit: maximum number of jobs per second
|
|
8017
|
+
// Rate limit is set on workers (see workers.ts), even global rate limits,
|
|
8018
|
+
// that is a bit counter-intuitive. Since queues are registered using .user
|
|
8019
|
+
// method of ExuluQueues we need to store the desired rate limit on the queue
|
|
8020
|
+
// here so we can use the value when creating workers for the queue instance
|
|
8021
|
+
// as there is no way to store a rate limit value natively on a bullm queue.
|
|
8022
|
+
register = (name, concurrency = 1, ratelimit = 1) => {
|
|
8023
|
+
const use = async () => {
|
|
8024
|
+
const existing = this.queues.find((x) => x.queue?.name === name);
|
|
8025
|
+
if (existing) {
|
|
8026
|
+
const globalConcurrency = await existing.queue.getGlobalConcurrency();
|
|
8027
|
+
if (globalConcurrency !== concurrency) {
|
|
8028
|
+
await existing.queue.setGlobalConcurrency(concurrency);
|
|
8029
|
+
}
|
|
8030
|
+
return {
|
|
8031
|
+
queue: existing.queue,
|
|
8032
|
+
ratelimit,
|
|
8033
|
+
concurrency
|
|
8034
|
+
};
|
|
8035
|
+
}
|
|
8036
|
+
if (!redisServer.host?.length || !redisServer.port?.length) {
|
|
8037
|
+
console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or embedder (look for ExuluQueues.register().use() ).`);
|
|
8038
|
+
throw new Error(`[EXULU] no redis server configured.`);
|
|
8039
|
+
}
|
|
8040
|
+
const newQueue = new Queue3(
|
|
8041
|
+
`${name}`,
|
|
8042
|
+
{
|
|
8043
|
+
connection: {
|
|
8044
|
+
...redisServer,
|
|
8045
|
+
enableOfflineQueue: false
|
|
8046
|
+
},
|
|
8047
|
+
telemetry: new BullMQOtel("simple-guide")
|
|
8048
|
+
}
|
|
8049
|
+
);
|
|
8050
|
+
await newQueue.setGlobalConcurrency(concurrency);
|
|
8051
|
+
this.queues.push({
|
|
8052
|
+
queue: newQueue,
|
|
8053
|
+
ratelimit,
|
|
8054
|
+
concurrency
|
|
8055
|
+
});
|
|
8056
|
+
return {
|
|
8057
|
+
queue: newQueue,
|
|
8058
|
+
ratelimit,
|
|
8059
|
+
concurrency
|
|
8060
|
+
};
|
|
8061
|
+
};
|
|
8062
|
+
this.list.set(name, {
|
|
8063
|
+
name,
|
|
8064
|
+
concurrency,
|
|
8065
|
+
ratelimit,
|
|
8066
|
+
use
|
|
8067
|
+
});
|
|
8068
|
+
return {
|
|
8069
|
+
use
|
|
8070
|
+
};
|
|
8071
|
+
};
|
|
8072
|
+
};
|
|
8073
|
+
var queues = new ExuluQueues();
|
|
8074
|
+
|
|
8026
8075
|
// src/chunking/types/base.ts
|
|
8027
8076
|
var Chunk = class _Chunk {
|
|
8028
8077
|
/** The text of the chunk. */
|