@exulu/backend 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -0
- package/dist/{chunk-ZDH5S2WF.js → chunk-5FTX543Z.js} +879 -489
- package/dist/{convert-exulu-tools-to-ai-sdk-tools-FN6WZSIQ.js → convert-exulu-tools-to-ai-sdk-tools-WQWYMU7G.js} +1 -1
- package/dist/index.cjs +3958 -5159
- package/dist/index.d.cts +321 -389
- package/dist/index.d.ts +321 -389
- package/dist/index.js +1844 -3493
- package/ee/agentic-retrieval/pipeline/index.test.ts +4 -4
- package/ee/agentic-retrieval/pipeline/index.ts +6 -5
- package/ee/agentic-retrieval/pipeline/memory.test.ts +4 -2
- package/ee/agentic-retrieval/pipeline/memory.ts +20 -13
- package/ee/agentic-retrieval/pipeline/search.test.ts +19 -4
- package/ee/agentic-retrieval/pipeline/search.ts +8 -4
- package/ee/agentic-retrieval/pipeline/types.ts +1 -1
- package/ee/python/documents/processing/doc_processor.ts +0 -1
- package/ee/queues/queues.ts +13 -0
- package/ee/schemas.ts +11 -10
- package/ee/workers.ts +42 -56
- package/package.json +1 -1
- package/ee/workers.flow.test.ts +0 -236
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
} from "./chunk-7CCMW3IW.js";
|
|
5
5
|
|
|
6
6
|
// src/templates/tools/convert-exulu-tools-to-ai-sdk-tools.ts
|
|
7
|
-
import { S3Client as
|
|
7
|
+
import { S3Client as S3Client3, PutObjectCommand as PutObjectCommand3, S3ServiceException } from "@aws-sdk/client-s3";
|
|
8
8
|
|
|
9
9
|
// src/exulu/tool.ts
|
|
10
10
|
import { tool } from "ai";
|
|
@@ -33,210 +33,8 @@ var exuluApp = {
|
|
|
33
33
|
};
|
|
34
34
|
|
|
35
35
|
// src/exulu/resolve-model.ts
|
|
36
|
-
import CryptoJS from "crypto-js";
|
|
37
36
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
38
37
|
|
|
39
|
-
// src/postgres/client.ts
|
|
40
|
-
import Knex from "knex";
|
|
41
|
-
import "knex";
|
|
42
|
-
import "pgvector/knex";
|
|
43
|
-
var db = {};
|
|
44
|
-
var databaseExistsChecked = false;
|
|
45
|
-
var getDbName = () => process.env.POSTGRES_DB_NAME || "exulu";
|
|
46
|
-
async function ensureDatabaseExists() {
|
|
47
|
-
const dbName = getDbName();
|
|
48
|
-
const defaultKnex = Knex({
|
|
49
|
-
client: "pg",
|
|
50
|
-
connection: {
|
|
51
|
-
host: process.env.POSTGRES_DB_HOST,
|
|
52
|
-
port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
|
|
53
|
-
user: process.env.POSTGRES_DB_USER,
|
|
54
|
-
database: "postgres",
|
|
55
|
-
// Connect to default database
|
|
56
|
-
password: process.env.POSTGRES_DB_PASSWORD,
|
|
57
|
-
ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false,
|
|
58
|
-
connectionTimeoutMillis: 1e4
|
|
59
|
-
},
|
|
60
|
-
pool: {
|
|
61
|
-
min: 2,
|
|
62
|
-
max: 4,
|
|
63
|
-
acquireTimeoutMillis: 3e4,
|
|
64
|
-
createTimeoutMillis: 3e4,
|
|
65
|
-
idleTimeoutMillis: 3e4,
|
|
66
|
-
reapIntervalMillis: 1e3,
|
|
67
|
-
createRetryIntervalMillis: 200
|
|
68
|
-
}
|
|
69
|
-
});
|
|
70
|
-
try {
|
|
71
|
-
const result = await defaultKnex.raw(`
|
|
72
|
-
SELECT 1 FROM pg_database WHERE datname = '${dbName}'
|
|
73
|
-
`);
|
|
74
|
-
if (result.rows.length === 0) {
|
|
75
|
-
console.log(`[EXULU] Database '${dbName}' does not exist. Creating it...`);
|
|
76
|
-
await defaultKnex.raw(`CREATE DATABASE ${dbName}`);
|
|
77
|
-
console.log(`[EXULU] Database '${dbName}' created successfully.`);
|
|
78
|
-
} else {
|
|
79
|
-
console.log(`[EXULU] Database '${dbName}' already exists.`);
|
|
80
|
-
}
|
|
81
|
-
} catch (error) {
|
|
82
|
-
console.error(
|
|
83
|
-
"[EXULU] Error while checking to ensure the database exists, this could be if the user running the server does not have database admin rights, it is fine to ignore this if you are sure the database exists.",
|
|
84
|
-
error
|
|
85
|
-
);
|
|
86
|
-
return;
|
|
87
|
-
} finally {
|
|
88
|
-
await defaultKnex.destroy();
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
async function postgresClient() {
|
|
92
|
-
if (!db["exulu"]) {
|
|
93
|
-
try {
|
|
94
|
-
if (!databaseExistsChecked) {
|
|
95
|
-
await ensureDatabaseExists();
|
|
96
|
-
databaseExistsChecked = true;
|
|
97
|
-
}
|
|
98
|
-
const knex = Knex({
|
|
99
|
-
client: "pg",
|
|
100
|
-
connection: {
|
|
101
|
-
host: process.env.POSTGRES_DB_HOST,
|
|
102
|
-
port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
|
|
103
|
-
user: process.env.POSTGRES_DB_USER,
|
|
104
|
-
database: getDbName(),
|
|
105
|
-
password: process.env.POSTGRES_DB_PASSWORD,
|
|
106
|
-
ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false,
|
|
107
|
-
// TCP keepalive prevents idle sockets from being silently dropped by
|
|
108
|
-
// intermediate network devices (NAT, firewalls) between us and Hetzner.
|
|
109
|
-
keepAlive: true,
|
|
110
|
-
keepAliveInitialDelayMillis: 1e4,
|
|
111
|
-
connectionTimeoutMillis: 3e4,
|
|
112
|
-
statement_timeout: 18e5,
|
|
113
|
-
query_timeout: 18e5
|
|
114
|
-
},
|
|
115
|
-
pool: {
|
|
116
|
-
min: 10,
|
|
117
|
-
max: 300,
|
|
118
|
-
acquireTimeoutMillis: 12e4,
|
|
119
|
-
createTimeoutMillis: 3e4,
|
|
120
|
-
idleTimeoutMillis: 3e4,
|
|
121
|
-
reapIntervalMillis: 1e3,
|
|
122
|
-
createRetryIntervalMillis: 200,
|
|
123
|
-
// Enable propagateCreateError to properly handle connection creation failures
|
|
124
|
-
propagateCreateError: false,
|
|
125
|
-
// Log pool events to help debug connection issues
|
|
126
|
-
afterCreate: (conn, done) => {
|
|
127
|
-
console.log("[EXULU] New database connection created");
|
|
128
|
-
conn.query("SET statement_timeout = 1800000; SET hnsw.ef_search = 20", (err) => {
|
|
129
|
-
if (err) {
|
|
130
|
-
console.error("[EXULU] Error setting connection parameters:", err);
|
|
131
|
-
}
|
|
132
|
-
done(err, conn);
|
|
133
|
-
});
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
});
|
|
137
|
-
try {
|
|
138
|
-
await knex.schema.createExtensionIfNotExists("vector");
|
|
139
|
-
} catch (error) {
|
|
140
|
-
console.error(
|
|
141
|
-
"[EXULU] Error creating vector extension, this might be fine if you already activated the extension and the 'user' running this script does not have higher level database permissions.",
|
|
142
|
-
error
|
|
143
|
-
);
|
|
144
|
-
}
|
|
145
|
-
db["exulu"] = knex;
|
|
146
|
-
} catch (error) {
|
|
147
|
-
console.error("[EXULU] Error initializing exulu database.", error);
|
|
148
|
-
throw error;
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
return {
|
|
152
|
-
db: db["exulu"]
|
|
153
|
-
};
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
// src/utils/check-record-access.ts
|
|
157
|
-
var checkRecordAccessCache = /* @__PURE__ */ new Map();
|
|
158
|
-
var checkRecordAccess = async (record, request, user) => {
|
|
159
|
-
const setRecordAccessCache = (hasAccess2) => {
|
|
160
|
-
checkRecordAccessCache.set(`${record.id}-${request}-${user?.id}`, {
|
|
161
|
-
hasAccess: hasAccess2,
|
|
162
|
-
expiresAt: new Date(Date.now() + 1e3 * 60 * 1)
|
|
163
|
-
// 1 minute
|
|
164
|
-
});
|
|
165
|
-
};
|
|
166
|
-
const cachedAccess = checkRecordAccessCache.get(`${record.id}-${request}-${user?.id}`);
|
|
167
|
-
if (cachedAccess && cachedAccess.expiresAt > /* @__PURE__ */ new Date()) {
|
|
168
|
-
return cachedAccess.hasAccess;
|
|
169
|
-
}
|
|
170
|
-
const isPublic = record.rights_mode === "public";
|
|
171
|
-
const byUsers = record.rights_mode === "users";
|
|
172
|
-
const byRoles = record.rights_mode === "roles";
|
|
173
|
-
const byTeams = record.rights_mode === "teams";
|
|
174
|
-
const createdBy = typeof record.created_by === "string" ? record.created_by : record.created_by?.toString();
|
|
175
|
-
const isCreator = user ? createdBy === user.id.toString() : false;
|
|
176
|
-
const isAdmin = user ? user.super_admin : false;
|
|
177
|
-
const isApi = user ? user.type === "api" : false;
|
|
178
|
-
const isAdminApi = isApi && (!user.scope_mode || user.scope_mode === "admin");
|
|
179
|
-
const isAgentsScopedApi = isApi && user.scope_mode === "agents" && request === "read" && Array.isArray(user.agent_ids) && user.agent_ids.includes(String(record.id));
|
|
180
|
-
let hasAccess = "none";
|
|
181
|
-
if (isPublic || isCreator || isAdmin || isAdminApi || isAgentsScopedApi) {
|
|
182
|
-
setRecordAccessCache(true);
|
|
183
|
-
return true;
|
|
184
|
-
}
|
|
185
|
-
if (byUsers) {
|
|
186
|
-
if (!user) {
|
|
187
|
-
setRecordAccessCache(false);
|
|
188
|
-
return false;
|
|
189
|
-
}
|
|
190
|
-
hasAccess = record.RBAC?.users?.find((x) => x.id === user.id)?.rights || "none";
|
|
191
|
-
if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
|
|
192
|
-
console.error(
|
|
193
|
-
`[EXULU] Your current user ${user.id} does not have access to this record, current access type is: ${hasAccess}.`
|
|
194
|
-
);
|
|
195
|
-
setRecordAccessCache(false);
|
|
196
|
-
return false;
|
|
197
|
-
} else {
|
|
198
|
-
setRecordAccessCache(true);
|
|
199
|
-
return true;
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
if (byRoles) {
|
|
203
|
-
if (!user) {
|
|
204
|
-
setRecordAccessCache(false);
|
|
205
|
-
return false;
|
|
206
|
-
}
|
|
207
|
-
hasAccess = record.RBAC?.roles?.find((x) => x.id === user.role?.id)?.rights || "none";
|
|
208
|
-
if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
|
|
209
|
-
console.error(
|
|
210
|
-
`[EXULU] Your current role ${user.role?.name} does not have access to this record, current access type is: ${hasAccess}.`
|
|
211
|
-
);
|
|
212
|
-
setRecordAccessCache(false);
|
|
213
|
-
return false;
|
|
214
|
-
} else {
|
|
215
|
-
setRecordAccessCache(true);
|
|
216
|
-
return true;
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
if (byTeams) {
|
|
220
|
-
if (!user) {
|
|
221
|
-
setRecordAccessCache(false);
|
|
222
|
-
return false;
|
|
223
|
-
}
|
|
224
|
-
hasAccess = record.RBAC?.teams?.find((x) => x.id === user.team?.id)?.rights || "none";
|
|
225
|
-
if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
|
|
226
|
-
console.error(
|
|
227
|
-
`[EXULU] Your current team ${user.team?.name} does not have access to this record, current access type is: ${hasAccess}.`
|
|
228
|
-
);
|
|
229
|
-
setRecordAccessCache(false);
|
|
230
|
-
return false;
|
|
231
|
-
} else {
|
|
232
|
-
setRecordAccessCache(true);
|
|
233
|
-
return true;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
setRecordAccessCache(false);
|
|
237
|
-
return false;
|
|
238
|
-
};
|
|
239
|
-
|
|
240
38
|
// src/exulu/litellm/supervisor.ts
|
|
241
39
|
import { spawn } from "child_process";
|
|
242
40
|
import { existsSync } from "fs";
|
|
@@ -616,6 +414,123 @@ function createTaggedFetch(tags) {
|
|
|
616
414
|
return labeled;
|
|
617
415
|
}
|
|
618
416
|
|
|
417
|
+
// src/postgres/client.ts
|
|
418
|
+
import Knex from "knex";
|
|
419
|
+
import "knex";
|
|
420
|
+
import "pgvector/knex";
|
|
421
|
+
var db = {};
|
|
422
|
+
var databaseExistsChecked = false;
|
|
423
|
+
var getDbName = () => process.env.POSTGRES_DB_NAME || "exulu";
|
|
424
|
+
async function ensureDatabaseExists() {
|
|
425
|
+
const dbName = getDbName();
|
|
426
|
+
const defaultKnex = Knex({
|
|
427
|
+
client: "pg",
|
|
428
|
+
connection: {
|
|
429
|
+
host: process.env.POSTGRES_DB_HOST,
|
|
430
|
+
port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
|
|
431
|
+
user: process.env.POSTGRES_DB_USER,
|
|
432
|
+
database: "postgres",
|
|
433
|
+
// Connect to default database
|
|
434
|
+
password: process.env.POSTGRES_DB_PASSWORD,
|
|
435
|
+
ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false,
|
|
436
|
+
connectionTimeoutMillis: 1e4
|
|
437
|
+
},
|
|
438
|
+
pool: {
|
|
439
|
+
min: 2,
|
|
440
|
+
max: 4,
|
|
441
|
+
acquireTimeoutMillis: 3e4,
|
|
442
|
+
createTimeoutMillis: 3e4,
|
|
443
|
+
idleTimeoutMillis: 3e4,
|
|
444
|
+
reapIntervalMillis: 1e3,
|
|
445
|
+
createRetryIntervalMillis: 200
|
|
446
|
+
}
|
|
447
|
+
});
|
|
448
|
+
try {
|
|
449
|
+
const result = await defaultKnex.raw(`
|
|
450
|
+
SELECT 1 FROM pg_database WHERE datname = '${dbName}'
|
|
451
|
+
`);
|
|
452
|
+
if (result.rows.length === 0) {
|
|
453
|
+
console.log(`[EXULU] Database '${dbName}' does not exist. Creating it...`);
|
|
454
|
+
await defaultKnex.raw(`CREATE DATABASE ${dbName}`);
|
|
455
|
+
console.log(`[EXULU] Database '${dbName}' created successfully.`);
|
|
456
|
+
} else {
|
|
457
|
+
console.log(`[EXULU] Database '${dbName}' already exists.`);
|
|
458
|
+
}
|
|
459
|
+
} catch (error) {
|
|
460
|
+
console.error(
|
|
461
|
+
"[EXULU] Error while checking to ensure the database exists, this could be if the user running the server does not have database admin rights, it is fine to ignore this if you are sure the database exists.",
|
|
462
|
+
error
|
|
463
|
+
);
|
|
464
|
+
return;
|
|
465
|
+
} finally {
|
|
466
|
+
await defaultKnex.destroy();
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
async function postgresClient() {
|
|
470
|
+
if (!db["exulu"]) {
|
|
471
|
+
try {
|
|
472
|
+
if (!databaseExistsChecked) {
|
|
473
|
+
await ensureDatabaseExists();
|
|
474
|
+
databaseExistsChecked = true;
|
|
475
|
+
}
|
|
476
|
+
const knex = Knex({
|
|
477
|
+
client: "pg",
|
|
478
|
+
connection: {
|
|
479
|
+
host: process.env.POSTGRES_DB_HOST,
|
|
480
|
+
port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
|
|
481
|
+
user: process.env.POSTGRES_DB_USER,
|
|
482
|
+
database: getDbName(),
|
|
483
|
+
password: process.env.POSTGRES_DB_PASSWORD,
|
|
484
|
+
ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false,
|
|
485
|
+
// TCP keepalive prevents idle sockets from being silently dropped by
|
|
486
|
+
// intermediate network devices (NAT, firewalls) between us and Hetzner.
|
|
487
|
+
keepAlive: true,
|
|
488
|
+
keepAliveInitialDelayMillis: 1e4,
|
|
489
|
+
connectionTimeoutMillis: 3e4,
|
|
490
|
+
statement_timeout: 18e5,
|
|
491
|
+
query_timeout: 18e5
|
|
492
|
+
},
|
|
493
|
+
pool: {
|
|
494
|
+
min: 10,
|
|
495
|
+
max: 300,
|
|
496
|
+
acquireTimeoutMillis: 12e4,
|
|
497
|
+
createTimeoutMillis: 3e4,
|
|
498
|
+
idleTimeoutMillis: 3e4,
|
|
499
|
+
reapIntervalMillis: 1e3,
|
|
500
|
+
createRetryIntervalMillis: 200,
|
|
501
|
+
// Enable propagateCreateError to properly handle connection creation failures
|
|
502
|
+
propagateCreateError: false,
|
|
503
|
+
// Log pool events to help debug connection issues
|
|
504
|
+
afterCreate: (conn, done) => {
|
|
505
|
+
console.log("[EXULU] New database connection created");
|
|
506
|
+
conn.query("SET statement_timeout = 1800000; SET hnsw.ef_search = 20", (err) => {
|
|
507
|
+
if (err) {
|
|
508
|
+
console.error("[EXULU] Error setting connection parameters:", err);
|
|
509
|
+
}
|
|
510
|
+
done(err, conn);
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
});
|
|
515
|
+
try {
|
|
516
|
+
await knex.schema.createExtensionIfNotExists("vector");
|
|
517
|
+
} catch (error) {
|
|
518
|
+
console.error(
|
|
519
|
+
"[EXULU] Error creating vector extension, this might be fine if you already activated the extension and the 'user' running this script does not have higher level database permissions.",
|
|
520
|
+
error
|
|
521
|
+
);
|
|
522
|
+
}
|
|
523
|
+
db["exulu"] = knex;
|
|
524
|
+
} catch (error) {
|
|
525
|
+
console.error("[EXULU] Error initializing exulu database.", error);
|
|
526
|
+
throw error;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
return {
|
|
530
|
+
db: db["exulu"]
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
|
|
619
534
|
// src/exulu/litellm/env.ts
|
|
620
535
|
var LiteLLMAdminError = class extends Error {
|
|
621
536
|
constructor(message, status) {
|
|
@@ -635,9 +550,9 @@ function litellmBase() {
|
|
|
635
550
|
}
|
|
636
551
|
|
|
637
552
|
// src/exulu/litellm/admin-client.ts
|
|
638
|
-
async function call(
|
|
553
|
+
async function call(path3, body) {
|
|
639
554
|
const { url, masterKey } = litellmBase();
|
|
640
|
-
const res = await fetch(`${url}${
|
|
555
|
+
const res = await fetch(`${url}${path3}`, {
|
|
641
556
|
method: "POST",
|
|
642
557
|
headers: {
|
|
643
558
|
Authorization: `Bearer ${masterKey}`,
|
|
@@ -648,7 +563,7 @@ async function call(path, body) {
|
|
|
648
563
|
if (!res.ok) {
|
|
649
564
|
const text = await res.text().catch(() => "");
|
|
650
565
|
throw new LiteLLMAdminError(
|
|
651
|
-
`LiteLLM ${
|
|
566
|
+
`LiteLLM ${path3} returned ${res.status}: ${text}`,
|
|
652
567
|
res.status
|
|
653
568
|
);
|
|
654
569
|
}
|
|
@@ -1112,21 +1027,6 @@ async function getUserBudgetView(userId) {
|
|
|
1112
1027
|
}
|
|
1113
1028
|
|
|
1114
1029
|
// src/exulu/resolve-model.ts
|
|
1115
|
-
var LITELLM_PROVIDER_SENTINEL = new Proxy(
|
|
1116
|
-
{},
|
|
1117
|
-
{
|
|
1118
|
-
get(_target, prop) {
|
|
1119
|
-
if (prop === "id") return "litellm";
|
|
1120
|
-
if (prop === Symbol.toPrimitive || prop === "toString") {
|
|
1121
|
-
return () => "[LiteLLMProviderSentinel]";
|
|
1122
|
-
}
|
|
1123
|
-
console.error(`ExuluProvider.${String(prop)} is not available in LiteLLM mode. `, new Error().stack);
|
|
1124
|
-
throw new Error(
|
|
1125
|
-
`ExuluProvider.${String(prop)} is not available in LiteLLM mode. Code paths that depend on the in-code provider catalog must check isLiteLLMEnabled() and degrade.`
|
|
1126
|
-
);
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
);
|
|
1130
1030
|
var ResolveModelError = class extends Error {
|
|
1131
1031
|
constructor(code, message) {
|
|
1132
1032
|
super(message);
|
|
@@ -1188,101 +1088,43 @@ var getLiteLLMProvider = ({
|
|
|
1188
1088
|
});
|
|
1189
1089
|
};
|
|
1190
1090
|
async function resolveModel(input) {
|
|
1191
|
-
const { modelId, user,
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
try {
|
|
1195
|
-
await waitForLiteLLMReady();
|
|
1196
|
-
} catch (err) {
|
|
1197
|
-
throw new ResolveModelError(
|
|
1198
|
-
"LITELLM_NOT_READY",
|
|
1199
|
-
`LiteLLM is not ready: ${err.message}`
|
|
1200
|
-
);
|
|
1201
|
-
}
|
|
1202
|
-
if (user?.id) await provisionDefaultUserBudget(user.id);
|
|
1203
|
-
const litellm = getLiteLLMProvider({
|
|
1204
|
-
user,
|
|
1205
|
-
role: user?.role,
|
|
1206
|
-
// Fall back to the caller's own project (set on API keys) when no
|
|
1207
|
-
// explicit request project is supplied, so API-triggered requests are
|
|
1208
|
-
// attributed to the key's project.
|
|
1209
|
-
project: project ?? user?.project,
|
|
1210
|
-
agent,
|
|
1211
|
-
team: user?.team,
|
|
1212
|
-
routine
|
|
1213
|
-
});
|
|
1214
|
-
const languageModel2 = litellm(modelId);
|
|
1215
|
-
const syntheticModel = {
|
|
1216
|
-
id: modelId,
|
|
1217
|
-
name: modelId,
|
|
1218
|
-
provider: modelId,
|
|
1219
|
-
active: true,
|
|
1220
|
-
rights_mode: "public",
|
|
1221
|
-
created_by: "litellm"
|
|
1222
|
-
};
|
|
1223
|
-
return {
|
|
1224
|
-
languageModel: languageModel2,
|
|
1225
|
-
model: syntheticModel,
|
|
1226
|
-
exuluProvider: LITELLM_PROVIDER_SENTINEL,
|
|
1227
|
-
apiKey: void 0
|
|
1228
|
-
};
|
|
1229
|
-
}
|
|
1230
|
-
const { db: db2 } = await postgresClient();
|
|
1231
|
-
const model = await db2.from("models").where({ id: modelId }).first();
|
|
1232
|
-
if (!model) {
|
|
1233
|
-
throw new ResolveModelError("MODEL_NOT_FOUND", `Model ${modelId} not found`);
|
|
1234
|
-
}
|
|
1235
|
-
if (!model.active) {
|
|
1236
|
-
throw new ResolveModelError("MODEL_INACTIVE", `Model ${model.name} is inactive`);
|
|
1237
|
-
}
|
|
1238
|
-
if (!rbacBypass) {
|
|
1239
|
-
const ok = await checkRecordAccess(model, rbacRequest, user);
|
|
1240
|
-
if (!ok) {
|
|
1241
|
-
throw new ResolveModelError(
|
|
1242
|
-
"MODEL_FORBIDDEN",
|
|
1243
|
-
`No ${rbacRequest} access to model ${model.name}`
|
|
1244
|
-
);
|
|
1245
|
-
}
|
|
1246
|
-
}
|
|
1247
|
-
const exuluProvider = providers.find((p) => p.id === model.provider);
|
|
1248
|
-
if (!exuluProvider) {
|
|
1249
|
-
throw new ResolveModelError(
|
|
1250
|
-
"PROVIDER_NOT_FOUND",
|
|
1251
|
-
`ExuluProvider ${model.provider} (referenced by model ${model.name}) not registered in this instance`
|
|
1252
|
-
);
|
|
1091
|
+
const { modelId, user, agent, project, routine } = input;
|
|
1092
|
+
if (!isLiteLLMEnabled()) {
|
|
1093
|
+
throw new Error("Litellm not configured or available.");
|
|
1253
1094
|
}
|
|
1254
|
-
|
|
1095
|
+
try {
|
|
1096
|
+
await waitForLiteLLMReady();
|
|
1097
|
+
} catch (err) {
|
|
1255
1098
|
throw new ResolveModelError(
|
|
1256
|
-
"
|
|
1257
|
-
`
|
|
1099
|
+
"LITELLM_NOT_READY",
|
|
1100
|
+
`LiteLLM is not ready: ${err.message}`
|
|
1258
1101
|
);
|
|
1259
1102
|
}
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
"AUTH_VAR_NOT_ENCRYPTED",
|
|
1272
|
-
`Auth variable ${model.authvariable} must be encrypted`
|
|
1273
|
-
);
|
|
1274
|
-
}
|
|
1275
|
-
const bytes = CryptoJS.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
1276
|
-
apiKey = bytes.toString(CryptoJS.enc.Utf8);
|
|
1277
|
-
}
|
|
1278
|
-
const languageModel = exuluProvider.config.model.create({
|
|
1279
|
-
apiKey,
|
|
1280
|
-
user: user?.id,
|
|
1281
|
-
role: user?.role?.id,
|
|
1282
|
-
project: project?.id,
|
|
1283
|
-
agent: agent?.id
|
|
1103
|
+
if (user?.id) await provisionDefaultUserBudget(user.id);
|
|
1104
|
+
const litellm = getLiteLLMProvider({
|
|
1105
|
+
user,
|
|
1106
|
+
role: user?.role,
|
|
1107
|
+
// Fall back to the caller's own project (set on API keys) when no
|
|
1108
|
+
// explicit request project is supplied, so API-triggered requests are
|
|
1109
|
+
// attributed to the key's project.
|
|
1110
|
+
project: project ?? user?.project,
|
|
1111
|
+
agent,
|
|
1112
|
+
team: user?.team,
|
|
1113
|
+
routine
|
|
1284
1114
|
});
|
|
1285
|
-
|
|
1115
|
+
const languageModel = litellm(modelId);
|
|
1116
|
+
const syntheticModel = {
|
|
1117
|
+
id: modelId,
|
|
1118
|
+
name: modelId,
|
|
1119
|
+
provider: modelId,
|
|
1120
|
+
active: true,
|
|
1121
|
+
rights_mode: "public",
|
|
1122
|
+
created_by: "litellm"
|
|
1123
|
+
};
|
|
1124
|
+
return {
|
|
1125
|
+
languageModel,
|
|
1126
|
+
model: syntheticModel
|
|
1127
|
+
};
|
|
1286
1128
|
}
|
|
1287
1129
|
|
|
1288
1130
|
// src/exulu/auth/validate.ts
|
|
@@ -1431,14 +1273,14 @@ var authRegistry = {
|
|
|
1431
1273
|
};
|
|
1432
1274
|
|
|
1433
1275
|
// src/exulu/auth/flow.ts
|
|
1434
|
-
import
|
|
1276
|
+
import CryptoJS2 from "crypto-js";
|
|
1435
1277
|
import { createHash, randomBytes } from "crypto";
|
|
1436
1278
|
|
|
1437
1279
|
// src/exulu/auth/credential-store.ts
|
|
1438
|
-
import
|
|
1280
|
+
import CryptoJS from "crypto-js";
|
|
1439
1281
|
var TABLE = "user_credentials";
|
|
1440
|
-
var encrypt = (value) =>
|
|
1441
|
-
var decrypt = (value) =>
|
|
1282
|
+
var encrypt = (value) => CryptoJS.AES.encrypt(value, process.env.NEXTAUTH_SECRET).toString();
|
|
1283
|
+
var decrypt = (value) => CryptoJS.AES.decrypt(value, process.env.NEXTAUTH_SECRET).toString(CryptoJS.enc.Utf8);
|
|
1442
1284
|
async function get(provider, userId) {
|
|
1443
1285
|
const { db: db2 } = await postgresClient();
|
|
1444
1286
|
const row = await db2.from(TABLE).where({ provider, user_id: String(userId) }).first();
|
|
@@ -1517,12 +1359,12 @@ var fromBase64Url = (value) => {
|
|
|
1517
1359
|
}
|
|
1518
1360
|
return base64;
|
|
1519
1361
|
};
|
|
1520
|
-
var encryptOauthState = (state) => toBase64Url(
|
|
1362
|
+
var encryptOauthState = (state) => toBase64Url(CryptoJS2.AES.encrypt(JSON.stringify(state), process.env.NEXTAUTH_SECRET).toString());
|
|
1521
1363
|
var decryptOauthState = (value) => {
|
|
1522
1364
|
let json = "";
|
|
1523
1365
|
try {
|
|
1524
|
-
json =
|
|
1525
|
-
|
|
1366
|
+
json = CryptoJS2.AES.decrypt(fromBase64Url(value), process.env.NEXTAUTH_SECRET).toString(
|
|
1367
|
+
CryptoJS2.enc.Utf8
|
|
1526
1368
|
);
|
|
1527
1369
|
} catch {
|
|
1528
1370
|
throw new Error("[EXULU] Invalid OAuth state.");
|
|
@@ -1891,26 +1733,13 @@ var ExuluTool = class _ExuluTool {
|
|
|
1891
1733
|
if (!agent) {
|
|
1892
1734
|
throw new Error("Agent not found.");
|
|
1893
1735
|
}
|
|
1894
|
-
|
|
1895
|
-
if (agent.model) {
|
|
1896
|
-
const providers = exuluApp.get().providers;
|
|
1897
|
-
const resolved = await resolveModel({
|
|
1898
|
-
modelId: agent.model,
|
|
1899
|
-
user,
|
|
1900
|
-
providers,
|
|
1901
|
-
agent,
|
|
1902
|
-
rbacBypass: true
|
|
1903
|
-
});
|
|
1904
|
-
providerapikey = resolved.apiKey;
|
|
1905
|
-
}
|
|
1906
|
-
const { convertExuluToolsToAiSdkTools: convertExuluToolsToAiSdkTools2 } = await import("./convert-exulu-tools-to-ai-sdk-tools-FN6WZSIQ.js");
|
|
1736
|
+
const { convertExuluToolsToAiSdkTools: convertExuluToolsToAiSdkTools2 } = await import("./convert-exulu-tools-to-ai-sdk-tools-WQWYMU7G.js");
|
|
1907
1737
|
const tools = await convertExuluToolsToAiSdkTools2(
|
|
1908
1738
|
[this],
|
|
1909
1739
|
[],
|
|
1910
1740
|
[],
|
|
1911
1741
|
[],
|
|
1912
1742
|
agent.tools,
|
|
1913
|
-
providerapikey,
|
|
1914
1743
|
void 0,
|
|
1915
1744
|
user,
|
|
1916
1745
|
config,
|
|
@@ -1984,7 +1813,7 @@ var updateStatistic = async (statistic) => {
|
|
|
1984
1813
|
};
|
|
1985
1814
|
|
|
1986
1815
|
// src/templates/tools/convert-exulu-tools-to-ai-sdk-tools.ts
|
|
1987
|
-
import
|
|
1816
|
+
import CryptoJS4 from "crypto-js";
|
|
1988
1817
|
|
|
1989
1818
|
// src/templates/tools/session-items-retrieval-tool.ts
|
|
1990
1819
|
import { z as z2 } from "zod";
|
|
@@ -3179,7 +3008,7 @@ function neutralResult(question, keywords, importantKeyword, steps = []) {
|
|
|
3179
3008
|
return {
|
|
3180
3009
|
memoryChunksForAnswer: [],
|
|
3181
3010
|
memoryOverride: { active: false, chunks: [], reason: "" },
|
|
3182
|
-
|
|
3011
|
+
memoryPinnedItemIdsByContext: /* @__PURE__ */ new Map(),
|
|
3183
3012
|
updatedQuestion: question,
|
|
3184
3013
|
updatedKeywords: keywords,
|
|
3185
3014
|
updatedImportantKeyword: importantKeyword,
|
|
@@ -3280,7 +3109,7 @@ async function runMemoryPhase({
|
|
|
3280
3109
|
chunks: [],
|
|
3281
3110
|
reason: ""
|
|
3282
3111
|
};
|
|
3283
|
-
|
|
3112
|
+
const memoryPinnedItemIdsByContext = /* @__PURE__ */ new Map();
|
|
3284
3113
|
let updatedQuestion = question;
|
|
3285
3114
|
let updatedKeywords = keywords;
|
|
3286
3115
|
let updatedImportantKeyword = importantKeyword;
|
|
@@ -3443,25 +3272,30 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
|
|
|
3443
3272
|
if (fileResult.output?.shouldPrioritizeFiles && fileResult.output?.fileNameHints?.length) {
|
|
3444
3273
|
const hints = fileResult.output.fileNameHints;
|
|
3445
3274
|
const pinResults = await Promise.all(
|
|
3446
|
-
documentContexts.map(
|
|
3447
|
-
|
|
3275
|
+
documentContexts.map(async (ctx) => ({
|
|
3276
|
+
ctxId: ctx.id,
|
|
3277
|
+
matches: await fuzzyPrefilter({
|
|
3448
3278
|
cacheKey: `memory-pin:${ctx.id}`,
|
|
3449
3279
|
relevantKeywords: hints,
|
|
3450
3280
|
context: ctx,
|
|
3451
3281
|
fields: ["name", "id", "external_id"],
|
|
3452
3282
|
normalize: (item) => item.external_id ? normalizeFileName(item.external_id) : item.name
|
|
3453
3283
|
}).catch(() => [])
|
|
3454
|
-
)
|
|
3284
|
+
}))
|
|
3455
3285
|
);
|
|
3456
|
-
|
|
3457
|
-
|
|
3458
|
-
|
|
3286
|
+
const pinnedNames = [];
|
|
3287
|
+
for (const { ctxId, matches } of pinResults) {
|
|
3288
|
+
if (!matches.length) continue;
|
|
3289
|
+
const set = memoryPinnedItemIdsByContext.get(ctxId) ?? /* @__PURE__ */ new Set();
|
|
3290
|
+
for (const m of matches) {
|
|
3291
|
+
set.add(m.id);
|
|
3292
|
+
pinnedNames.push(m.name);
|
|
3459
3293
|
}
|
|
3294
|
+
memoryPinnedItemIdsByContext.set(ctxId, set);
|
|
3460
3295
|
}
|
|
3461
|
-
if (
|
|
3462
|
-
const names = pinResults.flat().map((i) => i.name).join(", ");
|
|
3296
|
+
if (pinnedNames.length > 0) {
|
|
3463
3297
|
steps.push({
|
|
3464
|
-
text: `Memory prioritizes specific document(s); pinning ${
|
|
3298
|
+
text: `Memory prioritizes specific document(s); pinning ${pinnedNames.length} file(s) into the search: ${pinnedNames.join(", ")}`
|
|
3465
3299
|
});
|
|
3466
3300
|
}
|
|
3467
3301
|
}
|
|
@@ -3484,7 +3318,7 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
|
|
|
3484
3318
|
return {
|
|
3485
3319
|
memoryChunksForAnswer,
|
|
3486
3320
|
memoryOverride,
|
|
3487
|
-
|
|
3321
|
+
memoryPinnedItemIdsByContext,
|
|
3488
3322
|
updatedQuestion,
|
|
3489
3323
|
updatedKeywords,
|
|
3490
3324
|
updatedImportantKeyword,
|
|
@@ -3591,7 +3425,7 @@ async function searchContexts(opts) {
|
|
|
3591
3425
|
preselectedItems,
|
|
3592
3426
|
scopedItemsByContext,
|
|
3593
3427
|
identifierPinsByContext,
|
|
3594
|
-
|
|
3428
|
+
memoryPinnedItemIdsByContext,
|
|
3595
3429
|
userPinnedItemIdsByContext,
|
|
3596
3430
|
rewrites,
|
|
3597
3431
|
styleHint,
|
|
@@ -3622,7 +3456,8 @@ async function searchContexts(opts) {
|
|
|
3622
3456
|
const identifierPins = identifierPinsByContext.get(ctxId) ?? /* @__PURE__ */ new Set();
|
|
3623
3457
|
let pins = new Set(identifierPins);
|
|
3624
3458
|
if (kind === "documents") {
|
|
3625
|
-
|
|
3459
|
+
const memPins = memoryPinnedItemIdsByContext.get(ctxId);
|
|
3460
|
+
if (memPins) for (const id of memPins) pins.add(id);
|
|
3626
3461
|
}
|
|
3627
3462
|
const userPins = userPinnedItemIdsByContext.get(ctxId);
|
|
3628
3463
|
if (userPins && userPins.size > 0) {
|
|
@@ -4017,7 +3852,6 @@ function createAgenticRetrievalTool(opts) {
|
|
|
4017
3852
|
const resolved = await resolveModel({
|
|
4018
3853
|
modelId: cfg.utilityModel,
|
|
4019
3854
|
user,
|
|
4020
|
-
providers: exuluApp.get().providers,
|
|
4021
3855
|
rbacBypass: true
|
|
4022
3856
|
});
|
|
4023
3857
|
utilityModel = resolved.languageModel ?? model;
|
|
@@ -4135,7 +3969,7 @@ ${projectScope.customInstructions}` : ""
|
|
|
4135
3969
|
updatedQuestion,
|
|
4136
3970
|
updatedKeywords,
|
|
4137
3971
|
updatedImportantKeyword,
|
|
4138
|
-
|
|
3972
|
+
memoryPinnedItemIdsByContext,
|
|
4139
3973
|
memoryOverride
|
|
4140
3974
|
} = memResult;
|
|
4141
3975
|
const { pinsByContext: identifierPinsByContext, exactPinsByContext, steps: pinSteps } = await resolveIdentifierPins({
|
|
@@ -4168,7 +4002,7 @@ ${projectScope.customInstructions}` : ""
|
|
|
4168
4002
|
model: utilityModel,
|
|
4169
4003
|
preselectedItems,
|
|
4170
4004
|
identifierPinsByContext,
|
|
4171
|
-
|
|
4005
|
+
memoryPinnedItemIdsByContext,
|
|
4172
4006
|
userPinnedItemIdsByContext,
|
|
4173
4007
|
scopedItemsByContext: resolvedProject?.scopedItemsByContext,
|
|
4174
4008
|
rewrites: cfg.vocabulary.rewrites,
|
|
@@ -4188,7 +4022,7 @@ ${projectScope.customInstructions}` : ""
|
|
|
4188
4022
|
model: utilityModel,
|
|
4189
4023
|
preselectedItems,
|
|
4190
4024
|
identifierPinsByContext,
|
|
4191
|
-
|
|
4025
|
+
memoryPinnedItemIdsByContext,
|
|
4192
4026
|
userPinnedItemIdsByContext,
|
|
4193
4027
|
scopedItemsByContext: resolvedProject?.scopedItemsByContext,
|
|
4194
4028
|
rewrites: cfg.vocabulary.rewrites,
|
|
@@ -4198,7 +4032,9 @@ ${projectScope.customInstructions}` : ""
|
|
|
4198
4032
|
}) : Promise.resolve({ chunks: [] })
|
|
4199
4033
|
]);
|
|
4200
4034
|
const pinnedItemIds = /* @__PURE__ */ new Set([
|
|
4201
|
-
...
|
|
4035
|
+
...(function* () {
|
|
4036
|
+
for (const s of memoryPinnedItemIdsByContext.values()) yield* s;
|
|
4037
|
+
})(),
|
|
4202
4038
|
...(function* () {
|
|
4203
4039
|
for (const s of exactPinsByContext.values()) yield* s;
|
|
4204
4040
|
})(),
|
|
@@ -4368,7 +4204,7 @@ function sanitizeToolName(name) {
|
|
|
4368
4204
|
}
|
|
4369
4205
|
|
|
4370
4206
|
// src/templates/tools/convert-exulu-tools-to-ai-sdk-tools.ts
|
|
4371
|
-
import { randomUUID as
|
|
4207
|
+
import { randomUUID as randomUUID5 } from "crypto";
|
|
4372
4208
|
|
|
4373
4209
|
// types/enums/statistics.ts
|
|
4374
4210
|
var STATISTICS_TYPE_ENUM = {
|
|
@@ -6029,7 +5865,7 @@ ${body}`
|
|
|
6029
5865
|
import { createBashTool } from "bash-tool";
|
|
6030
5866
|
import { tool as tool2 } from "ai";
|
|
6031
5867
|
import { z as z11 } from "zod";
|
|
6032
|
-
import
|
|
5868
|
+
import CryptoJS3 from "crypto-js";
|
|
6033
5869
|
var getAllExuluVariables = async () => {
|
|
6034
5870
|
const { db: db2 } = await postgresClient();
|
|
6035
5871
|
const rows = await db2.from("variables").select("*");
|
|
@@ -6041,8 +5877,8 @@ var getAllExuluVariables = async () => {
|
|
|
6041
5877
|
let value = row.value;
|
|
6042
5878
|
if (row.encrypted) {
|
|
6043
5879
|
try {
|
|
6044
|
-
const bytes =
|
|
6045
|
-
value = bytes.toString(
|
|
5880
|
+
const bytes = CryptoJS3.AES.decrypt(value, process.env.NEXTAUTH_SECRET);
|
|
5881
|
+
value = bytes.toString(CryptoJS3.enc.Utf8);
|
|
6046
5882
|
} catch (err) {
|
|
6047
5883
|
console.error(
|
|
6048
5884
|
`[VARIABLES] Failed to decrypt variable "${row.name}"; skipping.`,
|
|
@@ -6326,11 +6162,11 @@ Probe error: ${probe.reason ?? "(no detail)"}`
|
|
|
6326
6162
|
async executeCommand(command) {
|
|
6327
6163
|
return await runWrapped(command);
|
|
6328
6164
|
},
|
|
6329
|
-
async readFile(
|
|
6330
|
-
const { stdout, stderr, exitCode } = await runWrapped(`cat ${shellQuote(
|
|
6165
|
+
async readFile(path3) {
|
|
6166
|
+
const { stdout, stderr, exitCode } = await runWrapped(`cat ${shellQuote(path3)}`);
|
|
6331
6167
|
if (exitCode !== 0) {
|
|
6332
6168
|
throw new Error(
|
|
6333
|
-
`readFile ${
|
|
6169
|
+
`readFile ${path3} failed (exit ${exitCode}): ${stderr.trim() || "no stderr captured"}`
|
|
6334
6170
|
);
|
|
6335
6171
|
}
|
|
6336
6172
|
return stdout;
|
|
@@ -6450,8 +6286,8 @@ Probe error: ${probe.reason ?? "(no detail)"}`
|
|
|
6450
6286
|
path: z11.string().describe("The path where the file should be written. Relative paths and leading-slash paths are both resolved against the session sandbox root."),
|
|
6451
6287
|
content: z11.string().describe("The content to write to the file")
|
|
6452
6288
|
}),
|
|
6453
|
-
execute: async ({ path, content }) => {
|
|
6454
|
-
const resolvedPath = resolveSessionPath(
|
|
6289
|
+
execute: async ({ path: path3, content }) => {
|
|
6290
|
+
const resolvedPath = resolveSessionPath(path3, sessionDir);
|
|
6455
6291
|
const results = await writeFilesInternal([{ path: resolvedPath, content }]);
|
|
6456
6292
|
const result = results[0];
|
|
6457
6293
|
if (!result) {
|
|
@@ -6470,8 +6306,8 @@ Probe error: ${probe.reason ?? "(no detail)"}`
|
|
|
6470
6306
|
inputSchema: z11.object({
|
|
6471
6307
|
path: z11.string().describe("The path of the file to read. Relative paths and leading-slash paths are both resolved against the session sandbox root.")
|
|
6472
6308
|
}),
|
|
6473
|
-
execute: async ({ path }) => {
|
|
6474
|
-
const resolvedPath = resolveSessionPath(
|
|
6309
|
+
execute: async ({ path: path3 }) => {
|
|
6310
|
+
const resolvedPath = resolveSessionPath(path3, sessionDir);
|
|
6475
6311
|
const content = await customSandbox.readFile(resolvedPath);
|
|
6476
6312
|
return { content };
|
|
6477
6313
|
}
|
|
@@ -6493,25 +6329,25 @@ Probe error: ${probe.reason ?? "(no detail)"}`
|
|
|
6493
6329
|
if (persistenceEnabled && before) {
|
|
6494
6330
|
const after = await snapshotSessionArtifacts();
|
|
6495
6331
|
const changedPaths = [];
|
|
6496
|
-
for (const [
|
|
6497
|
-
const beforeMtime = before.get(
|
|
6332
|
+
for (const [path3, mtime] of after) {
|
|
6333
|
+
const beforeMtime = before.get(path3);
|
|
6498
6334
|
if (beforeMtime === void 0 || beforeMtime < mtime) {
|
|
6499
|
-
changedPaths.push(
|
|
6335
|
+
changedPaths.push(path3);
|
|
6500
6336
|
}
|
|
6501
6337
|
}
|
|
6502
|
-
for (const
|
|
6338
|
+
for (const path3 of changedPaths) {
|
|
6503
6339
|
try {
|
|
6504
|
-
const content = await fsReadFile(
|
|
6505
|
-
const persisted = await persistArtifactToS3(
|
|
6340
|
+
const content = await fsReadFile(path3);
|
|
6341
|
+
const persisted = await persistArtifactToS3(path3, content);
|
|
6506
6342
|
artifacts.push({
|
|
6507
|
-
path,
|
|
6508
|
-
relativePath: relative(sessionDir,
|
|
6343
|
+
path: path3,
|
|
6344
|
+
relativePath: relative(sessionDir, path3),
|
|
6509
6345
|
key: persisted.key,
|
|
6510
6346
|
url: persisted.url
|
|
6511
6347
|
});
|
|
6512
6348
|
} catch (err) {
|
|
6513
6349
|
console.error(
|
|
6514
|
-
`[SKILLS] Failed to mirror bash-produced artifact ${
|
|
6350
|
+
`[SKILLS] Failed to mirror bash-produced artifact ${path3} to S3; continuing.`,
|
|
6515
6351
|
err
|
|
6516
6352
|
);
|
|
6517
6353
|
}
|
|
@@ -7277,9 +7113,523 @@ var createViewDocumentPageTool = ({
|
|
|
7277
7113
|
});
|
|
7278
7114
|
};
|
|
7279
7115
|
|
|
7116
|
+
// src/exulu/audit/config.ts
|
|
7117
|
+
import os from "os";
|
|
7118
|
+
import path from "path";
|
|
7119
|
+
var normalizePrefix = (p) => {
|
|
7120
|
+
const raw = (p ?? "audit").trim().replace(/^\/+|\/+$/g, "");
|
|
7121
|
+
return `${raw || "audit"}/`;
|
|
7122
|
+
};
|
|
7123
|
+
var hasAllS3Fields = (t) => !!t && !!t.s3region && !!t.s3key && !!t.s3secret && !!t.s3Bucket;
|
|
7124
|
+
var resolveAuditConfig = (config) => {
|
|
7125
|
+
const a = config.audit;
|
|
7126
|
+
if (!a || a.enabled !== true) return null;
|
|
7127
|
+
const dedicated = hasAllS3Fields(a.s3);
|
|
7128
|
+
const source = dedicated ? a.s3 : config.fileUploads;
|
|
7129
|
+
if (!hasAllS3Fields(source)) {
|
|
7130
|
+
throw new Error(
|
|
7131
|
+
"[EXULU] audit.enabled is true but no S3 target is configured. Set config.audit.s3 or config.fileUploads."
|
|
7132
|
+
);
|
|
7133
|
+
}
|
|
7134
|
+
if (!Number.isInteger(a.retentionDays) || a.retentionDays <= 0) {
|
|
7135
|
+
throw new Error(`[EXULU] audit.retentionDays must be a positive integer, got ${a.retentionDays}.`);
|
|
7136
|
+
}
|
|
7137
|
+
const usingSharedFileUploadsBucket = !dedicated;
|
|
7138
|
+
return {
|
|
7139
|
+
target: {
|
|
7140
|
+
s3region: source.s3region,
|
|
7141
|
+
s3key: source.s3key,
|
|
7142
|
+
s3secret: source.s3secret,
|
|
7143
|
+
s3Bucket: source.s3Bucket,
|
|
7144
|
+
s3prefix: normalizePrefix(source.s3prefix),
|
|
7145
|
+
...source.s3endpoint ? { s3endpoint: source.s3endpoint } : {}
|
|
7146
|
+
},
|
|
7147
|
+
retentionDays: a.retentionDays,
|
|
7148
|
+
manageLifecycle: a.manageLifecycle ?? !usingSharedFileUploadsBucket,
|
|
7149
|
+
usingSharedFileUploadsBucket,
|
|
7150
|
+
spoolDir: a.spoolDir ?? path.join(os.tmpdir(), "exulu-audit-spool"),
|
|
7151
|
+
flush: {
|
|
7152
|
+
maxRecords: a.flush?.maxRecords ?? 100,
|
|
7153
|
+
maxIntervalMs: a.flush?.maxIntervalMs ?? 5e3
|
|
7154
|
+
},
|
|
7155
|
+
payload: {
|
|
7156
|
+
maxBytes: a.payload?.maxBytes ?? 32768,
|
|
7157
|
+
captureOutput: a.payload?.captureOutput ?? true,
|
|
7158
|
+
redactKeys: a.payload?.redactKeys ?? []
|
|
7159
|
+
},
|
|
7160
|
+
failureMode: a.failureMode ?? "open",
|
|
7161
|
+
toolCalls: {
|
|
7162
|
+
enabled: a.sources?.toolCalls?.enabled ?? true,
|
|
7163
|
+
include: a.sources?.toolCalls?.include ?? [],
|
|
7164
|
+
exclude: a.sources?.toolCalls?.exclude ?? []
|
|
7165
|
+
}
|
|
7166
|
+
};
|
|
7167
|
+
};
|
|
7168
|
+
|
|
7169
|
+
// src/exulu/audit/s3-writer.ts
|
|
7170
|
+
import {
|
|
7171
|
+
S3Client as S3Client2,
|
|
7172
|
+
PutObjectCommand as PutObjectCommand2,
|
|
7173
|
+
GetBucketLifecycleConfigurationCommand,
|
|
7174
|
+
PutBucketLifecycleConfigurationCommand
|
|
7175
|
+
} from "@aws-sdk/client-s3";
|
|
7176
|
+
var RETRYABLE = /* @__PURE__ */ new Set(["SignatureDoesNotMatch", "InvalidAccessKeyId", "AccessDenied"]);
|
|
7177
|
+
var buildAuditS3Client = (t) => new S3Client2({
|
|
7178
|
+
region: t.s3region,
|
|
7179
|
+
...t.s3endpoint ? { forcePathStyle: true, endpoint: t.s3endpoint } : {},
|
|
7180
|
+
credentials: { accessKeyId: t.s3key, secretAccessKey: t.s3secret },
|
|
7181
|
+
requestChecksumCalculation: "WHEN_REQUIRED",
|
|
7182
|
+
responseChecksumValidation: "WHEN_REQUIRED"
|
|
7183
|
+
});
|
|
7184
|
+
var createAuditS3Writer = (target, client, opts) => {
|
|
7185
|
+
const c = client ?? buildAuditS3Client(target);
|
|
7186
|
+
const maxRetries = opts?.maxRetries ?? 3;
|
|
7187
|
+
const backoffMs = opts?.backoffMs ?? ((attempt) => Math.pow(2, attempt) * 1e3);
|
|
7188
|
+
const putNdjson = async (key, body) => {
|
|
7189
|
+
let lastError = null;
|
|
7190
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
7191
|
+
const command = new PutObjectCommand2({
|
|
7192
|
+
Bucket: target.s3Bucket,
|
|
7193
|
+
Key: key,
|
|
7194
|
+
Body: Buffer.from(body, "utf8"),
|
|
7195
|
+
ContentType: "application/x-ndjson"
|
|
7196
|
+
});
|
|
7197
|
+
try {
|
|
7198
|
+
await c.send(command);
|
|
7199
|
+
return;
|
|
7200
|
+
} catch (error) {
|
|
7201
|
+
lastError = error;
|
|
7202
|
+
if (RETRYABLE.has(error?.name) && attempt < maxRetries) {
|
|
7203
|
+
await new Promise((r) => setTimeout(r, backoffMs(attempt)));
|
|
7204
|
+
continue;
|
|
7205
|
+
}
|
|
7206
|
+
throw error;
|
|
7207
|
+
}
|
|
7208
|
+
}
|
|
7209
|
+
if (lastError) throw lastError;
|
|
7210
|
+
};
|
|
7211
|
+
const getLifecycle = async () => c.send(new GetBucketLifecycleConfigurationCommand({ Bucket: target.s3Bucket }));
|
|
7212
|
+
const putLifecycle = async (config) => {
|
|
7213
|
+
await c.send(
|
|
7214
|
+
new PutBucketLifecycleConfigurationCommand({
|
|
7215
|
+
Bucket: target.s3Bucket,
|
|
7216
|
+
LifecycleConfiguration: config
|
|
7217
|
+
})
|
|
7218
|
+
);
|
|
7219
|
+
};
|
|
7220
|
+
return { putNdjson, getLifecycle, putLifecycle };
|
|
7221
|
+
};
|
|
7222
|
+
|
|
7223
|
+
// src/exulu/audit/lifecycle.ts
|
|
7224
|
+
var AUDIT_LIFECYCLE_RULE_ID = "exulu-audit-retention";
|
|
7225
|
+
var buildRule = (prefix, retentionDays) => ({
|
|
7226
|
+
ID: AUDIT_LIFECYCLE_RULE_ID,
|
|
7227
|
+
Filter: { Prefix: prefix },
|
|
7228
|
+
Status: "Enabled",
|
|
7229
|
+
Expiration: { Days: retentionDays }
|
|
7230
|
+
});
|
|
7231
|
+
var applyRetentionLifecycle = async (writer, opts) => {
|
|
7232
|
+
const rule = buildRule(opts.prefix, opts.retentionDays);
|
|
7233
|
+
const config = { Rules: [rule] };
|
|
7234
|
+
if (!opts.manage) {
|
|
7235
|
+
console.warn(
|
|
7236
|
+
`[EXULU] audit retention: not managing the S3 lifecycle for this bucket. Apply this rule manually:
|
|
7237
|
+
${JSON.stringify(config, null, 2)}`
|
|
7238
|
+
);
|
|
7239
|
+
return;
|
|
7240
|
+
}
|
|
7241
|
+
try {
|
|
7242
|
+
let existing = [];
|
|
7243
|
+
try {
|
|
7244
|
+
const current = await writer.getLifecycle();
|
|
7245
|
+
existing = (current?.Rules ?? []).filter((r) => r.ID !== AUDIT_LIFECYCLE_RULE_ID);
|
|
7246
|
+
} catch (error) {
|
|
7247
|
+
if (error?.name !== "NoSuchLifecycleConfiguration") throw error;
|
|
7248
|
+
}
|
|
7249
|
+
await writer.putLifecycle({ Rules: [...existing, rule] });
|
|
7250
|
+
console.log(`[EXULU] audit retention: S3 lifecycle set to expire "${opts.prefix}" after ${opts.retentionDays} days.`);
|
|
7251
|
+
} catch (error) {
|
|
7252
|
+
console.warn(
|
|
7253
|
+
`[EXULU] audit retention: could not set the S3 lifecycle (${error?.name ?? "error"}). Apply this rule manually:
|
|
7254
|
+
${JSON.stringify(config, null, 2)}`
|
|
7255
|
+
);
|
|
7256
|
+
}
|
|
7257
|
+
};
|
|
7258
|
+
|
|
7259
|
+
// src/exulu/audit/sink.ts
|
|
7260
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
7261
|
+
import { promises as fs2 } from "fs";
|
|
7262
|
+
import path2 from "path";
|
|
7263
|
+
var createFsSpoolStore = (dir) => ({
|
|
7264
|
+
write: async (name, body) => {
|
|
7265
|
+
await fs2.mkdir(dir, { recursive: true });
|
|
7266
|
+
await fs2.writeFile(path2.join(dir, name), body, "utf8");
|
|
7267
|
+
},
|
|
7268
|
+
list: async () => {
|
|
7269
|
+
try {
|
|
7270
|
+
return (await fs2.readdir(dir)).filter((f) => f.endsWith(".ndjson"));
|
|
7271
|
+
} catch {
|
|
7272
|
+
return [];
|
|
7273
|
+
}
|
|
7274
|
+
},
|
|
7275
|
+
read: async (name) => fs2.readFile(path2.join(dir, name), "utf8"),
|
|
7276
|
+
remove: async (name) => {
|
|
7277
|
+
await fs2.rm(path2.join(dir, name), { force: true });
|
|
7278
|
+
}
|
|
7279
|
+
});
|
|
7280
|
+
var pad = (n) => String(n).padStart(2, "0");
|
|
7281
|
+
var AuditSink = class {
|
|
7282
|
+
constructor(cfg, writer, spool, opts) {
|
|
7283
|
+
this.cfg = cfg;
|
|
7284
|
+
this.writer = writer;
|
|
7285
|
+
this.spool = spool;
|
|
7286
|
+
this.now = opts?.now ?? (() => /* @__PURE__ */ new Date());
|
|
7287
|
+
}
|
|
7288
|
+
buffer = [];
|
|
7289
|
+
timer = null;
|
|
7290
|
+
now;
|
|
7291
|
+
objectKey() {
|
|
7292
|
+
const d = this.now();
|
|
7293
|
+
const dt = `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())}`;
|
|
7294
|
+
return `${this.cfg.target.s3prefix}dt=${dt}/${pad(d.getUTCHours())}/${d.getTime()}-${randomUUID4()}.ndjson`;
|
|
7295
|
+
}
|
|
7296
|
+
serialize(events) {
|
|
7297
|
+
return events.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
7298
|
+
}
|
|
7299
|
+
record(event) {
|
|
7300
|
+
this.buffer.push(event);
|
|
7301
|
+
if (this.buffer.length >= this.cfg.flush.maxRecords) {
|
|
7302
|
+
void this.flush();
|
|
7303
|
+
} else if (!this.timer) {
|
|
7304
|
+
this.timer = setTimeout(() => void this.flush(), this.cfg.flush.maxIntervalMs);
|
|
7305
|
+
this.timer.unref?.();
|
|
7306
|
+
}
|
|
7307
|
+
}
|
|
7308
|
+
async flush() {
|
|
7309
|
+
if (this.timer) {
|
|
7310
|
+
clearTimeout(this.timer);
|
|
7311
|
+
this.timer = null;
|
|
7312
|
+
}
|
|
7313
|
+
if (this.buffer.length === 0) return;
|
|
7314
|
+
const batch = this.buffer;
|
|
7315
|
+
this.buffer = [];
|
|
7316
|
+
const body = this.serialize(batch);
|
|
7317
|
+
try {
|
|
7318
|
+
await this.writer.putNdjson(this.objectKey(), body);
|
|
7319
|
+
await this.drainSpool();
|
|
7320
|
+
} catch (error) {
|
|
7321
|
+
const name = `${Date.now()}-${randomUUID4()}.ndjson`;
|
|
7322
|
+
try {
|
|
7323
|
+
await this.spool.write(name, body);
|
|
7324
|
+
console.warn(`[EXULU] audit: S3 write failed, spooled ${batch.length} record(s) to disk (${name}).`, error);
|
|
7325
|
+
} catch (spoolError) {
|
|
7326
|
+
console.error(`[EXULU] audit: S3 write AND local spool failed \u2014 ${batch.length} record(s) lost.`, spoolError);
|
|
7327
|
+
}
|
|
7328
|
+
}
|
|
7329
|
+
}
|
|
7330
|
+
async drainSpool() {
|
|
7331
|
+
const names = await this.spool.list();
|
|
7332
|
+
for (const name of names) {
|
|
7333
|
+
try {
|
|
7334
|
+
const body = await this.spool.read(name);
|
|
7335
|
+
await this.writer.putNdjson(this.objectKey(), body);
|
|
7336
|
+
await this.spool.remove(name);
|
|
7337
|
+
} catch {
|
|
7338
|
+
return;
|
|
7339
|
+
}
|
|
7340
|
+
}
|
|
7341
|
+
}
|
|
7342
|
+
async recordDurable(event) {
|
|
7343
|
+
await this.writer.putNdjson(this.objectKey(), this.serialize([event]));
|
|
7344
|
+
}
|
|
7345
|
+
async close() {
|
|
7346
|
+
await this.flush();
|
|
7347
|
+
}
|
|
7348
|
+
};
|
|
7349
|
+
|
|
7350
|
+
// src/exulu/audit/event.ts
|
|
7351
|
+
var AUDIT_EVENT_TYPES = {
|
|
7352
|
+
TOOL_CALL: "tool.call"
|
|
7353
|
+
};
|
|
7354
|
+
|
|
7355
|
+
// src/exulu/audit/redact.ts
|
|
7356
|
+
var SECRET_KEY_DENYLIST = [
|
|
7357
|
+
"oauth",
|
|
7358
|
+
"credentials",
|
|
7359
|
+
"accesstoken",
|
|
7360
|
+
"refreshtoken",
|
|
7361
|
+
"password",
|
|
7362
|
+
"secret",
|
|
7363
|
+
"token",
|
|
7364
|
+
"apikey",
|
|
7365
|
+
"authorization",
|
|
7366
|
+
"nonce"
|
|
7367
|
+
];
|
|
7368
|
+
var FRAMEWORK_INTERNAL_KEYS = /* @__PURE__ */ new Set([
|
|
7369
|
+
"req",
|
|
7370
|
+
"model",
|
|
7371
|
+
"contexts",
|
|
7372
|
+
"upload",
|
|
7373
|
+
"memory",
|
|
7374
|
+
"exuluConfig",
|
|
7375
|
+
"toolVariablesConfig",
|
|
7376
|
+
"allExuluTools",
|
|
7377
|
+
"currentTools",
|
|
7378
|
+
"sessionItems",
|
|
7379
|
+
"audit"
|
|
7380
|
+
]);
|
|
7381
|
+
var isSecretKey = (key, extra) => {
|
|
7382
|
+
const k = key.toLowerCase();
|
|
7383
|
+
if (extra.some((e) => k.includes(e.toLowerCase()))) return true;
|
|
7384
|
+
return SECRET_KEY_DENYLIST.some((term) => k.includes(term));
|
|
7385
|
+
};
|
|
7386
|
+
var redact = (value, redactKeys, seen) => {
|
|
7387
|
+
if (value === null || typeof value !== "object") return value;
|
|
7388
|
+
if (seen.has(value)) return "[circular]";
|
|
7389
|
+
seen.add(value);
|
|
7390
|
+
if (Array.isArray(value)) return value.map((v) => redact(v, redactKeys, seen));
|
|
7391
|
+
const out = {};
|
|
7392
|
+
for (const [key, val] of Object.entries(value)) {
|
|
7393
|
+
if (FRAMEWORK_INTERNAL_KEYS.has(key)) continue;
|
|
7394
|
+
if (isSecretKey(key, redactKeys)) {
|
|
7395
|
+
if (val !== null && typeof val === "object") {
|
|
7396
|
+
out[key] = "[redacted]";
|
|
7397
|
+
}
|
|
7398
|
+
continue;
|
|
7399
|
+
}
|
|
7400
|
+
out[key] = redact(val, redactKeys, seen);
|
|
7401
|
+
}
|
|
7402
|
+
return out;
|
|
7403
|
+
};
|
|
7404
|
+
var sanitizeData = (value, opts) => {
|
|
7405
|
+
let cleaned;
|
|
7406
|
+
try {
|
|
7407
|
+
cleaned = redact(value, opts.redactKeys ?? [], /* @__PURE__ */ new WeakSet());
|
|
7408
|
+
} catch {
|
|
7409
|
+
cleaned = "[unserializable]";
|
|
7410
|
+
}
|
|
7411
|
+
let serialized;
|
|
7412
|
+
try {
|
|
7413
|
+
serialized = JSON.stringify(cleaned) ?? "";
|
|
7414
|
+
} catch {
|
|
7415
|
+
return { value: "[unserializable]", truncated: false };
|
|
7416
|
+
}
|
|
7417
|
+
if (serialized.length <= opts.maxBytes) return { value: cleaned, truncated: false };
|
|
7418
|
+
return {
|
|
7419
|
+
value: { _truncated: true, preview: serialized.slice(0, opts.maxBytes) },
|
|
7420
|
+
truncated: true
|
|
7421
|
+
};
|
|
7422
|
+
};
|
|
7423
|
+
|
|
7424
|
+
// src/exulu/auth/describe.ts
|
|
7425
|
+
var describeCredentialIdentity = async (auth, userId, toolId) => {
|
|
7426
|
+
const provider = providerKeyFor(toolId ?? auth.provider, auth);
|
|
7427
|
+
const base = {
|
|
7428
|
+
provider,
|
|
7429
|
+
authType: auth.authType,
|
|
7430
|
+
account: String(userId)
|
|
7431
|
+
};
|
|
7432
|
+
if (auth.authType !== "oauth") return base;
|
|
7433
|
+
try {
|
|
7434
|
+
const row = await credentialStore.get(provider, userId);
|
|
7435
|
+
if (!row || row.authType !== "oauth") return base;
|
|
7436
|
+
const { scopes, expiresAt } = row.data;
|
|
7437
|
+
return {
|
|
7438
|
+
...base,
|
|
7439
|
+
...scopes ? { scopes: scopes.split(" ").filter(Boolean) } : {},
|
|
7440
|
+
...expiresAt !== void 0 ? { expiresAt } : {}
|
|
7441
|
+
};
|
|
7442
|
+
} catch (error) {
|
|
7443
|
+
console.error(`[EXULU] describeCredentialIdentity failed for provider "${provider}":`, error);
|
|
7444
|
+
return base;
|
|
7445
|
+
}
|
|
7446
|
+
};
|
|
7447
|
+
|
|
7448
|
+
// src/exulu/audit/emitters/tool-call.ts
|
|
7449
|
+
var str = (v) => v === void 0 || v === null ? void 0 : String(v);
|
|
7450
|
+
var isAuthShortCircuit = (output) => {
|
|
7451
|
+
if (!output || typeof output !== "object") return false;
|
|
7452
|
+
const o = output;
|
|
7453
|
+
return !!o.credentialRequest || !!o.oauth?.authorizationUrl;
|
|
7454
|
+
};
|
|
7455
|
+
var buildToolCallEvent = async (ctx, opts) => {
|
|
7456
|
+
const nowIso = opts.nowIso ?? (() => (/* @__PURE__ */ new Date()).toISOString());
|
|
7457
|
+
const status = ctx.status === "error" ? "error" : isAuthShortCircuit(ctx.output) ? "auth_required" : "ok";
|
|
7458
|
+
const input = sanitizeData(ctx.input, { maxBytes: opts.maxBytes, redactKeys: opts.redactKeys });
|
|
7459
|
+
const data = { input: input.value };
|
|
7460
|
+
const truncated = {};
|
|
7461
|
+
if (input.truncated) truncated.input = true;
|
|
7462
|
+
if (opts.captureOutput && status !== "auth_required") {
|
|
7463
|
+
const output = sanitizeData(ctx.output, { maxBytes: opts.maxBytes, redactKeys: opts.redactKeys });
|
|
7464
|
+
data.output = output.value;
|
|
7465
|
+
if (output.truncated) truncated.output = true;
|
|
7466
|
+
}
|
|
7467
|
+
let credential;
|
|
7468
|
+
if (ctx.tool.authentication && ctx.user?.id != null) {
|
|
7469
|
+
credential = await describeCredentialIdentity(
|
|
7470
|
+
ctx.tool.authentication,
|
|
7471
|
+
Number(ctx.user.id),
|
|
7472
|
+
ctx.tool.id
|
|
7473
|
+
);
|
|
7474
|
+
}
|
|
7475
|
+
const err = ctx.error;
|
|
7476
|
+
return {
|
|
7477
|
+
v: 1,
|
|
7478
|
+
ts: nowIso(),
|
|
7479
|
+
type: AUDIT_EVENT_TYPES.TOOL_CALL,
|
|
7480
|
+
actor: {
|
|
7481
|
+
kind: "user",
|
|
7482
|
+
userId: str(ctx.user?.id),
|
|
7483
|
+
email: ctx.user?.email,
|
|
7484
|
+
roleId: str(ctx.user?.role?.id),
|
|
7485
|
+
projectId: ctx.projectId
|
|
7486
|
+
},
|
|
7487
|
+
context: {
|
|
7488
|
+
sessionId: ctx.sessionID,
|
|
7489
|
+
agentId: ctx.agent?.id,
|
|
7490
|
+
agentName: ctx.agent?.name,
|
|
7491
|
+
toolCallId: ctx.toolCallId
|
|
7492
|
+
},
|
|
7493
|
+
target: { kind: "tool", id: ctx.tool.id, name: ctx.tool.name, category: ctx.tool.category, builtin: ctx.builtin },
|
|
7494
|
+
...ctx.client ? { client: ctx.client } : {},
|
|
7495
|
+
...credential ? { credential } : {},
|
|
7496
|
+
status,
|
|
7497
|
+
...status === "error" ? { error: { name: err?.name, message: String(err?.message ?? err ?? "unknown error") } } : {},
|
|
7498
|
+
data,
|
|
7499
|
+
durationMs: ctx.durationMs,
|
|
7500
|
+
...Object.keys(truncated).length ? { truncated } : {}
|
|
7501
|
+
};
|
|
7502
|
+
};
|
|
7503
|
+
|
|
7504
|
+
// src/exulu/audit/logger.ts
|
|
7505
|
+
var noop = {
|
|
7506
|
+
enabled: false,
|
|
7507
|
+
failClosed: false,
|
|
7508
|
+
isBuiltin: () => false,
|
|
7509
|
+
shouldAuditTool: () => false,
|
|
7510
|
+
record: () => {
|
|
7511
|
+
},
|
|
7512
|
+
recordToolCall: async () => {
|
|
7513
|
+
},
|
|
7514
|
+
flush: async () => {
|
|
7515
|
+
},
|
|
7516
|
+
close: async () => {
|
|
7517
|
+
}
|
|
7518
|
+
};
|
|
7519
|
+
var RealAuditLogger = class {
|
|
7520
|
+
constructor(resolved, builtinToolIds) {
|
|
7521
|
+
this.resolved = resolved;
|
|
7522
|
+
this.builtinToolIds = builtinToolIds;
|
|
7523
|
+
this.failClosed = resolved.failureMode === "closed";
|
|
7524
|
+
const writer = createAuditS3Writer(resolved.target);
|
|
7525
|
+
this.sink = new AuditSink(resolved, writer, createFsSpoolStore(resolved.spoolDir));
|
|
7526
|
+
}
|
|
7527
|
+
enabled = true;
|
|
7528
|
+
failClosed;
|
|
7529
|
+
sink;
|
|
7530
|
+
lifecycleWriter() {
|
|
7531
|
+
return createAuditS3Writer(this.resolved.target);
|
|
7532
|
+
}
|
|
7533
|
+
isBuiltin(id) {
|
|
7534
|
+
return this.builtinToolIds.has(id);
|
|
7535
|
+
}
|
|
7536
|
+
shouldAuditTool(id) {
|
|
7537
|
+
const t = this.resolved.toolCalls;
|
|
7538
|
+
if (!t.enabled) return false;
|
|
7539
|
+
if (t.exclude.includes(id)) return false;
|
|
7540
|
+
if (t.include.length > 0) return t.include.includes(id);
|
|
7541
|
+
return true;
|
|
7542
|
+
}
|
|
7543
|
+
record(event) {
|
|
7544
|
+
this.sink.record(event);
|
|
7545
|
+
}
|
|
7546
|
+
async recordToolCall(ctx) {
|
|
7547
|
+
const event = await buildToolCallEvent(ctx, {
|
|
7548
|
+
maxBytes: this.resolved.payload.maxBytes,
|
|
7549
|
+
captureOutput: this.resolved.payload.captureOutput,
|
|
7550
|
+
redactKeys: this.resolved.payload.redactKeys
|
|
7551
|
+
});
|
|
7552
|
+
if (this.failClosed) await this.sink.recordDurable(event);
|
|
7553
|
+
else this.sink.record(event);
|
|
7554
|
+
}
|
|
7555
|
+
flush() {
|
|
7556
|
+
return this.sink.flush();
|
|
7557
|
+
}
|
|
7558
|
+
close() {
|
|
7559
|
+
return this.sink.close();
|
|
7560
|
+
}
|
|
7561
|
+
get resolvedConfig() {
|
|
7562
|
+
return this.resolved;
|
|
7563
|
+
}
|
|
7564
|
+
};
|
|
7565
|
+
var _instance;
|
|
7566
|
+
var _signalClose;
|
|
7567
|
+
var build = (config, builtinToolIds) => {
|
|
7568
|
+
const resolved = resolveAuditConfig(config);
|
|
7569
|
+
return resolved ? new RealAuditLogger(resolved, builtinToolIds) : noop;
|
|
7570
|
+
};
|
|
7571
|
+
var getAuditLogger = (config) => {
|
|
7572
|
+
if (!_instance) _instance = build(config, /* @__PURE__ */ new Set());
|
|
7573
|
+
return _instance;
|
|
7574
|
+
};
|
|
7575
|
+
var initAudit = async (config, opts) => {
|
|
7576
|
+
_instance = build(config, opts?.builtinToolIds ?? /* @__PURE__ */ new Set());
|
|
7577
|
+
if (_instance instanceof RealAuditLogger) {
|
|
7578
|
+
const r = _instance.resolvedConfig;
|
|
7579
|
+
await applyRetentionLifecycle(_instance.lifecycleWriter(), {
|
|
7580
|
+
prefix: r.target.s3prefix,
|
|
7581
|
+
retentionDays: r.retentionDays,
|
|
7582
|
+
manage: r.manageLifecycle
|
|
7583
|
+
});
|
|
7584
|
+
if (_signalClose) {
|
|
7585
|
+
process.off("SIGTERM", _signalClose);
|
|
7586
|
+
process.off("SIGINT", _signalClose);
|
|
7587
|
+
}
|
|
7588
|
+
const close = () => {
|
|
7589
|
+
void _instance?.close();
|
|
7590
|
+
};
|
|
7591
|
+
_signalClose = close;
|
|
7592
|
+
process.on("SIGTERM", close);
|
|
7593
|
+
process.on("SIGINT", close);
|
|
7594
|
+
}
|
|
7595
|
+
return _instance;
|
|
7596
|
+
};
|
|
7597
|
+
|
|
7598
|
+
// src/exulu/audit/emit-tool-call.ts
|
|
7599
|
+
var emitToolCallAudit = async (logger, ctx) => {
|
|
7600
|
+
if (!logger.shouldAuditTool(ctx.tool.id)) return;
|
|
7601
|
+
const full = { ...ctx, builtin: logger.isBuiltin(ctx.tool.id) };
|
|
7602
|
+
if (logger.failClosed) {
|
|
7603
|
+
await logger.recordToolCall(full);
|
|
7604
|
+
return;
|
|
7605
|
+
}
|
|
7606
|
+
logger.recordToolCall(full).catch(
|
|
7607
|
+
(error) => console.error(`[EXULU] audit: recordToolCall failed for tool "${ctx.tool.id}":`, error)
|
|
7608
|
+
);
|
|
7609
|
+
};
|
|
7610
|
+
|
|
7611
|
+
// src/exulu/audit/client-info.ts
|
|
7612
|
+
var firstHeader = (v) => Array.isArray(v) ? v[0] : v ?? void 0;
|
|
7613
|
+
function extractClientInfo(req) {
|
|
7614
|
+
if (!req) return void 0;
|
|
7615
|
+
const headers = req.headers ?? {};
|
|
7616
|
+
const forwardedFor = firstHeader(headers["x-forwarded-for"]);
|
|
7617
|
+
const ip = (forwardedFor ? forwardedFor.split(",")[0]?.trim() : void 0) || req.ip || req.socket?.remoteAddress || void 0;
|
|
7618
|
+
const userAgent = firstHeader(headers["user-agent"]);
|
|
7619
|
+
const referer = firstHeader(headers["referer"]);
|
|
7620
|
+
const origin = firstHeader(headers["origin"]);
|
|
7621
|
+
const client = {};
|
|
7622
|
+
if (ip) client.ip = ip;
|
|
7623
|
+
if (userAgent) client.userAgent = userAgent;
|
|
7624
|
+
if (referer) client.referer = referer;
|
|
7625
|
+
if (origin) client.origin = origin;
|
|
7626
|
+
if (forwardedFor) client.forwardedFor = forwardedFor;
|
|
7627
|
+
return Object.keys(client).length > 0 ? client : void 0;
|
|
7628
|
+
}
|
|
7629
|
+
|
|
7280
7630
|
// src/templates/tools/convert-exulu-tools-to-ai-sdk-tools.ts
|
|
7281
7631
|
var OUTPUT_OFFLOAD_EXEMPT_TOOL_IDS = /* @__PURE__ */ new Set(["agentic_context_search"]);
|
|
7282
|
-
var generateS3Key = (filename) => `${
|
|
7632
|
+
var generateS3Key = (filename) => `${randomUUID5()}-${filename}`;
|
|
7283
7633
|
var s3Client2;
|
|
7284
7634
|
var getMimeType = (type) => {
|
|
7285
7635
|
switch (type) {
|
|
@@ -7368,8 +7718,8 @@ var hydrateVariables = async (tool3) => {
|
|
|
7368
7718
|
}
|
|
7369
7719
|
let value = variable.value;
|
|
7370
7720
|
if (variable.encrypted) {
|
|
7371
|
-
const bytes =
|
|
7372
|
-
value = bytes.toString(
|
|
7721
|
+
const bytes = CryptoJS4.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
|
|
7722
|
+
value = bytes.toString(CryptoJS4.enc.Utf8);
|
|
7373
7723
|
}
|
|
7374
7724
|
toolConfig.value = value;
|
|
7375
7725
|
return toolConfig;
|
|
@@ -7377,7 +7727,7 @@ var hydrateVariables = async (tool3) => {
|
|
|
7377
7727
|
await Promise.all(promises);
|
|
7378
7728
|
return tool3;
|
|
7379
7729
|
};
|
|
7380
|
-
var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approvedTools, allExuluTools, configs,
|
|
7730
|
+
var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approvedTools, allExuluTools, configs, contexts, user, exuluConfig, sessionID, req, project, sessionItems, model, agent, memoryItems, contextWindow, disabledTools) => {
|
|
7381
7731
|
if (!currentTools) return {};
|
|
7382
7732
|
if (!allExuluTools) {
|
|
7383
7733
|
allExuluTools = [];
|
|
@@ -7612,124 +7962,163 @@ var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approved
|
|
|
7612
7962
|
"and options",
|
|
7613
7963
|
options
|
|
7614
7964
|
);
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7620
|
-
|
|
7621
|
-
|
|
7622
|
-
|
|
7623
|
-
|
|
7624
|
-
|
|
7625
|
-
|
|
7626
|
-
|
|
7627
|
-
|
|
7628
|
-
|
|
7629
|
-
|
|
7630
|
-
|
|
7631
|
-
|
|
7632
|
-
|
|
7633
|
-
|
|
7634
|
-
|
|
7635
|
-
|
|
7636
|
-
|
|
7637
|
-
|
|
7638
|
-
type
|
|
7639
|
-
}) => {
|
|
7640
|
-
const mime = getMimeType(type);
|
|
7641
|
-
const prefix = exuluConfig?.fileUploads?.s3prefix ? `${exuluConfig.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
|
|
7642
|
-
const key = `${prefix}${user?.id}/${generateS3Key(name)}${type}`;
|
|
7643
|
-
const command = new PutObjectCommand2({
|
|
7644
|
-
Bucket: exuluConfig?.fileUploads?.s3Bucket,
|
|
7645
|
-
Key: key,
|
|
7646
|
-
Body: data,
|
|
7647
|
-
ContentType: mime
|
|
7648
|
-
});
|
|
7649
|
-
try {
|
|
7650
|
-
if (!s3Client2) {
|
|
7651
|
-
throw new Error("S3 client not initialized");
|
|
7965
|
+
const __auditStart = Date.now();
|
|
7966
|
+
let __auditOutput;
|
|
7967
|
+
let __auditStatus = "ok";
|
|
7968
|
+
let __auditError;
|
|
7969
|
+
try {
|
|
7970
|
+
if (!cur.tool?.execute) {
|
|
7971
|
+
console.error("[EXULU] Tool execute function is undefined.", cur.tool);
|
|
7972
|
+
throw new Error("Tool execute function is undefined.");
|
|
7973
|
+
}
|
|
7974
|
+
if (toolVariableConfig) {
|
|
7975
|
+
toolVariableConfig = await hydrateVariables(toolVariableConfig || []);
|
|
7976
|
+
}
|
|
7977
|
+
let upload = void 0;
|
|
7978
|
+
if (exuluConfig?.fileUploads?.s3endpoint && exuluConfig?.fileUploads?.s3key && exuluConfig?.fileUploads?.s3secret && exuluConfig?.fileUploads?.s3Bucket) {
|
|
7979
|
+
s3Client2 ??= new S3Client3({
|
|
7980
|
+
region: exuluConfig?.fileUploads?.s3region,
|
|
7981
|
+
...exuluConfig?.fileUploads?.s3endpoint && {
|
|
7982
|
+
forcePathStyle: true,
|
|
7983
|
+
endpoint: exuluConfig?.fileUploads?.s3endpoint
|
|
7984
|
+
},
|
|
7985
|
+
credentials: {
|
|
7986
|
+
accessKeyId: exuluConfig?.fileUploads?.s3key ?? "",
|
|
7987
|
+
secretAccessKey: exuluConfig?.fileUploads?.s3secret ?? ""
|
|
7652
7988
|
}
|
|
7653
|
-
|
|
7654
|
-
|
|
7655
|
-
|
|
7656
|
-
|
|
7657
|
-
|
|
7658
|
-
|
|
7659
|
-
|
|
7660
|
-
|
|
7661
|
-
|
|
7662
|
-
|
|
7663
|
-
|
|
7664
|
-
|
|
7665
|
-
|
|
7989
|
+
});
|
|
7990
|
+
upload = async ({
|
|
7991
|
+
name,
|
|
7992
|
+
data,
|
|
7993
|
+
type
|
|
7994
|
+
}) => {
|
|
7995
|
+
const mime = getMimeType(type);
|
|
7996
|
+
const prefix = exuluConfig?.fileUploads?.s3prefix ? `${exuluConfig.fileUploads.s3prefix.replace(/\/$/, "")}/` : "";
|
|
7997
|
+
const key = `${prefix}${user?.id}/${generateS3Key(name)}${type}`;
|
|
7998
|
+
const command = new PutObjectCommand3({
|
|
7999
|
+
Bucket: exuluConfig?.fileUploads?.s3Bucket,
|
|
8000
|
+
Key: key,
|
|
8001
|
+
Body: data,
|
|
8002
|
+
ContentType: mime
|
|
8003
|
+
});
|
|
8004
|
+
try {
|
|
8005
|
+
if (!s3Client2) {
|
|
8006
|
+
throw new Error("S3 client not initialized");
|
|
8007
|
+
}
|
|
8008
|
+
await s3Client2.send(command);
|
|
8009
|
+
const bucket = exuluConfig?.fileUploads?.s3Bucket ?? "";
|
|
8010
|
+
const presignedUrl = await getPresignedUrl(bucket, key, exuluConfig);
|
|
8011
|
+
return { url: presignedUrl, key: `${bucket}/${key}` };
|
|
8012
|
+
} catch (caught) {
|
|
8013
|
+
if (caught instanceof S3ServiceException && caught.name === "EntityTooLarge") {
|
|
8014
|
+
throw new Error(`[EXULU] Error from S3 while uploading object to ${exuluConfig?.fileUploads?.s3Bucket}. The object was too large. To upload objects larger than 5GB, use the S3 console (160GB max) or the multipart upload API (5TB max).`);
|
|
8015
|
+
} else if (caught instanceof S3ServiceException) {
|
|
8016
|
+
throw new Error(
|
|
8017
|
+
`[EXULU] Error from S3 while uploading object to ${exuluConfig?.fileUploads?.s3Bucket}. ${caught.name}: ${caught.message}`
|
|
8018
|
+
);
|
|
8019
|
+
} else {
|
|
8020
|
+
throw caught;
|
|
8021
|
+
}
|
|
7666
8022
|
}
|
|
7667
|
-
}
|
|
7668
|
-
}
|
|
7669
|
-
|
|
7670
|
-
|
|
7671
|
-
|
|
7672
|
-
|
|
7673
|
-
|
|
7674
|
-
|
|
7675
|
-
|
|
7676
|
-
|
|
7677
|
-
|
|
7678
|
-
|
|
7679
|
-
|
|
7680
|
-
|
|
7681
|
-
|
|
8023
|
+
};
|
|
8024
|
+
}
|
|
8025
|
+
const contextsMap = contexts?.reduce((acc, curr) => {
|
|
8026
|
+
acc[curr.id] = curr;
|
|
8027
|
+
return acc;
|
|
8028
|
+
}, {});
|
|
8029
|
+
const toolVariablesConfigData = toolVariableConfig ? toolVariableConfig.config.reduce((acc, curr) => {
|
|
8030
|
+
acc[curr.name] = curr.value;
|
|
8031
|
+
return acc;
|
|
8032
|
+
}, {}) : {};
|
|
8033
|
+
const response = await cur.tool.execute(
|
|
8034
|
+
{
|
|
8035
|
+
...inputs,
|
|
8036
|
+
model,
|
|
8037
|
+
sessionID,
|
|
8038
|
+
sessionItems,
|
|
8039
|
+
memory: memoryItems,
|
|
8040
|
+
req,
|
|
8041
|
+
// Convert config to object format if a config object
|
|
8042
|
+
// is available, after we added the .value property
|
|
8043
|
+
// by hydrating it from the variables table.
|
|
8044
|
+
allExuluTools,
|
|
8045
|
+
currentTools,
|
|
8046
|
+
user,
|
|
8047
|
+
contexts: contextsMap,
|
|
8048
|
+
upload,
|
|
8049
|
+
exuluConfig,
|
|
8050
|
+
toolVariablesConfig: toolVariablesConfigData
|
|
8051
|
+
},
|
|
8052
|
+
options
|
|
8053
|
+
);
|
|
8054
|
+
await updateStatistic({
|
|
8055
|
+
name: "count",
|
|
8056
|
+
label: cur.name,
|
|
8057
|
+
type: STATISTICS_TYPE_ENUM.TOOL_CALL,
|
|
8058
|
+
trigger: "agent",
|
|
8059
|
+
count: 1,
|
|
8060
|
+
user: user?.id,
|
|
8061
|
+
role: user?.role?.id
|
|
8062
|
+
});
|
|
8063
|
+
const guardCtx = {
|
|
8064
|
+
toolName: cur.name,
|
|
8065
|
+
contextWindow,
|
|
7682
8066
|
sessionID,
|
|
7683
|
-
sessionItems,
|
|
7684
|
-
memory: memoryItems,
|
|
7685
|
-
req,
|
|
7686
|
-
// Convert config to object format if a config object
|
|
7687
|
-
// is available, after we added the .value property
|
|
7688
|
-
// by hydrating it from the variables table.
|
|
7689
|
-
providerapikey,
|
|
7690
|
-
allExuluTools,
|
|
7691
|
-
currentTools,
|
|
7692
8067
|
user,
|
|
7693
|
-
|
|
7694
|
-
|
|
7695
|
-
|
|
7696
|
-
|
|
7697
|
-
|
|
7698
|
-
|
|
7699
|
-
|
|
7700
|
-
|
|
7701
|
-
|
|
7702
|
-
|
|
7703
|
-
|
|
7704
|
-
|
|
7705
|
-
|
|
7706
|
-
|
|
7707
|
-
|
|
7708
|
-
|
|
7709
|
-
|
|
7710
|
-
|
|
7711
|
-
|
|
7712
|
-
|
|
7713
|
-
|
|
7714
|
-
exuluConfig
|
|
7715
|
-
};
|
|
7716
|
-
const offloadExempt = OUTPUT_OFFLOAD_EXEMPT_TOOL_IDS.has(cur.id);
|
|
7717
|
-
if (response && typeof response === "object" && Symbol.asyncIterator in response) {
|
|
7718
|
-
let lastValue;
|
|
7719
|
-
for await (const value of response) {
|
|
7720
|
-
yield value;
|
|
7721
|
-
lastValue = value;
|
|
7722
|
-
}
|
|
7723
|
-
if (offloadExempt) return lastValue;
|
|
7724
|
-
const guarded = await guardToolOutput(lastValue, guardCtx);
|
|
7725
|
-
if (guarded !== lastValue) {
|
|
8068
|
+
exuluConfig
|
|
8069
|
+
};
|
|
8070
|
+
const offloadExempt = OUTPUT_OFFLOAD_EXEMPT_TOOL_IDS.has(cur.id);
|
|
8071
|
+
if (response && typeof response === "object" && Symbol.asyncIterator in response) {
|
|
8072
|
+
let lastValue;
|
|
8073
|
+
for await (const value of response) {
|
|
8074
|
+
yield value;
|
|
8075
|
+
lastValue = value;
|
|
8076
|
+
}
|
|
8077
|
+
if (offloadExempt) {
|
|
8078
|
+
__auditOutput = lastValue;
|
|
8079
|
+
return lastValue;
|
|
8080
|
+
}
|
|
8081
|
+
const guarded = await guardToolOutput(lastValue, guardCtx);
|
|
8082
|
+
if (guarded !== lastValue) {
|
|
8083
|
+
yield guarded;
|
|
8084
|
+
}
|
|
8085
|
+
__auditOutput = guarded;
|
|
8086
|
+
return guarded;
|
|
8087
|
+
} else {
|
|
8088
|
+
const guarded = offloadExempt ? response : await guardToolOutput(response, guardCtx);
|
|
7726
8089
|
yield guarded;
|
|
8090
|
+
__auditOutput = guarded;
|
|
8091
|
+
return guarded;
|
|
8092
|
+
}
|
|
8093
|
+
} catch (error) {
|
|
8094
|
+
__auditStatus = "error";
|
|
8095
|
+
__auditError = error;
|
|
8096
|
+
throw error;
|
|
8097
|
+
} finally {
|
|
8098
|
+
const __auditLogger = getAuditLogger(exuluConfig ?? {});
|
|
8099
|
+
if (__auditLogger.shouldAuditTool(cur.id)) {
|
|
8100
|
+
const __emit = emitToolCallAudit(__auditLogger, {
|
|
8101
|
+
durationMs: Date.now() - __auditStart,
|
|
8102
|
+
agent: agent ? { id: agent.id, name: agent.name, slug: agent.slug } : void 0,
|
|
8103
|
+
tool: { id: cur.id, name: cur.name, category: cur.category, authentication: cur.authentication },
|
|
8104
|
+
user,
|
|
8105
|
+
projectId: project ? String(project) : void 0,
|
|
8106
|
+
sessionID,
|
|
8107
|
+
toolCallId: options?.toolCallId,
|
|
8108
|
+
input: inputs,
|
|
8109
|
+
output: __auditOutput,
|
|
8110
|
+
status: __auditStatus,
|
|
8111
|
+
error: __auditError,
|
|
8112
|
+
client: extractClientInfo(req)
|
|
8113
|
+
});
|
|
8114
|
+
if (__auditLogger.failClosed) {
|
|
8115
|
+
await __emit;
|
|
8116
|
+
} else {
|
|
8117
|
+
__emit.catch(
|
|
8118
|
+
(error) => console.error(`[EXULU] audit: tool-call emit failed for "${cur.id}":`, error)
|
|
8119
|
+
);
|
|
8120
|
+
}
|
|
7727
8121
|
}
|
|
7728
|
-
return guarded;
|
|
7729
|
-
} else {
|
|
7730
|
-
const guarded = offloadExempt ? response : await guardToolOutput(response, guardCtx);
|
|
7731
|
-
yield guarded;
|
|
7732
|
-
return guarded;
|
|
7733
8122
|
}
|
|
7734
8123
|
}
|
|
7735
8124
|
}
|
|
@@ -7781,7 +8170,6 @@ export {
|
|
|
7781
8170
|
getUserBudgetView,
|
|
7782
8171
|
updateStatistic,
|
|
7783
8172
|
checkLicense,
|
|
7784
|
-
checkRecordAccess,
|
|
7785
8173
|
ResolveModelError,
|
|
7786
8174
|
resolveModel,
|
|
7787
8175
|
exuluApp,
|
|
@@ -7815,6 +8203,8 @@ export {
|
|
|
7815
8203
|
PreviewRenderError,
|
|
7816
8204
|
getPdfPreviewBytes,
|
|
7817
8205
|
imageAttachmentGuard,
|
|
8206
|
+
getAuditLogger,
|
|
8207
|
+
initAudit,
|
|
7818
8208
|
hydrateVariables,
|
|
7819
8209
|
convertExuluToolsToAiSdkTools,
|
|
7820
8210
|
ExuluTool,
|