@exulu/backend 1.42.2 → 1.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -3
- package/dist/index.cjs +390 -238
- package/dist/index.d.cts +4 -7
- package/dist/index.d.ts +4 -7
- package/dist/index.js +390 -237
- package/package.json +1 -1
- package/types/models/agent.ts +1 -0
package/dist/index.js
CHANGED
|
@@ -1435,6 +1435,10 @@ var addCoreFields = (schema) => {
|
|
|
1435
1435
|
name: "last_processed_at",
|
|
1436
1436
|
type: "date"
|
|
1437
1437
|
});
|
|
1438
|
+
schema.fields.push({
|
|
1439
|
+
name: "embeddings_updated_at",
|
|
1440
|
+
type: "date"
|
|
1441
|
+
});
|
|
1438
1442
|
if (schema.RBAC) {
|
|
1439
1443
|
if (!schema.fields.some((field) => field.name === "rights_mode")) {
|
|
1440
1444
|
schema.fields.push({
|
|
@@ -1648,7 +1652,8 @@ var checkRecordAccess = async (record, request, user) => {
|
|
|
1648
1652
|
const isPublic = record.rights_mode === "public";
|
|
1649
1653
|
const byUsers = record.rights_mode === "users";
|
|
1650
1654
|
const byRoles = record.rights_mode === "roles";
|
|
1651
|
-
const
|
|
1655
|
+
const createdBy = typeof record.created_by === "string" ? record.created_by : record.created_by?.toString();
|
|
1656
|
+
const isCreator = user ? createdBy === user.id.toString() : false;
|
|
1652
1657
|
const isAdmin = user ? user.super_admin : false;
|
|
1653
1658
|
const isApi = user ? user.type === "api" : false;
|
|
1654
1659
|
let hasAccess = "none";
|
|
@@ -1855,6 +1860,7 @@ ${enumValues}
|
|
|
1855
1860
|
fields.push(" maxContextLength: Int");
|
|
1856
1861
|
fields.push(" provider: String");
|
|
1857
1862
|
fields.push(" authenticationInformation: String");
|
|
1863
|
+
fields.push(" systemInstructions: String");
|
|
1858
1864
|
fields.push(" slug: String");
|
|
1859
1865
|
}
|
|
1860
1866
|
const rbacField = table.RBAC ? " RBAC: RBACData" : "";
|
|
@@ -1919,6 +1925,8 @@ input FilterOperatorDate {
|
|
|
1919
1925
|
input FilterOperatorFloat {
|
|
1920
1926
|
eq: Float
|
|
1921
1927
|
ne: Float
|
|
1928
|
+
lte: Float
|
|
1929
|
+
gte: Float
|
|
1922
1930
|
in: [Float]
|
|
1923
1931
|
and: [FilterOperatorFloat]
|
|
1924
1932
|
or: [FilterOperatorFloat]
|
|
@@ -1982,7 +1990,11 @@ var getRequestedFields = (info) => {
|
|
|
1982
1990
|
return fields.filter((field) => field !== "pageInfo" && field !== "items" && field !== "RBAC");
|
|
1983
1991
|
};
|
|
1984
1992
|
var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRbacRecords) => {
|
|
1985
|
-
const {
|
|
1993
|
+
const {
|
|
1994
|
+
users = [],
|
|
1995
|
+
roles = []
|
|
1996
|
+
/* projects = [] */
|
|
1997
|
+
} = rbacData;
|
|
1986
1998
|
if (!existingRbacRecords) {
|
|
1987
1999
|
existingRbacRecords = await db3.from("rbac").where({
|
|
1988
2000
|
entity: entityName,
|
|
@@ -1991,25 +2003,19 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
|
|
|
1991
2003
|
}
|
|
1992
2004
|
const newUserRecords = new Set(users.map((u) => `${u.id}:${u.rights}`));
|
|
1993
2005
|
const newRoleRecords = new Set(roles.map((r) => `${r.id}:${r.rights}`));
|
|
1994
|
-
const newProjectRecords = new Set(projects.map((p) => `${p.id}:${p.rights}`));
|
|
1995
2006
|
const existingUserRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "User").map((r) => `${r.user_id}:${r.rights}`));
|
|
1996
2007
|
const existingRoleRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "Role").map((r) => `${r.role_id}:${r.rights}`));
|
|
1997
2008
|
const existingProjectRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "Project").map((r) => `${r.project_id}:${r.rights}`));
|
|
1998
2009
|
const usersToCreate = users.filter((u) => !existingUserRecords.has(`${u.id}:${u.rights}`));
|
|
1999
2010
|
const rolesToCreate = roles.filter((r) => !existingRoleRecords.has(`${r.id}:${r.rights}`));
|
|
2000
|
-
const projectsToCreate = projects.filter((p) => !existingProjectRecords.has(`${p.id}:${p.rights}`));
|
|
2001
2011
|
const usersToRemove = existingRbacRecords.filter((r) => r.access_type === "User" && !newUserRecords.has(`${r.user_id}:${r.rights}`));
|
|
2002
2012
|
const rolesToRemove = existingRbacRecords.filter((r) => r.access_type === "Role" && !newRoleRecords.has(`${r.role_id}:${r.rights}`));
|
|
2003
|
-
const projectsToRemove = existingRbacRecords.filter((r) => r.access_type === "Project" && !newProjectRecords.has(`${r.project_id}:${r.rights}`));
|
|
2004
2013
|
if (usersToRemove.length > 0) {
|
|
2005
2014
|
await db3.from("rbac").whereIn("id", usersToRemove.map((r) => r.id)).del();
|
|
2006
2015
|
}
|
|
2007
2016
|
if (rolesToRemove.length > 0) {
|
|
2008
2017
|
await db3.from("rbac").whereIn("id", rolesToRemove.map((r) => r.id)).del();
|
|
2009
2018
|
}
|
|
2010
|
-
if (projectsToRemove.length > 0) {
|
|
2011
|
-
await db3.from("rbac").whereIn("id", projectsToRemove.map((r) => r.id)).del();
|
|
2012
|
-
}
|
|
2013
2019
|
const recordsToInsert = [];
|
|
2014
2020
|
usersToCreate.forEach((user) => {
|
|
2015
2021
|
recordsToInsert.push({
|
|
@@ -2033,17 +2039,6 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
|
|
|
2033
2039
|
updatedAt: /* @__PURE__ */ new Date()
|
|
2034
2040
|
});
|
|
2035
2041
|
});
|
|
2036
|
-
projectsToCreate.forEach((project) => {
|
|
2037
|
-
recordsToInsert.push({
|
|
2038
|
-
entity: entityName,
|
|
2039
|
-
access_type: "Project",
|
|
2040
|
-
target_resource_id: resourceId,
|
|
2041
|
-
project_id: project.id,
|
|
2042
|
-
rights: project.rights,
|
|
2043
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
2044
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
2045
|
-
});
|
|
2046
|
-
});
|
|
2047
2042
|
if (recordsToInsert.length > 0) {
|
|
2048
2043
|
await db3.from("rbac").insert(recordsToInsert);
|
|
2049
2044
|
}
|
|
@@ -2057,7 +2052,7 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2057
2052
|
if (user.super_admin === true) {
|
|
2058
2053
|
return true;
|
|
2059
2054
|
}
|
|
2060
|
-
if (!user.super_admin && (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write") && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && user.role.evals === "write"))) {
|
|
2055
|
+
if (!user.super_admin && (table.name.plural === "agents" || table.name.plural === "workflow_templates" || table.name.plural === "variables" || table.name.plural === "users" || table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write") && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && user.role.evals === "write"))) {
|
|
2061
2056
|
console.error("Access control error: no role found for current user or no access to entity type.");
|
|
2062
2057
|
throw new Error("Access control error: no role found for current user or no access to entity type.");
|
|
2063
2058
|
}
|
|
@@ -2111,6 +2106,61 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2111
2106
|
}
|
|
2112
2107
|
};
|
|
2113
2108
|
const mutations = {
|
|
2109
|
+
[`${tableNamePlural}CopyOneById`]: async (_, args, context, info) => {
|
|
2110
|
+
const { db: db3 } = context;
|
|
2111
|
+
const requestedFields = getRequestedFields(info);
|
|
2112
|
+
let { id } = args;
|
|
2113
|
+
if (!id) {
|
|
2114
|
+
throw new Error("ID is required for copying a record.");
|
|
2115
|
+
}
|
|
2116
|
+
await validateWriteAccess(id, context);
|
|
2117
|
+
const item = await db3.from(tableNamePlural).select("*").where({ id }).first();
|
|
2118
|
+
if (!item) {
|
|
2119
|
+
throw new Error("Record not found");
|
|
2120
|
+
}
|
|
2121
|
+
if (item.rights_mode) {
|
|
2122
|
+
item.rights_mode = "private";
|
|
2123
|
+
}
|
|
2124
|
+
if (item.created_at) {
|
|
2125
|
+
item.created_at = /* @__PURE__ */ new Date();
|
|
2126
|
+
}
|
|
2127
|
+
if (item.createdAt) {
|
|
2128
|
+
item.createdAt = /* @__PURE__ */ new Date();
|
|
2129
|
+
}
|
|
2130
|
+
if (item.updated_at) {
|
|
2131
|
+
item.updated_at = /* @__PURE__ */ new Date();
|
|
2132
|
+
}
|
|
2133
|
+
if (item.updatedAt) {
|
|
2134
|
+
item.updatedAt = /* @__PURE__ */ new Date();
|
|
2135
|
+
}
|
|
2136
|
+
if (item.created_by) {
|
|
2137
|
+
item.created_by = context.user.id;
|
|
2138
|
+
}
|
|
2139
|
+
if (item.createdBy) {
|
|
2140
|
+
item.createdBy = context.user.id;
|
|
2141
|
+
}
|
|
2142
|
+
if (item.name) {
|
|
2143
|
+
item.name = item.name + " (Copy)";
|
|
2144
|
+
}
|
|
2145
|
+
Object.keys(item).forEach((key) => {
|
|
2146
|
+
if (table.fields.find((field) => field.name === key)?.type === "json") {
|
|
2147
|
+
if (typeof item[key] === "object" || Array.isArray(item[key])) {
|
|
2148
|
+
item[key] = JSON.stringify(item[key]);
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
});
|
|
2152
|
+
const insert = db3(tableNamePlural).insert({
|
|
2153
|
+
...item,
|
|
2154
|
+
id: db3.fn.uuid()
|
|
2155
|
+
}).returning("*");
|
|
2156
|
+
const result = await insert;
|
|
2157
|
+
if (!result[0]) {
|
|
2158
|
+
throw new Error("Failed to copy record.");
|
|
2159
|
+
}
|
|
2160
|
+
return {
|
|
2161
|
+
item: finalizeRequestedFields({ args, table, requestedFields, agents, contexts, tools, result: result[0], user: context.user })
|
|
2162
|
+
};
|
|
2163
|
+
},
|
|
2114
2164
|
[`${tableNamePlural}CreateOne`]: async (_, args, context, info) => {
|
|
2115
2165
|
const { db: db3 } = context;
|
|
2116
2166
|
const requestedFields = getRequestedFields(info);
|
|
@@ -2368,7 +2418,7 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2368
2418
|
}
|
|
2369
2419
|
const { limit = 10, filters = [], sort } = args;
|
|
2370
2420
|
const { db: db3 } = context;
|
|
2371
|
-
const { items } = await
|
|
2421
|
+
const { items } = await itemsPaginationRequest({
|
|
2372
2422
|
db: db3,
|
|
2373
2423
|
limit,
|
|
2374
2424
|
page: 0,
|
|
@@ -2467,9 +2517,6 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2467
2517
|
};
|
|
2468
2518
|
};
|
|
2469
2519
|
mutations[`${tableNameSingular}GenerateChunks`] = async (_, args, context, info) => {
|
|
2470
|
-
if (!context.user?.super_admin) {
|
|
2471
|
-
throw new Error("You are not authorized to generate chunks via API, user must be super admin.");
|
|
2472
|
-
}
|
|
2473
2520
|
const { db: db3 } = await postgresClient();
|
|
2474
2521
|
const exists = contexts.find((context2) => context2.id === table.id);
|
|
2475
2522
|
if (!exists) {
|
|
@@ -2480,13 +2527,17 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2480
2527
|
const columns = await db3(mainTable).columnInfo();
|
|
2481
2528
|
let query = db3.from(mainTable).select(Object.keys(columns));
|
|
2482
2529
|
if (!args.where) {
|
|
2530
|
+
if (!context.user?.super_admin) {
|
|
2531
|
+
throw new Error("You are not authorized to generate all chunks via API, user must be super admin.");
|
|
2532
|
+
}
|
|
2483
2533
|
const {
|
|
2484
2534
|
jobs: jobs2,
|
|
2485
2535
|
items: items2
|
|
2486
2536
|
} = await embeddings.generate.all(
|
|
2487
2537
|
config,
|
|
2488
2538
|
context.user.id,
|
|
2489
|
-
context.user.role?.id
|
|
2539
|
+
context.user.role?.id,
|
|
2540
|
+
args.limit
|
|
2490
2541
|
);
|
|
2491
2542
|
return {
|
|
2492
2543
|
message: "Chunks generated successfully.",
|
|
@@ -2495,6 +2546,9 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2495
2546
|
};
|
|
2496
2547
|
}
|
|
2497
2548
|
query = applyFilters(query, args.where, table);
|
|
2549
|
+
if (args.limit) {
|
|
2550
|
+
query = query.limit(args.limit);
|
|
2551
|
+
}
|
|
2498
2552
|
const items = await query;
|
|
2499
2553
|
if (items.length === 0) {
|
|
2500
2554
|
throw new Error("No items found to generate chunks for.");
|
|
@@ -2519,9 +2573,6 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2519
2573
|
};
|
|
2520
2574
|
};
|
|
2521
2575
|
mutations[`${tableNameSingular}DeleteChunks`] = async (_, args, context, info) => {
|
|
2522
|
-
if (!context.user?.super_admin) {
|
|
2523
|
-
throw new Error("You are not authorized to delete chunks via API, user must be super admin.");
|
|
2524
|
-
}
|
|
2525
2576
|
const { db: db3 } = await postgresClient();
|
|
2526
2577
|
const id = contexts.find((context2) => context2.id === table.id)?.id;
|
|
2527
2578
|
if (!id) {
|
|
@@ -2530,6 +2581,10 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2530
2581
|
if (args.where) {
|
|
2531
2582
|
let query = db3.from(getTableName(id)).select("id");
|
|
2532
2583
|
query = applyFilters(query, args.where, table);
|
|
2584
|
+
query = applyAccessControl(table, query, context.user);
|
|
2585
|
+
if (args.limit) {
|
|
2586
|
+
query = query.limit(args.limit);
|
|
2587
|
+
}
|
|
2533
2588
|
const items = await query;
|
|
2534
2589
|
if (items.length === 0) {
|
|
2535
2590
|
throw new Error("No items found to delete chunks for.");
|
|
@@ -2543,11 +2598,20 @@ function createMutations(table, agents, contexts, tools, config) {
|
|
|
2543
2598
|
jobs: []
|
|
2544
2599
|
};
|
|
2545
2600
|
} else {
|
|
2546
|
-
|
|
2547
|
-
|
|
2601
|
+
if (!context.user?.super_admin) {
|
|
2602
|
+
throw new Error("You are not authorized to delete all chunks via API, user must be super admin.");
|
|
2603
|
+
}
|
|
2604
|
+
let count = 0;
|
|
2605
|
+
if (!args.limit) {
|
|
2606
|
+
const result = await db3.from(getChunksTableName(id)).count();
|
|
2607
|
+
count = parseInt(result[0].count);
|
|
2608
|
+
await db3.from(getChunksTableName(id)).truncate();
|
|
2609
|
+
} else {
|
|
2610
|
+
count = await db3.from(getChunksTableName(id)).limit(args.limit).delete();
|
|
2611
|
+
}
|
|
2548
2612
|
return {
|
|
2549
2613
|
message: "Chunks deleted successfully.",
|
|
2550
|
-
items:
|
|
2614
|
+
items: count,
|
|
2551
2615
|
jobs: []
|
|
2552
2616
|
};
|
|
2553
2617
|
}
|
|
@@ -2562,8 +2626,8 @@ var applyAccessControl = (table, query, user, field_prefix) => {
|
|
|
2562
2626
|
}
|
|
2563
2627
|
console.log("[EXULU] user.role", user?.role);
|
|
2564
2628
|
console.log("[EXULU] table.name.plural", table.name.plural);
|
|
2565
|
-
if (user && !user?.super_admin && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
|
|
2566
|
-
console.error("==== Access control error: no role found or no access to entity type. ====");
|
|
2629
|
+
if (user && !user?.super_admin && (table.name.plural === "agents" || table.name.plural === "workflow_templates" || table.name.plural === "variables" || table.name.plural === "users" || table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
|
|
2630
|
+
console.error("==== Access control error: no role found or no access to entity type. ====", user, table.name.plural);
|
|
2567
2631
|
throw new Error("Access control error: no role found or no access to entity type.");
|
|
2568
2632
|
}
|
|
2569
2633
|
const hasRBAC = table.RBAC === true;
|
|
@@ -2606,6 +2670,7 @@ var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) =
|
|
|
2606
2670
|
const isJsonField = field?.type === "json";
|
|
2607
2671
|
const prefix = field_prefix ? field_prefix + "." : "";
|
|
2608
2672
|
fieldName = prefix + fieldName;
|
|
2673
|
+
console.log("[EXULU] operators", operators);
|
|
2609
2674
|
if (operators.eq !== void 0) {
|
|
2610
2675
|
if (isJsonField) {
|
|
2611
2676
|
query = query.whereRaw(`?? = ?::jsonb`, [fieldName, JSON.stringify(operators.eq)]);
|
|
@@ -2637,7 +2702,13 @@ var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) =
|
|
|
2637
2702
|
}
|
|
2638
2703
|
}
|
|
2639
2704
|
if (operators.lte !== void 0) {
|
|
2640
|
-
|
|
2705
|
+
console.log("[EXULU] operators.lte", operators.lte);
|
|
2706
|
+
console.log("[EXULU] fieldName", fieldName);
|
|
2707
|
+
if (operators.lte === 0 || operators.lte === "0") {
|
|
2708
|
+
query = query.whereNull(fieldName).orWhere(fieldName, "=", 0);
|
|
2709
|
+
} else {
|
|
2710
|
+
query = query.where(fieldName, "<=", operators.lte);
|
|
2711
|
+
}
|
|
2641
2712
|
}
|
|
2642
2713
|
if (operators.gte !== void 0) {
|
|
2643
2714
|
query = query.where(fieldName, ">=", operators.gte);
|
|
@@ -2653,7 +2724,8 @@ var backendAgentFields = [
|
|
|
2653
2724
|
"capabilities",
|
|
2654
2725
|
"maxContextLength",
|
|
2655
2726
|
"provider",
|
|
2656
|
-
"authenticationInformation"
|
|
2727
|
+
"authenticationInformation",
|
|
2728
|
+
"systemInstructions"
|
|
2657
2729
|
];
|
|
2658
2730
|
var removeAgentFields = (requestedFields) => {
|
|
2659
2731
|
const filtered = requestedFields.filter((field) => !backendAgentFields.includes(field));
|
|
@@ -2738,6 +2810,9 @@ var addAgentFields = async (args, requestedFields, agents, result, tools, user,
|
|
|
2738
2810
|
if (requestedFields.includes("provider")) {
|
|
2739
2811
|
result.provider = backend?.provider || "";
|
|
2740
2812
|
}
|
|
2813
|
+
if (requestedFields.includes("systemInstructions")) {
|
|
2814
|
+
result.systemInstructions = backend?.config?.instructions || void 0;
|
|
2815
|
+
}
|
|
2741
2816
|
if (!requestedFields.includes("backend")) {
|
|
2742
2817
|
delete result.backend;
|
|
2743
2818
|
}
|
|
@@ -2785,13 +2860,16 @@ var postprocessUpdate = async ({
|
|
|
2785
2860
|
if (!context.embedder) {
|
|
2786
2861
|
return result;
|
|
2787
2862
|
}
|
|
2788
|
-
const { db: db3 } = await postgresClient();
|
|
2789
|
-
console.log("[EXULU] Deleting chunks for item", result.id);
|
|
2790
|
-
await db3.from(getChunksTableName(context.id)).where({ source: result.id }).delete();
|
|
2791
|
-
console.log("[EXULU] Deleted chunks for item", result.id);
|
|
2792
|
-
console.log("[EXULU] Embedder", context.embedder);
|
|
2793
|
-
console.log("[EXULU] Configuration", context.configuration);
|
|
2794
2863
|
if (context.embedder && (context.configuration.calculateVectors === "onUpdate" || context.configuration.calculateVectors === "always")) {
|
|
2864
|
+
const { db: db3 } = await postgresClient();
|
|
2865
|
+
console.log("[EXULU] Deleting chunks for item", result.id);
|
|
2866
|
+
const exists = await context.chunksTableExists();
|
|
2867
|
+
if (exists) {
|
|
2868
|
+
await db3.from(getChunksTableName(context.id)).where({ source: result.id }).delete();
|
|
2869
|
+
console.log("[EXULU] Deleted chunks for item", result.id);
|
|
2870
|
+
}
|
|
2871
|
+
console.log("[EXULU] Embedder", context.embedder);
|
|
2872
|
+
console.log("[EXULU] Configuration", context.configuration);
|
|
2795
2873
|
console.log("[EXULU] Generating embeddings for item", result.id);
|
|
2796
2874
|
const { job } = await context.embeddings.generate.one({
|
|
2797
2875
|
item: result,
|
|
@@ -2823,7 +2901,14 @@ var postprocessDeletion = async ({
|
|
|
2823
2901
|
}
|
|
2824
2902
|
if (Array.isArray(result)) {
|
|
2825
2903
|
result = result.map((item) => {
|
|
2826
|
-
return postprocessDeletion({
|
|
2904
|
+
return postprocessDeletion({
|
|
2905
|
+
table,
|
|
2906
|
+
requestedFields,
|
|
2907
|
+
agents,
|
|
2908
|
+
contexts,
|
|
2909
|
+
tools,
|
|
2910
|
+
result: item
|
|
2911
|
+
});
|
|
2827
2912
|
});
|
|
2828
2913
|
} else {
|
|
2829
2914
|
if (table.type === "items") {
|
|
@@ -2852,6 +2937,14 @@ var postprocessDeletion = async ({
|
|
|
2852
2937
|
const { db: db3 } = await postgresClient();
|
|
2853
2938
|
await db3.from("agent_messages").where({ session: result.id }).where({ session: result.id }).delete();
|
|
2854
2939
|
}
|
|
2940
|
+
if (table.type === "eval_runs") {
|
|
2941
|
+
if (!result.id) {
|
|
2942
|
+
return result;
|
|
2943
|
+
}
|
|
2944
|
+
const { db: db3 } = await postgresClient();
|
|
2945
|
+
await db3.from("job_results").where({ label: { contains: result.id } }).del();
|
|
2946
|
+
await db3.from("eval_runs").where({ id: result.id }).del();
|
|
2947
|
+
}
|
|
2855
2948
|
}
|
|
2856
2949
|
return result;
|
|
2857
2950
|
};
|
|
@@ -2962,7 +3055,7 @@ var applySorting = (query, sort, field_prefix) => {
|
|
|
2962
3055
|
}
|
|
2963
3056
|
return query;
|
|
2964
3057
|
};
|
|
2965
|
-
var
|
|
3058
|
+
var itemsPaginationRequest = async ({
|
|
2966
3059
|
db: db3,
|
|
2967
3060
|
limit,
|
|
2968
3061
|
page,
|
|
@@ -2992,7 +3085,8 @@ var paginationRequest = async ({
|
|
|
2992
3085
|
if (page > 1) {
|
|
2993
3086
|
dataQuery = dataQuery.offset((page - 1) * limit);
|
|
2994
3087
|
}
|
|
2995
|
-
|
|
3088
|
+
dataQuery = dataQuery.select(fields ? fields : "*").limit(limit);
|
|
3089
|
+
let items = await dataQuery;
|
|
2996
3090
|
return {
|
|
2997
3091
|
items,
|
|
2998
3092
|
pageInfo: {
|
|
@@ -3043,7 +3137,7 @@ function createQueries(table, agents, tools, contexts) {
|
|
|
3043
3137
|
const { limit = 10, page = 0, filters = [], sort } = args;
|
|
3044
3138
|
const requestedFields = getRequestedFields(info);
|
|
3045
3139
|
const sanitizedFields = sanitizeRequestedFields(table, requestedFields);
|
|
3046
|
-
const { items, pageInfo } = await
|
|
3140
|
+
const { items, pageInfo } = await itemsPaginationRequest({
|
|
3047
3141
|
db: db3,
|
|
3048
3142
|
limit,
|
|
3049
3143
|
page,
|
|
@@ -3118,6 +3212,46 @@ function createQueries(table, agents, tools, contexts) {
|
|
|
3118
3212
|
expand: args.expand
|
|
3119
3213
|
});
|
|
3120
3214
|
};
|
|
3215
|
+
queries[`${tableNameSingular}ChunkById`] = async (_, args, context, info) => {
|
|
3216
|
+
const exists = contexts.find((ctx) => ctx.id === table.id);
|
|
3217
|
+
if (!exists) {
|
|
3218
|
+
throw new Error("Context " + table.id + " not found in registry.");
|
|
3219
|
+
}
|
|
3220
|
+
const { db: db3 } = context;
|
|
3221
|
+
const chunksTable = getChunksTableName(exists.id);
|
|
3222
|
+
const mainTable = getTableName(exists.id);
|
|
3223
|
+
const chunk = await db3(chunksTable + " as chunks").select([
|
|
3224
|
+
"chunks.id as chunk_id",
|
|
3225
|
+
"chunks.source as chunk_source",
|
|
3226
|
+
"chunks.content as chunk_content",
|
|
3227
|
+
"chunks.chunk_index",
|
|
3228
|
+
"chunks.metadata as chunk_metadata",
|
|
3229
|
+
db3.raw('chunks."createdAt" as chunk_created_at'),
|
|
3230
|
+
db3.raw('chunks."updatedAt" as chunk_updated_at'),
|
|
3231
|
+
"items.id as item_id",
|
|
3232
|
+
"items.name as item_name",
|
|
3233
|
+
"items.external_id as item_external_id",
|
|
3234
|
+
db3.raw('items."updatedAt" as item_updated_at'),
|
|
3235
|
+
db3.raw('items."createdAt" as item_created_at')
|
|
3236
|
+
]).leftJoin(mainTable + " as items", "chunks.source", "items.id").where("chunks.id", args.id).first();
|
|
3237
|
+
if (!chunk) {
|
|
3238
|
+
return null;
|
|
3239
|
+
}
|
|
3240
|
+
return {
|
|
3241
|
+
chunk_content: chunk.chunk_content,
|
|
3242
|
+
chunk_index: chunk.chunk_index,
|
|
3243
|
+
chunk_id: chunk.chunk_id,
|
|
3244
|
+
chunk_source: chunk.chunk_source,
|
|
3245
|
+
chunk_metadata: chunk.chunk_metadata,
|
|
3246
|
+
chunk_created_at: chunk.chunk_created_at,
|
|
3247
|
+
chunk_updated_at: chunk.chunk_updated_at,
|
|
3248
|
+
item_id: chunk.item_id,
|
|
3249
|
+
item_name: chunk.item_name,
|
|
3250
|
+
item_external_id: chunk.item_external_id,
|
|
3251
|
+
item_updated_at: chunk.item_updated_at,
|
|
3252
|
+
item_created_at: chunk.item_created_at
|
|
3253
|
+
};
|
|
3254
|
+
};
|
|
3121
3255
|
}
|
|
3122
3256
|
return queries;
|
|
3123
3257
|
}
|
|
@@ -3250,120 +3384,52 @@ var vectorSearch = async ({
|
|
|
3250
3384
|
const fullTextWeight = 2;
|
|
3251
3385
|
const semanticWeight = 1;
|
|
3252
3386
|
const rrfK = 50;
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
(1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance,
|
|
3300
|
-
|
|
3301
|
-
/* Hybrid RRF score */
|
|
3302
|
-
(
|
|
3303
|
-
COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
|
|
3304
|
-
+
|
|
3305
|
-
COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
|
|
3306
|
-
)::float AS hybrid_score
|
|
3307
|
-
|
|
3308
|
-
FROM full_text ft
|
|
3309
|
-
FULL OUTER JOIN semantic se
|
|
3310
|
-
ON ft.id = se.id
|
|
3311
|
-
JOIN ${chunksTable} as chunks
|
|
3312
|
-
ON COALESCE(ft.id, se.id) = chunks.id
|
|
3313
|
-
JOIN ${mainTable} as items
|
|
3314
|
-
ON items.id = chunks.source
|
|
3315
|
-
WHERE (
|
|
3316
|
-
COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
|
|
3317
|
-
+
|
|
3318
|
-
COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
|
|
3319
|
-
) >= ?
|
|
3320
|
-
AND (chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)
|
|
3321
|
-
AND (chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)
|
|
3322
|
-
ORDER BY hybrid_score DESC
|
|
3323
|
-
LIMIT LEAST(?, 250)
|
|
3324
|
-
OFFSET 0
|
|
3325
|
-
`;
|
|
3326
|
-
const bindings = [
|
|
3327
|
-
// full_text: plainto_tsquery(lang, query) in rank and where
|
|
3328
|
-
language,
|
|
3329
|
-
query,
|
|
3330
|
-
language,
|
|
3331
|
-
query,
|
|
3332
|
-
language,
|
|
3333
|
-
query,
|
|
3334
|
-
cutoffs?.tsvector || 0,
|
|
3335
|
-
// full_text tsvector cutoff
|
|
3336
|
-
matchCount,
|
|
3337
|
-
// full_text limit
|
|
3338
|
-
cutoffs?.cosineDistance || 0,
|
|
3339
|
-
// semantic cosine distance cutoff
|
|
3340
|
-
matchCount,
|
|
3341
|
-
// semantic limit
|
|
3342
|
-
// fts_rank (ts_rank) call
|
|
3343
|
-
language,
|
|
3344
|
-
query,
|
|
3345
|
-
// RRF fusion parameters
|
|
3346
|
-
rrfK,
|
|
3347
|
-
fullTextWeight,
|
|
3348
|
-
rrfK,
|
|
3349
|
-
semanticWeight,
|
|
3350
|
-
// WHERE clause hybrid_score filter
|
|
3351
|
-
rrfK,
|
|
3352
|
-
fullTextWeight,
|
|
3353
|
-
rrfK,
|
|
3354
|
-
semanticWeight,
|
|
3355
|
-
cutoffs?.hybrid || 0,
|
|
3356
|
-
// Additional cutoff filters in main WHERE clause
|
|
3357
|
-
language,
|
|
3358
|
-
query,
|
|
3359
|
-
cutoffs?.tsvector || 0,
|
|
3360
|
-
// tsvector cutoff for results from semantic CTE
|
|
3361
|
-
cutoffs?.cosineDistance || 0,
|
|
3362
|
-
// cosine distance cutoff for results from full_text CTE
|
|
3363
|
-
matchCount
|
|
3364
|
-
// final limit
|
|
3365
|
-
];
|
|
3366
|
-
resultChunks = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
|
|
3387
|
+
let fullTextQuery = db3(chunksTable + " as chunks").select([
|
|
3388
|
+
"chunks.id",
|
|
3389
|
+
"chunks.source",
|
|
3390
|
+
db3.raw(`row_number() OVER (ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC) AS rank_ix`, [language, query])
|
|
3391
|
+
]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereRaw(`chunks.fts @@ plainto_tsquery(?, ?)`, [language, query]).whereRaw(`ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?`, [language, query, cutoffs?.tsvector || 0]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
|
|
3392
|
+
fullTextQuery = applyFilters(fullTextQuery, filters, table, "items");
|
|
3393
|
+
fullTextQuery = applyAccessControl(table, fullTextQuery, user, "items");
|
|
3394
|
+
let semanticQuery = db3(chunksTable + " as chunks").select([
|
|
3395
|
+
"chunks.id",
|
|
3396
|
+
"chunks.source",
|
|
3397
|
+
db3.raw(`row_number() OVER (ORDER BY chunks.embedding <=> ${vectorExpr} ASC) AS rank_ix`)
|
|
3398
|
+
]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereNotNull("chunks.embedding").whereRaw(`(1 - (chunks.embedding <=> ${vectorExpr})) >= ?`, [cutoffs?.cosineDistance || 0]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
|
|
3399
|
+
semanticQuery = applyFilters(semanticQuery, filters, table, "items");
|
|
3400
|
+
semanticQuery = applyAccessControl(table, semanticQuery, user, "items");
|
|
3401
|
+
let hybridQuery = db3.with("full_text", fullTextQuery).with("semantic", semanticQuery).select([
|
|
3402
|
+
"items.id as item_id",
|
|
3403
|
+
"items.name as item_name",
|
|
3404
|
+
"items.external_id as item_external_id",
|
|
3405
|
+
"chunks.id as chunk_id",
|
|
3406
|
+
"chunks.source",
|
|
3407
|
+
"chunks.content",
|
|
3408
|
+
"chunks.chunk_index",
|
|
3409
|
+
"chunks.metadata",
|
|
3410
|
+
db3.raw('chunks."createdAt" as chunk_created_at'),
|
|
3411
|
+
db3.raw('chunks."updatedAt" as chunk_updated_at'),
|
|
3412
|
+
db3.raw('items."updatedAt" as item_updated_at'),
|
|
3413
|
+
db3.raw('items."createdAt" as item_created_at'),
|
|
3414
|
+
db3.raw(`ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank`, [language, query]),
|
|
3415
|
+
db3.raw(`(1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance`),
|
|
3416
|
+
db3.raw(`
|
|
3417
|
+
(
|
|
3418
|
+
COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
|
|
3419
|
+
+
|
|
3420
|
+
COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
|
|
3421
|
+
)::float AS hybrid_score
|
|
3422
|
+
`, [rrfK, fullTextWeight, rrfK, semanticWeight])
|
|
3423
|
+
]).from("full_text as ft").fullOuterJoin("semantic as se", "ft.id", "se.id").join(chunksTable + " as chunks", function() {
|
|
3424
|
+
this.on(db3.raw("COALESCE(ft.id, se.id)"), "=", "chunks.id");
|
|
3425
|
+
}).join(mainTable + " as items", "items.id", "chunks.source").whereRaw(`
|
|
3426
|
+
(
|
|
3427
|
+
COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
|
|
3428
|
+
+
|
|
3429
|
+
COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
|
|
3430
|
+
) >= ?
|
|
3431
|
+
`, [rrfK, fullTextWeight, rrfK, semanticWeight, cutoffs?.hybrid || 0]).whereRaw(`(chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)`, [language, query, cutoffs?.tsvector || 0]).whereRaw(`(chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)`, [cutoffs?.cosineDistance || 0]).orderByRaw("hybrid_score DESC").limit(Math.min(matchCount, 250));
|
|
3432
|
+
resultChunks = await hybridQuery;
|
|
3367
3433
|
}
|
|
3368
3434
|
console.log("[EXULU] Vector search chunk results:", resultChunks?.length);
|
|
3369
3435
|
let results = resultChunks.map((chunk) => ({
|
|
@@ -3599,8 +3665,8 @@ var contextToTableDefinition = (context) => {
|
|
|
3599
3665
|
type: "text"
|
|
3600
3666
|
});
|
|
3601
3667
|
definition.fields.push({
|
|
3602
|
-
name: "
|
|
3603
|
-
type: "
|
|
3668
|
+
name: "chunks_count",
|
|
3669
|
+
type: "number"
|
|
3604
3670
|
});
|
|
3605
3671
|
definition.fields.push({
|
|
3606
3672
|
name: "name",
|
|
@@ -3703,10 +3769,13 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
|
|
|
3703
3769
|
if (table.type === "items") {
|
|
3704
3770
|
typeDefs += `
|
|
3705
3771
|
${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}], cutoffs: SearchCutoffs, expand: SearchExpand): ${tableNameSingular}VectorSearchResult
|
|
3772
|
+
${tableNameSingular}ChunkById(id: ID!): ${tableNameSingular}VectorSearchChunk
|
|
3706
3773
|
`;
|
|
3707
3774
|
}
|
|
3708
3775
|
mutationDefs += `
|
|
3709
3776
|
${tableNamePlural}CreateOne(input: ${tableNameSingular}Input!, upsert: Boolean): ${tableNameSingular}MutationPayload
|
|
3777
|
+
${tableNamePlural}CopyOneById(id: ID!): ${tableNameSingular}MutationPayload
|
|
3778
|
+
|
|
3710
3779
|
${tableNamePlural}UpdateOne(where: [Filter${tableNameSingularUpperCaseFirst}], input: ${tableNameSingular}Input!): ${tableNameSingular}MutationPayload
|
|
3711
3780
|
${tableNamePlural}UpdateOneById(id: ID!, input: ${tableNameSingular}Input!): ${tableNameSingular}MutationPayload
|
|
3712
3781
|
${tableNamePlural}RemoveOneById(id: ID!): ${tableNameSingular}
|
|
@@ -3714,9 +3783,9 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
|
|
|
3714
3783
|
`;
|
|
3715
3784
|
if (table.type === "items") {
|
|
3716
3785
|
mutationDefs += `
|
|
3717
|
-
${tableNameSingular}GenerateChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}GenerateChunksReturnPayload
|
|
3786
|
+
${tableNameSingular}GenerateChunks(where: [Filter${tableNameSingularUpperCaseFirst}], limit: Int): ${tableNameSingular}GenerateChunksReturnPayload
|
|
3718
3787
|
${tableNameSingular}ExecuteSource(source: ID!, inputs: JSON!): ${tableNameSingular}ExecuteSourceReturnPayload
|
|
3719
|
-
${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}DeleteChunksReturnPayload
|
|
3788
|
+
${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}], limit: Int): ${tableNameSingular}DeleteChunksReturnPayload
|
|
3720
3789
|
`;
|
|
3721
3790
|
if (table.processor) {
|
|
3722
3791
|
mutationDefs += `
|
|
@@ -3850,8 +3919,11 @@ type PageInfo {
|
|
|
3850
3919
|
typeDefs += `
|
|
3851
3920
|
contextById(id: ID!): Context
|
|
3852
3921
|
`;
|
|
3922
|
+
typeDefs += `
|
|
3923
|
+
getUniquePromptTags: [String!]!
|
|
3924
|
+
`;
|
|
3853
3925
|
mutationDefs += `
|
|
3854
|
-
runEval(id: ID!,
|
|
3926
|
+
runEval(id: ID!, cases: [ID!]): RunEvalReturnPayload
|
|
3855
3927
|
`;
|
|
3856
3928
|
mutationDefs += `
|
|
3857
3929
|
drainQueue(queue: QueueEnum!): JobActionReturnPayload
|
|
@@ -4307,6 +4379,38 @@ type PageInfo {
|
|
|
4307
4379
|
array.push("agents");
|
|
4308
4380
|
return [...new Set(array)].sort();
|
|
4309
4381
|
};
|
|
4382
|
+
resolvers.Query["getUniquePromptTags"] = async (_, args, context, info) => {
|
|
4383
|
+
const { db: db3 } = context;
|
|
4384
|
+
const user = context.user;
|
|
4385
|
+
const promptTable = tables.find((t) => t.name.plural === "prompt_library");
|
|
4386
|
+
if (!promptTable) {
|
|
4387
|
+
throw new Error("Prompt library table not found");
|
|
4388
|
+
}
|
|
4389
|
+
let query = db3.from("prompt_library").select("tags");
|
|
4390
|
+
query = applyAccessControl(promptTable, query, user);
|
|
4391
|
+
const results = await query;
|
|
4392
|
+
const allTags = [];
|
|
4393
|
+
for (const row of results) {
|
|
4394
|
+
if (row.tags) {
|
|
4395
|
+
let tags = [];
|
|
4396
|
+
if (typeof row.tags === "string") {
|
|
4397
|
+
try {
|
|
4398
|
+
tags = JSON.parse(row.tags);
|
|
4399
|
+
} catch (e) {
|
|
4400
|
+
tags = [row.tags];
|
|
4401
|
+
}
|
|
4402
|
+
} else if (Array.isArray(row.tags)) {
|
|
4403
|
+
tags = row.tags;
|
|
4404
|
+
}
|
|
4405
|
+
tags.forEach((tag) => {
|
|
4406
|
+
if (tag && typeof tag === "string" && tag.trim()) {
|
|
4407
|
+
allTags.push(tag.trim().toLowerCase());
|
|
4408
|
+
}
|
|
4409
|
+
});
|
|
4410
|
+
}
|
|
4411
|
+
}
|
|
4412
|
+
return [...new Set(allTags)].sort();
|
|
4413
|
+
};
|
|
4310
4414
|
modelDefs += `
|
|
4311
4415
|
type ProviderPaginationResult {
|
|
4312
4416
|
items: [Provider]!
|
|
@@ -4697,14 +4801,16 @@ var addBucketPrefixToKey = (key, bucket) => {
|
|
|
4697
4801
|
}
|
|
4698
4802
|
return `${bucket}/${key}`;
|
|
4699
4803
|
};
|
|
4700
|
-
var uploadFile = async (file, fileName, config, options = {}, user, customBucket) => {
|
|
4804
|
+
var uploadFile = async (file, fileName, config, options = {}, user, customBucket, global) => {
|
|
4701
4805
|
if (!config.fileUploads) {
|
|
4702
4806
|
throw new Error("File uploads are not configured (in the exported uploadFile function)");
|
|
4703
4807
|
}
|
|
4704
4808
|
const client2 = getS3Client(config);
|
|
4705
4809
|
let defaultBucket = config.fileUploads.s3Bucket;
|
|
4706
4810
|
let key = fileName;
|
|
4707
|
-
|
|
4811
|
+
if (!global) {
|
|
4812
|
+
key = addUserPrefixToKey(key, user || "api");
|
|
4813
|
+
}
|
|
4708
4814
|
key = addGeneralPrefixToKey(key, config);
|
|
4709
4815
|
const sanitizedMetadata = sanitizeMetadata(options.metadata);
|
|
4710
4816
|
const command = new PutObjectCommand({
|
|
@@ -4855,7 +4961,7 @@ var createUppyRoutes = async (app, contexts, config) => {
|
|
|
4855
4961
|
return;
|
|
4856
4962
|
}
|
|
4857
4963
|
let allowed = false;
|
|
4858
|
-
if (user.type === "api" || user.super_admin || key.includes(`user_${user.id}/`)) {
|
|
4964
|
+
if (user.type === "api" || user.super_admin || !key.includes(`user_`) || key.includes(`user_${user.id}/`)) {
|
|
4859
4965
|
allowed = true;
|
|
4860
4966
|
}
|
|
4861
4967
|
if (!allowed) {
|
|
@@ -4933,9 +5039,16 @@ var createUppyRoutes = async (app, contexts, config) => {
|
|
|
4933
5039
|
return;
|
|
4934
5040
|
}
|
|
4935
5041
|
const client2 = getS3Client(config);
|
|
5042
|
+
let prefix = `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}`;
|
|
5043
|
+
if (!req.headers.global) {
|
|
5044
|
+
prefix += `user_${authenticationResult.user.id}`;
|
|
5045
|
+
} else {
|
|
5046
|
+
prefix += "global";
|
|
5047
|
+
}
|
|
5048
|
+
console.log("[EXULU] prefix", prefix);
|
|
4936
5049
|
const command = new ListObjectsV2Command({
|
|
4937
5050
|
Bucket: config.fileUploads.s3Bucket,
|
|
4938
|
-
Prefix:
|
|
5051
|
+
Prefix: prefix,
|
|
4939
5052
|
MaxKeys: 9,
|
|
4940
5053
|
...req.query.continuationToken && { ContinuationToken: req.query.continuationToken }
|
|
4941
5054
|
});
|
|
@@ -5018,7 +5131,13 @@ var createUppyRoutes = async (app, contexts, config) => {
|
|
|
5018
5131
|
const { filename, contentType } = extractFileParameters(req);
|
|
5019
5132
|
validateFileParameters(filename, contentType);
|
|
5020
5133
|
const key = generateS3Key2(filename);
|
|
5021
|
-
let fullKey =
|
|
5134
|
+
let fullKey = key;
|
|
5135
|
+
console.log("[EXULU] global", req.headers.global);
|
|
5136
|
+
if (!req.headers.global) {
|
|
5137
|
+
fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
|
|
5138
|
+
} else {
|
|
5139
|
+
fullKey = "global/" + key;
|
|
5140
|
+
}
|
|
5022
5141
|
fullKey = addGeneralPrefixToKey(fullKey, config);
|
|
5023
5142
|
console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
|
|
5024
5143
|
getSignedUrl(
|
|
@@ -5074,7 +5193,13 @@ var createUppyRoutes = async (app, contexts, config) => {
|
|
|
5074
5193
|
return res.status(400).json({ error: "s3: content type must be a string" });
|
|
5075
5194
|
}
|
|
5076
5195
|
const key = `${randomUUID()}-_EXULU_${filename}`;
|
|
5077
|
-
let fullKey =
|
|
5196
|
+
let fullKey = key;
|
|
5197
|
+
console.log("[EXULU] global", req.headers.global);
|
|
5198
|
+
if (!req.headers.global) {
|
|
5199
|
+
fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
|
|
5200
|
+
} else {
|
|
5201
|
+
fullKey = "global/" + key;
|
|
5202
|
+
}
|
|
5078
5203
|
fullKey = addGeneralPrefixToKey(fullKey, config);
|
|
5079
5204
|
console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
|
|
5080
5205
|
const params = {
|
|
@@ -5722,6 +5847,31 @@ var ExuluAgent2 = class {
|
|
|
5722
5847
|
they are talking with the current date in mind as a reference.`;
|
|
5723
5848
|
let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
|
|
5724
5849
|
system += "\n\n" + genericContext;
|
|
5850
|
+
const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
|
|
5851
|
+
console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
|
|
5852
|
+
console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
|
|
5853
|
+
if (includesContextSearchTool) {
|
|
5854
|
+
system += `
|
|
5855
|
+
|
|
5856
|
+
|
|
5857
|
+
|
|
5858
|
+
When you use a context search tool, you will include references to the items
|
|
5859
|
+
retrieved from the context search tool inline in the response using this exact JSON format
|
|
5860
|
+
(all on one line, no line breaks):
|
|
5861
|
+
{item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
|
|
5862
|
+
|
|
5863
|
+
IMPORTANT formatting rules:
|
|
5864
|
+
- Use the exact format shown above, all on ONE line
|
|
5865
|
+
- Do NOT use quotes around field names or values
|
|
5866
|
+
- Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
|
|
5867
|
+
- Include the file/item name, not the full path
|
|
5868
|
+
- Separate multiple citations with spaces
|
|
5869
|
+
|
|
5870
|
+
Example: {item_name: document.pdf, item_id: abc123, context: my-context-id, chunk_id: chunk_456, chunk_index: 0}
|
|
5871
|
+
|
|
5872
|
+
The citations will be rendered as interactive badges in the UI.
|
|
5873
|
+
`;
|
|
5874
|
+
}
|
|
5725
5875
|
if (prompt) {
|
|
5726
5876
|
let result = { object: null, text: "" };
|
|
5727
5877
|
let inputTokens = 0;
|
|
@@ -5756,7 +5906,7 @@ var ExuluAgent2 = class {
|
|
|
5756
5906
|
req,
|
|
5757
5907
|
project
|
|
5758
5908
|
),
|
|
5759
|
-
stopWhen: [stepCountIs(
|
|
5909
|
+
stopWhen: [stepCountIs(5)]
|
|
5760
5910
|
});
|
|
5761
5911
|
result.text = text;
|
|
5762
5912
|
inputTokens = totalUsage?.inputTokens || 0;
|
|
@@ -5817,7 +5967,7 @@ var ExuluAgent2 = class {
|
|
|
5817
5967
|
req,
|
|
5818
5968
|
project
|
|
5819
5969
|
),
|
|
5820
|
-
stopWhen: [stepCountIs(
|
|
5970
|
+
stopWhen: [stepCountIs(5)]
|
|
5821
5971
|
});
|
|
5822
5972
|
if (statistics) {
|
|
5823
5973
|
await Promise.all([
|
|
@@ -5982,6 +6132,31 @@ ${extractedText}
|
|
|
5982
6132
|
const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
|
|
5983
6133
|
let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
|
|
5984
6134
|
system += "\n\n" + genericContext;
|
|
6135
|
+
const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
|
|
6136
|
+
console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
|
|
6137
|
+
console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
|
|
6138
|
+
if (includesContextSearchTool) {
|
|
6139
|
+
system += `
|
|
6140
|
+
|
|
6141
|
+
|
|
6142
|
+
|
|
6143
|
+
When you use a context search tool, you will include references to the items
|
|
6144
|
+
retrieved from the context search tool inline in the response using this exact JSON format
|
|
6145
|
+
(all on one line, no line breaks):
|
|
6146
|
+
{item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
|
|
6147
|
+
|
|
6148
|
+
IMPORTANT formatting rules:
|
|
6149
|
+
- Use the exact format shown above, all on ONE line
|
|
6150
|
+
- Do NOT use quotes around field names or values
|
|
6151
|
+
- Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
|
|
6152
|
+
- Include the file/item name, not the full path
|
|
6153
|
+
- Separate multiple citations with spaces
|
|
6154
|
+
|
|
6155
|
+
Example: {item_name: document.pdf, item_id: abc123, context: my-context-id, chunk_id: chunk_456, chunk_index: 0}
|
|
6156
|
+
|
|
6157
|
+
The citations will be rendered as interactive badges in the UI.
|
|
6158
|
+
`;
|
|
6159
|
+
}
|
|
5985
6160
|
const result = streamText({
|
|
5986
6161
|
model,
|
|
5987
6162
|
// Should be a LanguageModelV1
|
|
@@ -6012,8 +6187,8 @@ ${extractedText}
|
|
|
6012
6187
|
onError: (error) => {
|
|
6013
6188
|
console.error("[EXULU] chat stream error.", error);
|
|
6014
6189
|
throw new Error(`Chat stream error: ${error instanceof Error ? error.message : String(error)}`);
|
|
6015
|
-
}
|
|
6016
|
-
|
|
6190
|
+
},
|
|
6191
|
+
stopWhen: [stepCountIs(5)]
|
|
6017
6192
|
});
|
|
6018
6193
|
return {
|
|
6019
6194
|
stream: result,
|
|
@@ -6525,6 +6700,7 @@ var ExuluContext = class {
|
|
|
6525
6700
|
})));
|
|
6526
6701
|
}
|
|
6527
6702
|
await db3.from(getTableName(this.id)).where({ id: item.id }).update({
|
|
6703
|
+
chunks_count: chunks?.length || 0,
|
|
6528
6704
|
embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
6529
6705
|
}).returning("id");
|
|
6530
6706
|
return {
|
|
@@ -6540,6 +6716,13 @@ var ExuluContext = class {
|
|
|
6540
6716
|
throw new Error("Item id or external id is required for upsert.");
|
|
6541
6717
|
}
|
|
6542
6718
|
const { db: db3 } = await postgresClient();
|
|
6719
|
+
Object.keys(item).forEach((key) => {
|
|
6720
|
+
if (this.fields.find((field) => field.name === key)?.type === "json") {
|
|
6721
|
+
if (typeof item[key] === "object" || Array.isArray(item[key])) {
|
|
6722
|
+
item[key] = JSON.stringify(item[key]);
|
|
6723
|
+
}
|
|
6724
|
+
}
|
|
6725
|
+
});
|
|
6543
6726
|
const mutation = db3.from(getTableName(
|
|
6544
6727
|
this.id
|
|
6545
6728
|
)).insert(
|
|
@@ -6629,6 +6812,13 @@ var ExuluContext = class {
|
|
|
6629
6812
|
if (!record) {
|
|
6630
6813
|
throw new Error("Item not found.");
|
|
6631
6814
|
}
|
|
6815
|
+
Object.keys(item).forEach((key) => {
|
|
6816
|
+
if (this.fields.find((field) => field.name === key)?.type === "json") {
|
|
6817
|
+
if (typeof item[key] === "object" || Array.isArray(item[key])) {
|
|
6818
|
+
item[key] = JSON.stringify(item[key]);
|
|
6819
|
+
}
|
|
6820
|
+
}
|
|
6821
|
+
});
|
|
6632
6822
|
const mutation = db3.from(
|
|
6633
6823
|
getTableName(this.id)
|
|
6634
6824
|
).where(
|
|
@@ -6794,9 +6984,13 @@ var ExuluContext = class {
|
|
|
6794
6984
|
trigger: trigger || "agent"
|
|
6795
6985
|
}, role, void 0);
|
|
6796
6986
|
},
|
|
6797
|
-
all: async (config, userId, roleId) => {
|
|
6987
|
+
all: async (config, userId, roleId, limit) => {
|
|
6798
6988
|
const { db: db3 } = await postgresClient();
|
|
6799
|
-
|
|
6989
|
+
let query = db3.from(getTableName(this.id)).select("*");
|
|
6990
|
+
if (limit) {
|
|
6991
|
+
query = query.limit(limit);
|
|
6992
|
+
}
|
|
6993
|
+
const items = await query;
|
|
6800
6994
|
const jobs = [];
|
|
6801
6995
|
const queue = await this.embedder?.queue;
|
|
6802
6996
|
if (!queue?.queue.name && items.length > 2e3) {
|
|
@@ -6839,9 +7033,11 @@ var ExuluContext = class {
|
|
|
6839
7033
|
table.text("created_by");
|
|
6840
7034
|
table.text("ttl");
|
|
6841
7035
|
table.text("rights_mode").defaultTo(this.configuration?.defaultRightsMode ?? "private");
|
|
7036
|
+
table.timestamp("embeddings_updated_at").defaultTo(null);
|
|
7037
|
+
table.timestamp("last_processed_at").defaultTo(null);
|
|
6842
7038
|
table.integer("textlength");
|
|
6843
7039
|
table.text("source");
|
|
6844
|
-
table.
|
|
7040
|
+
table.integer("chunks_count").defaultTo(0);
|
|
6845
7041
|
for (const field of this.fields) {
|
|
6846
7042
|
let { type, name, unique } = field;
|
|
6847
7043
|
if (!type || !name) {
|
|
@@ -7370,12 +7566,6 @@ Mood: friendly and intelligent.
|
|
|
7370
7566
|
return;
|
|
7371
7567
|
}
|
|
7372
7568
|
}
|
|
7373
|
-
if (user?.type !== "api" && !user?.super_admin && req.body.resourceId !== user?.id) {
|
|
7374
|
-
res.status(400).json({
|
|
7375
|
-
message: "The provided user id in the resourceId field is not the same as the authenticated user. Only super admins and API users can impersonate other users."
|
|
7376
|
-
});
|
|
7377
|
-
return;
|
|
7378
|
-
}
|
|
7379
7569
|
console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
|
|
7380
7570
|
const disabledTools = req.body.disabledTools ? req.body.disabledTools : [];
|
|
7381
7571
|
let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
|
|
@@ -7838,6 +8028,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
|
|
|
7838
8028
|
console.log("[EXULU] creating workers for " + queues2?.length + " queues.");
|
|
7839
8029
|
console.log("[EXULU] queues", queues2.map((q) => q.queue.name));
|
|
7840
8030
|
installGlobalErrorHandlers();
|
|
8031
|
+
process.setMaxListeners(Math.max(queues2.length * 2 + 5, 15));
|
|
7841
8032
|
if (!redisServer.host || !redisServer.port) {
|
|
7842
8033
|
console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
|
|
7843
8034
|
throw new Error("No redis server configured in the environment, so cannot start worker.");
|
|
@@ -7870,7 +8061,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
|
|
|
7870
8061
|
}));
|
|
7871
8062
|
const { db: db3 } = await postgresClient();
|
|
7872
8063
|
const data = bullmqJob.data;
|
|
7873
|
-
const timeoutInSeconds = data.timeoutInSeconds || 600;
|
|
8064
|
+
const timeoutInSeconds = data.timeoutInSeconds || queue.timeoutInSeconds || 600;
|
|
7874
8065
|
const timeoutMs = timeoutInSeconds * 1e3;
|
|
7875
8066
|
let timeoutHandle;
|
|
7876
8067
|
const timeoutPromise = new Promise((_, reject) => {
|
|
@@ -8074,7 +8265,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
|
|
|
8074
8265
|
attempts: queue2.retries || 3,
|
|
8075
8266
|
// todo make this configurable?
|
|
8076
8267
|
removeOnComplete: 5e3,
|
|
8077
|
-
removeOnFail:
|
|
8268
|
+
removeOnFail: 5e3,
|
|
8078
8269
|
backoff: queue2.backoff || {
|
|
8079
8270
|
type: "exponential",
|
|
8080
8271
|
delay: 2e3
|
|
@@ -8296,8 +8487,8 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
|
|
|
8296
8487
|
const { db: db3 } = await postgresClient();
|
|
8297
8488
|
await db3.from("job_results").where({ job_id: job.id }).update({
|
|
8298
8489
|
state: JOB_STATUS_ENUM.completed,
|
|
8299
|
-
result: returnvalue.result ? JSON.stringify(returnvalue.result) : null,
|
|
8300
|
-
metadata: returnvalue.metadata ? JSON.stringify(returnvalue.metadata) : null
|
|
8490
|
+
result: returnvalue.result != null ? JSON.stringify(returnvalue.result) : null,
|
|
8491
|
+
metadata: returnvalue.metadata != null ? JSON.stringify(returnvalue.metadata) : null
|
|
8301
8492
|
});
|
|
8302
8493
|
});
|
|
8303
8494
|
worker.on("failed", async (job, error, prev) => {
|
|
@@ -8408,9 +8599,11 @@ var pollJobResult = async ({ queue, jobId }) => {
|
|
|
8408
8599
|
console.log(`[EXULU] eval function job ${job.name} completed, getting result from database...`);
|
|
8409
8600
|
const { db: db3 } = await postgresClient();
|
|
8410
8601
|
const entry = await db3.from("job_results").where({ job_id: job.id }).first();
|
|
8602
|
+
console.log("[EXULU] eval function job ${job.name} result", entry);
|
|
8411
8603
|
result = entry?.result;
|
|
8412
8604
|
if (result === void 0 || result === null || result === "") {
|
|
8413
|
-
throw new Error(`Eval function ${job.id} result not found in database
|
|
8605
|
+
throw new Error(`Eval function ${job.id} result not found in database
|
|
8606
|
+
for job eval function job ${job.name}. Entry data from DB: ${JSON.stringify(entry)}.`);
|
|
8414
8607
|
}
|
|
8415
8608
|
console.log(`[EXULU] eval function ${job.id} result: ${result}`);
|
|
8416
8609
|
break;
|
|
@@ -9420,45 +9613,6 @@ var createLogger = ({
|
|
|
9420
9613
|
};
|
|
9421
9614
|
var logger_default = createLogger;
|
|
9422
9615
|
|
|
9423
|
-
// src/templates/contexts/code-standards.ts
|
|
9424
|
-
var codeStandardsContext = new ExuluContext({
|
|
9425
|
-
id: "code_standards",
|
|
9426
|
-
name: "Code Standards",
|
|
9427
|
-
description: "Code standards that can be used with the Exulu CLI.",
|
|
9428
|
-
configuration: {
|
|
9429
|
-
defaultRightsMode: "public"
|
|
9430
|
-
},
|
|
9431
|
-
fields: [{
|
|
9432
|
-
name: "Best practices",
|
|
9433
|
-
type: "longText"
|
|
9434
|
-
}, {
|
|
9435
|
-
name: "Code style",
|
|
9436
|
-
type: "longText"
|
|
9437
|
-
}, {
|
|
9438
|
-
name: "Tech stack",
|
|
9439
|
-
type: "longText"
|
|
9440
|
-
}],
|
|
9441
|
-
active: true
|
|
9442
|
-
});
|
|
9443
|
-
|
|
9444
|
-
// src/templates/contexts/outputs.ts
|
|
9445
|
-
var outputsContext = new ExuluContext({
|
|
9446
|
-
id: "outputs_default_context",
|
|
9447
|
-
name: "Outputs",
|
|
9448
|
-
description: "Outputs from agent sessions that you have saved for re-used later.",
|
|
9449
|
-
configuration: {
|
|
9450
|
-
defaultRightsMode: "private",
|
|
9451
|
-
calculateVectors: "manual"
|
|
9452
|
-
},
|
|
9453
|
-
fields: [
|
|
9454
|
-
{
|
|
9455
|
-
name: "content",
|
|
9456
|
-
type: "longText"
|
|
9457
|
-
}
|
|
9458
|
-
],
|
|
9459
|
-
active: true
|
|
9460
|
-
});
|
|
9461
|
-
|
|
9462
9616
|
// src/registry/index.ts
|
|
9463
9617
|
import winston2 from "winston";
|
|
9464
9618
|
import util from "util";
|
|
@@ -9572,8 +9726,14 @@ var llmAsJudgeEval = () => {
|
|
|
9572
9726
|
console.error("[EXULU] prompt is required for llm as judge eval but none is provided.");
|
|
9573
9727
|
throw new Error("Prompt is required for llm as judge eval but none is provided.");
|
|
9574
9728
|
}
|
|
9729
|
+
console.log("[EXULU] messages", messages);
|
|
9730
|
+
const lastTypes = messages[messages.length - 1]?.parts?.map((part) => ({
|
|
9731
|
+
type: part.type,
|
|
9732
|
+
text: part.type === "text" ? part.text?.slice(0, 100) : void 0
|
|
9733
|
+
}));
|
|
9575
9734
|
const lastMessage = messages[messages.length - 1]?.parts?.filter((part) => part.type === "text").map((part) => part.text).join("\n");
|
|
9576
9735
|
console.log("[EXULU] last message", lastMessage);
|
|
9736
|
+
console.log("[EXULU] last types", lastTypes);
|
|
9577
9737
|
if (!lastMessage) {
|
|
9578
9738
|
return 0;
|
|
9579
9739
|
}
|
|
@@ -10585,9 +10745,7 @@ var ExuluApp = class {
|
|
|
10585
10745
|
...evals ?? []
|
|
10586
10746
|
] : [];
|
|
10587
10747
|
this._contexts = {
|
|
10588
|
-
...contexts
|
|
10589
|
-
codeStandardsContext,
|
|
10590
|
-
outputsContext
|
|
10748
|
+
...contexts
|
|
10591
10749
|
};
|
|
10592
10750
|
this._agents = [
|
|
10593
10751
|
claudeSonnet4Agent,
|
|
@@ -10644,8 +10802,8 @@ var ExuluApp = class {
|
|
|
10644
10802
|
const queueSet = /* @__PURE__ */ new Set();
|
|
10645
10803
|
if (redisServer.host?.length && redisServer.port?.length) {
|
|
10646
10804
|
queues.register(global_queues.eval_runs, {
|
|
10647
|
-
worker:
|
|
10648
|
-
queue:
|
|
10805
|
+
worker: 10,
|
|
10806
|
+
queue: 10
|
|
10649
10807
|
}, 1);
|
|
10650
10808
|
for (const queue of queues.list.values()) {
|
|
10651
10809
|
const config2 = await queue.use();
|
|
@@ -12323,10 +12481,6 @@ var ExuluJobs = {
|
|
|
12323
12481
|
validate: validateJob
|
|
12324
12482
|
}
|
|
12325
12483
|
};
|
|
12326
|
-
var ExuluDefaultContexts = {
|
|
12327
|
-
codeStandards: codeStandardsContext,
|
|
12328
|
-
outputs: outputsContext
|
|
12329
|
-
};
|
|
12330
12484
|
var ExuluDefaultAgents = {
|
|
12331
12485
|
anthropic: {
|
|
12332
12486
|
opus4: claudeOpus4Agent,
|
|
@@ -12484,7 +12638,6 @@ export {
|
|
|
12484
12638
|
ExuluChunkers,
|
|
12485
12639
|
ExuluContext,
|
|
12486
12640
|
ExuluDefaultAgents,
|
|
12487
|
-
ExuluDefaultContexts,
|
|
12488
12641
|
ExuluEmbedder,
|
|
12489
12642
|
ExuluEval2 as ExuluEval,
|
|
12490
12643
|
ExuluJobs,
|