@exulu/backend 2.0.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-IJ4HNHOT.js → chunk-Y7JPNBFM.js} +603 -279
- package/dist/{convert-exulu-tools-to-ai-sdk-tools-2PEDFZ2X.js → convert-exulu-tools-to-ai-sdk-tools-M7I2TZQQ.js} +1 -1
- package/dist/index.cjs +1689 -525
- package/dist/index.d.cts +37 -2
- package/dist/index.d.ts +37 -2
- package/dist/index.js +1108 -307
- package/ee/agentic-retrieval/pipeline/config.test.ts +15 -0
- package/ee/agentic-retrieval/pipeline/config.ts +6 -0
- package/ee/agentic-retrieval/pipeline/global-ids.ts +30 -0
- package/ee/agentic-retrieval/pipeline/index.test.ts +86 -1
- package/ee/agentic-retrieval/pipeline/index.ts +96 -46
- package/ee/agentic-retrieval/pipeline/project-scope.test.ts +73 -0
- package/ee/agentic-retrieval/pipeline/project-scope.ts +77 -0
- package/ee/agentic-retrieval/pipeline/search.test.ts +27 -0
- package/ee/agentic-retrieval/pipeline/search.ts +7 -0
- package/package.json +1 -1
|
@@ -665,6 +665,9 @@ async function tagUpdate(input) {
|
|
|
665
665
|
budget_duration: input.budget_duration
|
|
666
666
|
});
|
|
667
667
|
}
|
|
668
|
+
async function budgetUpdate(budget_id, patch) {
|
|
669
|
+
await call("/budget/update", { budget_id, ...patch });
|
|
670
|
+
}
|
|
668
671
|
async function tagDelete(name) {
|
|
669
672
|
await call("/tag/delete", { name });
|
|
670
673
|
}
|
|
@@ -673,8 +676,9 @@ function extractBudget(raw) {
|
|
|
673
676
|
const max_budget = bt.max_budget ?? raw?.max_budget ?? null;
|
|
674
677
|
const budget_duration = bt.budget_duration ?? raw?.budget_duration ?? null;
|
|
675
678
|
const budget_reset_at = bt.budget_reset_at ?? raw?.budget_reset_at ?? null;
|
|
679
|
+
const budget_id = bt.budget_id ?? raw?.budget_id ?? null;
|
|
676
680
|
const spend = typeof raw?.spend === "number" ? raw.spend : typeof bt.spend === "number" ? bt.spend : 0;
|
|
677
|
-
return { max_budget, budget_duration, budget_reset_at, spend };
|
|
681
|
+
return { max_budget, budget_duration, budget_reset_at, budget_id, spend };
|
|
678
682
|
}
|
|
679
683
|
async function listTags() {
|
|
680
684
|
const { url, masterKey } = litellmBase();
|
|
@@ -707,7 +711,8 @@ async function listTagBudgets() {
|
|
|
707
711
|
spend: b.spend,
|
|
708
712
|
max_budget: b.max_budget,
|
|
709
713
|
budget_duration: b.budget_duration,
|
|
710
|
-
budget_reset_at: b.budget_reset_at
|
|
714
|
+
budget_reset_at: b.budget_reset_at,
|
|
715
|
+
budget_id: b.budget_id
|
|
711
716
|
};
|
|
712
717
|
}
|
|
713
718
|
const names = Object.keys(map);
|
|
@@ -753,7 +758,8 @@ async function tagInfo(names) {
|
|
|
753
758
|
spend: b.spend,
|
|
754
759
|
max_budget: b.max_budget,
|
|
755
760
|
budget_duration: b.budget_duration,
|
|
756
|
-
budget_reset_at: b.budget_reset_at
|
|
761
|
+
budget_reset_at: b.budget_reset_at,
|
|
762
|
+
budget_id: b.budget_id
|
|
757
763
|
};
|
|
758
764
|
}
|
|
759
765
|
return out;
|
|
@@ -975,7 +981,16 @@ async function setBudgetSettings(settings) {
|
|
|
975
981
|
}).onConflict("config_key").merge({ config_value: JSON.stringify(settings) });
|
|
976
982
|
return settings;
|
|
977
983
|
}
|
|
978
|
-
|
|
984
|
+
function parseResetAt(raw) {
|
|
985
|
+
if (raw === void 0 || raw === null || raw === "") {
|
|
986
|
+
return { valid: true, value: void 0 };
|
|
987
|
+
}
|
|
988
|
+
if (typeof raw !== "string") return { valid: false };
|
|
989
|
+
const t = Date.parse(raw);
|
|
990
|
+
if (Number.isNaN(t)) return { valid: false };
|
|
991
|
+
return { valid: true, value: new Date(t).toISOString() };
|
|
992
|
+
}
|
|
993
|
+
async function upsertBudget(tag, max_budget, budget_duration, budget_reset_at) {
|
|
979
994
|
const info = await tagInfo([tag]);
|
|
980
995
|
try {
|
|
981
996
|
if (info[tag]) {
|
|
@@ -990,6 +1005,15 @@ async function upsertBudget(tag, max_budget, budget_duration) {
|
|
|
990
1005
|
await tagUpdate({ name: tag, max_budget, budget_duration });
|
|
991
1006
|
}
|
|
992
1007
|
}
|
|
1008
|
+
if (budget_reset_at) {
|
|
1009
|
+
const after = await tagInfo([tag]);
|
|
1010
|
+
const budgetId = after[tag]?.budget_id ?? null;
|
|
1011
|
+
if (budgetId) {
|
|
1012
|
+
await budgetUpdate(budgetId, { budget_reset_at });
|
|
1013
|
+
} else {
|
|
1014
|
+
console.warn(`[EXULU] upsertBudget: no budget_id for ${tag}; reset date not applied`);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
993
1017
|
invalidateBudgetCaches(tag);
|
|
994
1018
|
}
|
|
995
1019
|
function invalidateBudgetCaches(tag) {
|
|
@@ -1691,7 +1715,7 @@ var ExuluTool = class _ExuluTool {
|
|
|
1691
1715
|
});
|
|
1692
1716
|
providerapikey = resolved.apiKey;
|
|
1693
1717
|
}
|
|
1694
|
-
const { convertExuluToolsToAiSdkTools: convertExuluToolsToAiSdkTools2 } = await import("./convert-exulu-tools-to-ai-sdk-tools-
|
|
1718
|
+
const { convertExuluToolsToAiSdkTools: convertExuluToolsToAiSdkTools2 } = await import("./convert-exulu-tools-to-ai-sdk-tools-M7I2TZQQ.js");
|
|
1695
1719
|
const tools = await convertExuluToolsToAiSdkTools2(
|
|
1696
1720
|
[this],
|
|
1697
1721
|
[],
|
|
@@ -1774,111 +1798,8 @@ var updateStatistic = async (statistic) => {
|
|
|
1774
1798
|
// src/templates/tools/convert-exulu-tools-to-ai-sdk-tools.ts
|
|
1775
1799
|
import CryptoJS5 from "crypto-js";
|
|
1776
1800
|
|
|
1777
|
-
// src/templates/tools/project-retrieval-tool.ts
|
|
1778
|
-
import { z as z2 } from "zod";
|
|
1779
|
-
var createProjectItemsRetrievalTool = async ({
|
|
1780
|
-
user,
|
|
1781
|
-
role,
|
|
1782
|
-
contexts,
|
|
1783
|
-
projectId
|
|
1784
|
-
}) => {
|
|
1785
|
-
let project;
|
|
1786
|
-
const { db: db2 } = await postgresClient();
|
|
1787
|
-
project = await db2.from("projects").where("id", projectId).first();
|
|
1788
|
-
if (!project) {
|
|
1789
|
-
return;
|
|
1790
|
-
}
|
|
1791
|
-
console.log("[EXULU] Project search tool created for project", project);
|
|
1792
|
-
if (!project.project_items?.length) {
|
|
1793
|
-
return;
|
|
1794
|
-
}
|
|
1795
|
-
const projectRetrievalTool = ExuluTool.internal({
|
|
1796
|
-
id: "context_search_in_knowledge_items_added_to_project_" + projectId,
|
|
1797
|
-
name: "context_search in knowledge items added to project " + project.name,
|
|
1798
|
-
description: "This tool retrieves information about a project from conversations and items that were added to the project " + project.name + ".",
|
|
1799
|
-
inputSchema: z2.object({
|
|
1800
|
-
query: z2.string().describe("The query to retrieve information about the project " + project.name + "."),
|
|
1801
|
-
keywords: z2.array(z2.string()).describe(
|
|
1802
|
-
"The most relevant keywords in the query, such as names of people, companies, products, etc. in the project " + project.name + "."
|
|
1803
|
-
)
|
|
1804
|
-
}),
|
|
1805
|
-
type: "context",
|
|
1806
|
-
category: "project",
|
|
1807
|
-
config: [],
|
|
1808
|
-
execute: async ({ query }) => {
|
|
1809
|
-
console.log("[EXULU] Project search tool searching for project", project);
|
|
1810
|
-
const items = project.project_items;
|
|
1811
|
-
const set = {};
|
|
1812
|
-
for (const item of items) {
|
|
1813
|
-
const context = item.split("/")[0];
|
|
1814
|
-
if (!context) {
|
|
1815
|
-
throw new Error(
|
|
1816
|
-
"The item added to the project does not have a valid gid with the context id as the prefix before the first slash."
|
|
1817
|
-
);
|
|
1818
|
-
}
|
|
1819
|
-
const id = item.split("/").slice(1).join("/");
|
|
1820
|
-
if (set[context]) {
|
|
1821
|
-
set[context].push(id);
|
|
1822
|
-
} else {
|
|
1823
|
-
set[context] = [id];
|
|
1824
|
-
}
|
|
1825
|
-
}
|
|
1826
|
-
console.log("[EXULU] Project search tool searching through contexts", Object.keys(set));
|
|
1827
|
-
const results = await Promise.all(
|
|
1828
|
-
Object.keys(set).map(async (contextName) => {
|
|
1829
|
-
const context = contexts.find((context2) => context2.id === contextName);
|
|
1830
|
-
if (!context) {
|
|
1831
|
-
console.error(
|
|
1832
|
-
"[EXULU] Context not found for project information retrieval tool.",
|
|
1833
|
-
contextName
|
|
1834
|
-
);
|
|
1835
|
-
return [];
|
|
1836
|
-
}
|
|
1837
|
-
const itemIds = set[contextName];
|
|
1838
|
-
console.log("[EXULU] Project search tool searching through items", itemIds);
|
|
1839
|
-
const result = await context.search({
|
|
1840
|
-
query,
|
|
1841
|
-
itemFilters: [
|
|
1842
|
-
{
|
|
1843
|
-
id: {
|
|
1844
|
-
in: itemIds
|
|
1845
|
-
}
|
|
1846
|
-
}
|
|
1847
|
-
],
|
|
1848
|
-
chunkFilters: [],
|
|
1849
|
-
user,
|
|
1850
|
-
role,
|
|
1851
|
-
method: "hybridSearch",
|
|
1852
|
-
sort: {
|
|
1853
|
-
field: "updatedAt",
|
|
1854
|
-
direction: "desc"
|
|
1855
|
-
},
|
|
1856
|
-
trigger: "tool",
|
|
1857
|
-
limit: 10,
|
|
1858
|
-
page: 1
|
|
1859
|
-
});
|
|
1860
|
-
return {
|
|
1861
|
-
result: result.chunks.map((chunk) => ({
|
|
1862
|
-
...chunk,
|
|
1863
|
-
context: {
|
|
1864
|
-
name: context.name,
|
|
1865
|
-
id: context.id
|
|
1866
|
-
}
|
|
1867
|
-
}))
|
|
1868
|
-
};
|
|
1869
|
-
})
|
|
1870
|
-
);
|
|
1871
|
-
console.log("[EXULU] Project search tool results", results);
|
|
1872
|
-
return {
|
|
1873
|
-
result: JSON.stringify(results.flat())
|
|
1874
|
-
};
|
|
1875
|
-
}
|
|
1876
|
-
});
|
|
1877
|
-
return projectRetrievalTool;
|
|
1878
|
-
};
|
|
1879
|
-
|
|
1880
1801
|
// src/templates/tools/session-items-retrieval-tool.ts
|
|
1881
|
-
import { z as
|
|
1802
|
+
import { z as z2 } from "zod";
|
|
1882
1803
|
var createSessionItemsRetrievalTool = async ({
|
|
1883
1804
|
user,
|
|
1884
1805
|
role,
|
|
@@ -1890,8 +1811,8 @@ var createSessionItemsRetrievalTool = async ({
|
|
|
1890
1811
|
id: "session_items_information_context_search",
|
|
1891
1812
|
name: "context_search in knowledge items added to session.",
|
|
1892
1813
|
description: "Context search in knowledge items added to session.",
|
|
1893
|
-
inputSchema:
|
|
1894
|
-
query:
|
|
1814
|
+
inputSchema: z2.object({
|
|
1815
|
+
query: z2.string().describe("The query to retrieve information from knowledge items added to the session.")
|
|
1895
1816
|
}),
|
|
1896
1817
|
type: "context",
|
|
1897
1818
|
category: "session",
|
|
@@ -1962,7 +1883,7 @@ var createSessionItemsRetrievalTool = async ({
|
|
|
1962
1883
|
};
|
|
1963
1884
|
|
|
1964
1885
|
// ee/agentic-retrieval/pipeline/index.ts
|
|
1965
|
-
import { z as
|
|
1886
|
+
import { z as z7 } from "zod";
|
|
1966
1887
|
|
|
1967
1888
|
// ee/entitlements.ts
|
|
1968
1889
|
var ENTITLEMENTS = {
|
|
@@ -2106,57 +2027,57 @@ async function resolveReranker(input) {
|
|
|
2106
2027
|
}
|
|
2107
2028
|
|
|
2108
2029
|
// ee/agentic-retrieval/pipeline/config.ts
|
|
2109
|
-
import { z as
|
|
2030
|
+
import { z as z3 } from "zod";
|
|
2110
2031
|
var KB_KINDS = ["documents", "conversations", "records"];
|
|
2111
2032
|
var DEFAULT_PREFILTER_CUTOFF = 2.5;
|
|
2112
2033
|
var RRF_K = 60;
|
|
2113
2034
|
var CHUNK_GROUP_MAX = 10;
|
|
2114
|
-
var kbProfileSchema =
|
|
2115
|
-
enabled:
|
|
2116
|
-
kind:
|
|
2117
|
-
instructions:
|
|
2118
|
-
overrides:
|
|
2119
|
-
limit:
|
|
2120
|
-
expand:
|
|
2121
|
-
multiQuery:
|
|
2122
|
-
hyde:
|
|
2035
|
+
var kbProfileSchema = z3.object({
|
|
2036
|
+
enabled: z3.boolean().default(true),
|
|
2037
|
+
kind: z3.enum(KB_KINDS).default("documents"),
|
|
2038
|
+
instructions: z3.string().default(""),
|
|
2039
|
+
overrides: z3.object({
|
|
2040
|
+
limit: z3.number().int().positive().optional(),
|
|
2041
|
+
expand: z3.number().int().min(0).optional(),
|
|
2042
|
+
multiQuery: z3.boolean().optional(),
|
|
2043
|
+
hyde: z3.boolean().optional()
|
|
2123
2044
|
}).default({})
|
|
2124
2045
|
});
|
|
2125
|
-
var knowledgeBasesSchema =
|
|
2126
|
-
var routingRuleSchema =
|
|
2127
|
-
id:
|
|
2128
|
-
label:
|
|
2129
|
-
description:
|
|
2130
|
-
main:
|
|
2131
|
-
fallback:
|
|
2046
|
+
var knowledgeBasesSchema = z3.record(z3.string(), kbProfileSchema);
|
|
2047
|
+
var routingRuleSchema = z3.object({
|
|
2048
|
+
id: z3.string(),
|
|
2049
|
+
label: z3.string(),
|
|
2050
|
+
description: z3.string(),
|
|
2051
|
+
main: z3.array(z3.string()),
|
|
2052
|
+
fallback: z3.array(z3.string()).default([])
|
|
2132
2053
|
});
|
|
2133
|
-
var routingSchema =
|
|
2134
|
-
var identifierSetSchema =
|
|
2135
|
-
name:
|
|
2136
|
-
description:
|
|
2137
|
-
examples:
|
|
2138
|
-
strategy:
|
|
2139
|
-
contexts:
|
|
2054
|
+
var routingSchema = z3.object({ rules: z3.array(routingRuleSchema).default([]) });
|
|
2055
|
+
var identifierSetSchema = z3.object({
|
|
2056
|
+
name: z3.string(),
|
|
2057
|
+
description: z3.string().default(""),
|
|
2058
|
+
examples: z3.array(z3.string()).default([]),
|
|
2059
|
+
strategy: z3.enum(["fuzzy", "exact"]),
|
|
2060
|
+
contexts: z3.array(z3.string()).default([])
|
|
2140
2061
|
});
|
|
2141
|
-
var vocabularySchema =
|
|
2142
|
-
glossary:
|
|
2143
|
-
identifiers:
|
|
2144
|
-
rewrites:
|
|
2145
|
-
styleHint:
|
|
2062
|
+
var vocabularySchema = z3.object({
|
|
2063
|
+
glossary: z3.array(z3.object({ term: z3.string(), meaning: z3.string() })).default([]),
|
|
2064
|
+
identifiers: z3.array(identifierSetSchema).default([]),
|
|
2065
|
+
rewrites: z3.array(z3.object({ find: z3.string(), replace: z3.string() })).default([]),
|
|
2066
|
+
styleHint: z3.string().default("")
|
|
2146
2067
|
});
|
|
2147
|
-
var memorySchema =
|
|
2148
|
-
enabled:
|
|
2149
|
-
override:
|
|
2150
|
-
filePrioritization:
|
|
2151
|
-
queryAugmentation:
|
|
2068
|
+
var memorySchema = z3.object({
|
|
2069
|
+
enabled: z3.boolean().default(true),
|
|
2070
|
+
override: z3.boolean().default(false),
|
|
2071
|
+
filePrioritization: z3.boolean().default(false),
|
|
2072
|
+
queryAugmentation: z3.boolean().default(true)
|
|
2152
2073
|
});
|
|
2153
|
-
var tuningSchema =
|
|
2154
|
-
topK:
|
|
2155
|
-
fallbackThreshold:
|
|
2156
|
-
pinBoost:
|
|
2157
|
-
identifierBoost:
|
|
2158
|
-
pageWindow:
|
|
2159
|
-
maxQueriesPerContext:
|
|
2074
|
+
var tuningSchema = z3.object({
|
|
2075
|
+
topK: z3.number().int().positive().default(5),
|
|
2076
|
+
fallbackThreshold: z3.number().min(0).max(1).default(0.95),
|
|
2077
|
+
pinBoost: z3.number().min(0).max(1).default(0.15),
|
|
2078
|
+
identifierBoost: z3.number().min(0).max(1).default(0.15),
|
|
2079
|
+
pageWindow: z3.number().int().min(0).default(1),
|
|
2080
|
+
maxQueriesPerContext: z3.number().int().positive().default(5)
|
|
2160
2081
|
});
|
|
2161
2082
|
var boolVal = (v) => v === true || v === "true" || v === 1;
|
|
2162
2083
|
var strVal = (v, fallback) => typeof v === "string" && v.length > 0 ? v : fallback;
|
|
@@ -2212,6 +2133,7 @@ function parsePipelineConfig(raw) {
|
|
|
2212
2133
|
managedContext: boolVal(r["managed_context"]),
|
|
2213
2134
|
requirePreselectedContexts: boolVal(r["require_preselected_contexts"]),
|
|
2214
2135
|
logging: boolVal(r["logging"]),
|
|
2136
|
+
projectSearch: r["project_search"] === void 0 || r["project_search"] === "" ? true : boolVal(r["project_search"]),
|
|
2215
2137
|
utilityModel: strVal(r["utility_model"], ""),
|
|
2216
2138
|
knowledgeBases: jsonVal("knowledge_bases", knowledgeBasesSchema, r["knowledge_bases"]),
|
|
2217
2139
|
routing: jsonVal("routing", routingSchema, r["routing"]),
|
|
@@ -2244,9 +2166,69 @@ function effectiveKbSettings(profile, ctx) {
|
|
|
2244
2166
|
};
|
|
2245
2167
|
}
|
|
2246
2168
|
|
|
2169
|
+
// ee/agentic-retrieval/pipeline/global-ids.ts
|
|
2170
|
+
function parsePreselectedItems(globalIds) {
|
|
2171
|
+
const map = /* @__PURE__ */ new Map();
|
|
2172
|
+
for (const gid of globalIds) {
|
|
2173
|
+
const slashIdx = gid.indexOf("/");
|
|
2174
|
+
if (slashIdx === -1) {
|
|
2175
|
+
if (gid) map.set(gid, null);
|
|
2176
|
+
continue;
|
|
2177
|
+
}
|
|
2178
|
+
const contextId = gid.slice(0, slashIdx);
|
|
2179
|
+
const itemId = gid.slice(slashIdx + 1);
|
|
2180
|
+
if (!contextId || !itemId) continue;
|
|
2181
|
+
if (map.get(contextId) === null) continue;
|
|
2182
|
+
const existing = map.get(contextId) ?? [];
|
|
2183
|
+
existing.push(itemId);
|
|
2184
|
+
map.set(contextId, existing);
|
|
2185
|
+
}
|
|
2186
|
+
return map;
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
// ee/agentic-retrieval/pipeline/project-scope.ts
|
|
2190
|
+
function resolveProjectScope(opts) {
|
|
2191
|
+
const { scope, enabledContextIds, availableContextIds } = opts;
|
|
2192
|
+
if (!scope || scope.items.length === 0) return void 0;
|
|
2193
|
+
const itemsByContext = parsePreselectedItems(scope.items);
|
|
2194
|
+
const pinsByContext = /* @__PURE__ */ new Map();
|
|
2195
|
+
const scopedItemsByContext = /* @__PURE__ */ new Map();
|
|
2196
|
+
const addedContextIds = [];
|
|
2197
|
+
const allProjectContextIds = [];
|
|
2198
|
+
for (const [ctxId, itemIds] of itemsByContext) {
|
|
2199
|
+
if (!availableContextIds.has(ctxId)) {
|
|
2200
|
+
console.warn(
|
|
2201
|
+
`[EXULU pipeline] project "${scope.name}" references unknown context "${ctxId}" \u2014 skipping those items.`
|
|
2202
|
+
);
|
|
2203
|
+
continue;
|
|
2204
|
+
}
|
|
2205
|
+
allProjectContextIds.push(ctxId);
|
|
2206
|
+
if (enabledContextIds.has(ctxId)) {
|
|
2207
|
+
if (itemIds && itemIds.length > 0) pinsByContext.set(ctxId, new Set(itemIds));
|
|
2208
|
+
} else {
|
|
2209
|
+
scopedItemsByContext.set(ctxId, itemIds);
|
|
2210
|
+
addedContextIds.push(ctxId);
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
if (allProjectContextIds.length === 0) return void 0;
|
|
2214
|
+
return { pinsByContext, scopedItemsByContext, addedContextIds, allProjectContextIds };
|
|
2215
|
+
}
|
|
2216
|
+
var TRANSCRIPTIONS_CONTEXT_ID = "transcriptions";
|
|
2217
|
+
function buildProjectKbProfileDefaults(items) {
|
|
2218
|
+
const defaults = {};
|
|
2219
|
+
for (const gid of items) {
|
|
2220
|
+
const slashIdx = gid.indexOf("/");
|
|
2221
|
+
const ctxId = slashIdx === -1 ? gid : gid.slice(0, slashIdx);
|
|
2222
|
+
if (ctxId === TRANSCRIPTIONS_CONTEXT_ID && !defaults[ctxId]) {
|
|
2223
|
+
defaults[ctxId] = { enabled: true, kind: "conversations", instructions: "", overrides: {} };
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
return defaults;
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2247
2229
|
// ee/agentic-retrieval/pipeline/routing.ts
|
|
2248
2230
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
2249
|
-
import { z as
|
|
2231
|
+
import { z as z5 } from "zod";
|
|
2250
2232
|
|
|
2251
2233
|
// src/utils/with-retry.ts
|
|
2252
2234
|
async function withRetry(generateFn, maxRetries = 3) {
|
|
@@ -2269,7 +2251,7 @@ async function withRetry(generateFn, maxRetries = 3) {
|
|
|
2269
2251
|
// ee/agentic-retrieval/pipeline/prefilter.ts
|
|
2270
2252
|
import Fuse from "fuse.js";
|
|
2271
2253
|
import { generateText, Output } from "ai";
|
|
2272
|
-
import { z as
|
|
2254
|
+
import { z as z4 } from "zod";
|
|
2273
2255
|
|
|
2274
2256
|
// ee/agentic-retrieval/pipeline/text-utils.ts
|
|
2275
2257
|
var normalizeFileName = (fileName) => {
|
|
@@ -2514,9 +2496,9 @@ async function resolveIdentifierPins({
|
|
|
2514
2496
|
system: set.strategy === "exact" ? EXACT_EXTRACTION_PROMPT(set) : FUZZY_EXTRACTION_PROMPT(set),
|
|
2515
2497
|
messages: [{ role: "user", content: question }],
|
|
2516
2498
|
output: Output.object({
|
|
2517
|
-
schema:
|
|
2518
|
-
hasMatches:
|
|
2519
|
-
matches:
|
|
2499
|
+
schema: z4.object({
|
|
2500
|
+
hasMatches: z4.boolean(),
|
|
2501
|
+
matches: z4.array(z4.string()).optional()
|
|
2520
2502
|
})
|
|
2521
2503
|
}),
|
|
2522
2504
|
maxOutputTokens: 300
|
|
@@ -2627,11 +2609,11 @@ If explicit, return the knowledge base ids. If not, return an empty array.`;
|
|
|
2627
2609
|
system: buildDocPagePrompt(knownIdentifiers),
|
|
2628
2610
|
messages: [{ role: "user", content: question }],
|
|
2629
2611
|
output: Output2.object({
|
|
2630
|
-
schema:
|
|
2631
|
-
hasFilenameHint:
|
|
2632
|
-
filenameHints:
|
|
2633
|
-
hasPageHint:
|
|
2634
|
-
pageNumber:
|
|
2612
|
+
schema: z5.object({
|
|
2613
|
+
hasFilenameHint: z5.boolean(),
|
|
2614
|
+
filenameHints: z5.array(z5.string()).optional(),
|
|
2615
|
+
hasPageHint: z5.boolean(),
|
|
2616
|
+
pageNumber: z5.number().int().nullable().optional()
|
|
2635
2617
|
})
|
|
2636
2618
|
}),
|
|
2637
2619
|
maxOutputTokens: 300
|
|
@@ -2658,9 +2640,9 @@ If explicit, return the knowledge base ids. If not, return an empty array.`;
|
|
|
2658
2640
|
temperature: 0,
|
|
2659
2641
|
system: kbSystemPrompt,
|
|
2660
2642
|
output: Output2.object({
|
|
2661
|
-
schema:
|
|
2662
|
-
explicitlyRequestedKnowledgeBases:
|
|
2663
|
-
|
|
2643
|
+
schema: z5.object({
|
|
2644
|
+
explicitlyRequestedKnowledgeBases: z5.array(
|
|
2645
|
+
z5.enum(enabledContexts.map((c) => c.id))
|
|
2664
2646
|
)
|
|
2665
2647
|
})
|
|
2666
2648
|
}),
|
|
@@ -2758,9 +2740,9 @@ ${extraInstructions}
|
|
|
2758
2740
|
system: classifyPrompt,
|
|
2759
2741
|
messages: [{ role: "user", content: question }],
|
|
2760
2742
|
output: Output2.object({
|
|
2761
|
-
schema:
|
|
2762
|
-
ruleId:
|
|
2763
|
-
reason:
|
|
2743
|
+
schema: z5.object({
|
|
2744
|
+
ruleId: z5.enum(ruleIds),
|
|
2745
|
+
reason: z5.string()
|
|
2764
2746
|
})
|
|
2765
2747
|
}),
|
|
2766
2748
|
maxOutputTokens: 200
|
|
@@ -2822,7 +2804,7 @@ ${extraInstructions}
|
|
|
2822
2804
|
|
|
2823
2805
|
// ee/agentic-retrieval/pipeline/memory.ts
|
|
2824
2806
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
2825
|
-
import { z as
|
|
2807
|
+
import { z as z6 } from "zod";
|
|
2826
2808
|
|
|
2827
2809
|
// ee/agentic-retrieval/pipeline/multi-query.ts
|
|
2828
2810
|
async function singleSearch({
|
|
@@ -3063,8 +3045,8 @@ async function runMemoryPhase({
|
|
|
3063
3045
|
}
|
|
3064
3046
|
],
|
|
3065
3047
|
output: Output3.object({
|
|
3066
|
-
schema:
|
|
3067
|
-
relevantChunkIds:
|
|
3048
|
+
schema: z6.object({
|
|
3049
|
+
relevantChunkIds: z6.array(z6.string()).describe(
|
|
3068
3050
|
"The chunk_ids (UUIDs at the start of each bullet) of chunks containing information relevant to the user's question. Empty array if none are relevant."
|
|
3069
3051
|
)
|
|
3070
3052
|
})
|
|
@@ -3180,17 +3162,17 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
|
|
|
3180
3162
|
}
|
|
3181
3163
|
],
|
|
3182
3164
|
output: Output3.object({
|
|
3183
|
-
schema:
|
|
3184
|
-
overrides:
|
|
3165
|
+
schema: z6.object({
|
|
3166
|
+
overrides: z6.boolean().describe(
|
|
3185
3167
|
"True ONLY if a memory chunk directly and sufficiently answers the user's question and should be authoritative over the documents. Be strict; when unsure, false."
|
|
3186
3168
|
),
|
|
3187
|
-
confidence:
|
|
3169
|
+
confidence: z6.enum(["high", "medium", "low"]).describe(
|
|
3188
3170
|
"Confidence that the selected memory chunk(s) fully and directly answer the question."
|
|
3189
3171
|
),
|
|
3190
|
-
authoritativeChunkIds:
|
|
3172
|
+
authoritativeChunkIds: z6.array(z6.string()).describe(
|
|
3191
3173
|
"The chunk_ids of the memory chunk(s) that directly answer the question. Empty if overrides is false."
|
|
3192
3174
|
),
|
|
3193
|
-
reason:
|
|
3175
|
+
reason: z6.string().describe(
|
|
3194
3176
|
"One short sentence: why this memory does or does not directly answer the question."
|
|
3195
3177
|
)
|
|
3196
3178
|
})
|
|
@@ -3221,9 +3203,9 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
|
|
|
3221
3203
|
system: "You are a helpful assistant that will strictly follow the user's instructions.",
|
|
3222
3204
|
messages: [{ role: "user", content: PROMPT_EXTRACT_PRIORITIZED_FILES }],
|
|
3223
3205
|
output: Output3.object({
|
|
3224
|
-
schema:
|
|
3225
|
-
shouldPrioritizeFiles:
|
|
3226
|
-
fileNameHints:
|
|
3206
|
+
schema: z6.object({
|
|
3207
|
+
shouldPrioritizeFiles: z6.boolean(),
|
|
3208
|
+
fileNameHints: z6.array(z6.string()).optional()
|
|
3227
3209
|
})
|
|
3228
3210
|
}),
|
|
3229
3211
|
maxOutputTokens: 300
|
|
@@ -3242,10 +3224,10 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
|
|
|
3242
3224
|
system: "You are a helpful assistant that will strictly follow the user's instructions.",
|
|
3243
3225
|
messages: [{ role: "user", content: QUERY_AUGMENTATION_PROMPT }],
|
|
3244
3226
|
output: Output3.object({
|
|
3245
|
-
schema:
|
|
3246
|
-
updatedUserQuestion:
|
|
3247
|
-
updatedRelevantKeywords:
|
|
3248
|
-
updatedImportantKeyword:
|
|
3227
|
+
schema: z6.object({
|
|
3228
|
+
updatedUserQuestion: z6.string(),
|
|
3229
|
+
updatedRelevantKeywords: z6.array(z6.string()),
|
|
3230
|
+
updatedImportantKeyword: z6.string()
|
|
3249
3231
|
})
|
|
3250
3232
|
}),
|
|
3251
3233
|
maxOutputTokens: 600
|
|
@@ -3426,6 +3408,7 @@ async function searchContexts(opts) {
|
|
|
3426
3408
|
role,
|
|
3427
3409
|
model,
|
|
3428
3410
|
preselectedItems,
|
|
3411
|
+
scopedItemsByContext,
|
|
3429
3412
|
identifierPinsByContext,
|
|
3430
3413
|
memoryPinnedItemIds,
|
|
3431
3414
|
userPinnedItemIdsByContext,
|
|
@@ -3452,6 +3435,8 @@ async function searchContexts(opts) {
|
|
|
3452
3435
|
let pinnedItemIds;
|
|
3453
3436
|
if (hasPreselection) {
|
|
3454
3437
|
pinnedItemIds = preselectedItems.get(ctxId) ?? [];
|
|
3438
|
+
} else if (scopedItemsByContext?.has(ctxId)) {
|
|
3439
|
+
pinnedItemIds = scopedItemsByContext.get(ctxId) ?? [];
|
|
3455
3440
|
} else if (!skipPrefilter) {
|
|
3456
3441
|
const identifierPins = identifierPinsByContext.get(ctxId) ?? /* @__PURE__ */ new Set();
|
|
3457
3442
|
let pins = new Set(identifierPins);
|
|
@@ -3659,24 +3644,6 @@ async function rerankResults(opts) {
|
|
|
3659
3644
|
}
|
|
3660
3645
|
|
|
3661
3646
|
// ee/agentic-retrieval/pipeline/index.ts
|
|
3662
|
-
function parsePreselectedItems(globalIds) {
|
|
3663
|
-
const map = /* @__PURE__ */ new Map();
|
|
3664
|
-
for (const gid of globalIds) {
|
|
3665
|
-
const slashIdx = gid.indexOf("/");
|
|
3666
|
-
if (slashIdx === -1) {
|
|
3667
|
-
if (gid) map.set(gid, null);
|
|
3668
|
-
continue;
|
|
3669
|
-
}
|
|
3670
|
-
const contextId = gid.slice(0, slashIdx);
|
|
3671
|
-
const itemId = gid.slice(slashIdx + 1);
|
|
3672
|
-
if (!contextId || !itemId) continue;
|
|
3673
|
-
if (map.get(contextId) === null) continue;
|
|
3674
|
-
const existing = map.get(contextId) ?? [];
|
|
3675
|
-
existing.push(itemId);
|
|
3676
|
-
map.set(contextId, existing);
|
|
3677
|
-
}
|
|
3678
|
-
return map;
|
|
3679
|
-
}
|
|
3680
3647
|
function addChunks(result, chunks) {
|
|
3681
3648
|
const seen = new Set(result.chunks.map((c) => c.chunk_id));
|
|
3682
3649
|
for (const chunk of chunks) {
|
|
@@ -3706,7 +3673,8 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3706
3673
|
model,
|
|
3707
3674
|
instructions: adminInstructions,
|
|
3708
3675
|
preselected,
|
|
3709
|
-
memoryItems
|
|
3676
|
+
memoryItems,
|
|
3677
|
+
projectScope
|
|
3710
3678
|
} = opts;
|
|
3711
3679
|
const license = checkLicense();
|
|
3712
3680
|
if (!license["agentic-retrieval"]) {
|
|
@@ -3716,7 +3684,9 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3716
3684
|
return ExuluTool.internal({
|
|
3717
3685
|
id: "agentic_context_search",
|
|
3718
3686
|
name: "Context Search",
|
|
3719
|
-
description: `Intelligent knowledge search across the available knowledge bases: ${contexts.map((c) => c.name || c.id).join(", ")}. Routes the question to the right sources, searches them with query expansion, and returns reranked passages. Results are exhaustive for the given query: do NOT repeat the call with a rephrased version of the same question \u2014 re-call only with genuinely new information (a different product or model, an explicitly named source or document, or new details from the user)
|
|
3687
|
+
description: `Intelligent knowledge search across the available knowledge bases: ${contexts.map((c) => c.name || c.id).join(", ")}. Routes the question to the right sources, searches them with query expansion, and returns reranked passages. Results are exhaustive for the given query: do NOT repeat the call with a rephrased version of the same question \u2014 re-call only with genuinely new information (a different product or model, an explicitly named source or document, or new details from the user).` + // Note: the description suffix intentionally remains even when the per-agent project_search
|
|
3688
|
+
// config is off — the config is only known at execute time, not at factory time.
|
|
3689
|
+
(projectScope ? ` Also searches the knowledge items attached to the project "${projectScope.name}".` : ""),
|
|
3720
3690
|
category: "contexts",
|
|
3721
3691
|
needsApproval: false,
|
|
3722
3692
|
type: "context",
|
|
@@ -3759,10 +3729,16 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3759
3729
|
},
|
|
3760
3730
|
{
|
|
3761
3731
|
name: "max_steps",
|
|
3762
|
-
description: "Maximum
|
|
3732
|
+
description: "Maximum knowledge searches the agent may run for one message. Once spent, the search tool is disabled for the rest of the turn. 0 = no search-specific cap (the agent's overall tool-step budget still applies).",
|
|
3763
3733
|
type: "number",
|
|
3764
3734
|
default: 0
|
|
3765
3735
|
},
|
|
3736
|
+
{
|
|
3737
|
+
name: "project_search",
|
|
3738
|
+
description: "Automatically include items attached to the chat's project as an additional knowledge source (boosts them in shared sources, adds scoped search for others).",
|
|
3739
|
+
type: "boolean",
|
|
3740
|
+
default: true
|
|
3741
|
+
},
|
|
3766
3742
|
{
|
|
3767
3743
|
name: "knowledge_bases",
|
|
3768
3744
|
description: "Per-knowledge-base profiles: enabled, kind (documents | conversations | records), instructions, and per-KB overrides (limit, expand, multiQuery, hyde). JSON object keyed by context id.",
|
|
@@ -3794,11 +3770,11 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3794
3770
|
default: '{"topK":5,"fallbackThreshold":0.95,"pinBoost":0.15,"identifierBoost":0.15,"pageWindow":1,"maxQueriesPerContext":5}'
|
|
3795
3771
|
}
|
|
3796
3772
|
],
|
|
3797
|
-
inputSchema:
|
|
3798
|
-
userQuery:
|
|
3799
|
-
relevantKeywords:
|
|
3800
|
-
importantKeyword:
|
|
3801
|
-
confirmedContextIds:
|
|
3773
|
+
inputSchema: z7.object({
|
|
3774
|
+
userQuery: z7.string().describe("The original unaltered question from the user"),
|
|
3775
|
+
relevantKeywords: z7.array(z7.string()).describe("Keywords extracted from the user's question relevant to the search"),
|
|
3776
|
+
importantKeyword: z7.string().describe("The single most important keyword from the user's question"),
|
|
3777
|
+
confirmedContextIds: z7.array(z7.string()).optional().describe(
|
|
3802
3778
|
"Knowledge base IDs explicitly confirmed by the user to be used in the retrieval. When present, only searches these contexts."
|
|
3803
3779
|
)
|
|
3804
3780
|
}),
|
|
@@ -3872,6 +3848,23 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3872
3848
|
}
|
|
3873
3849
|
}
|
|
3874
3850
|
const preselectedItems = parsePreselectedItems(preselected ?? []);
|
|
3851
|
+
const availableContextsById = new Map(contexts.map((c) => [c.id, c]));
|
|
3852
|
+
const resolvedProject = cfg.projectSearch ? resolveProjectScope({
|
|
3853
|
+
scope: projectScope,
|
|
3854
|
+
enabledContextIds: new Set(enabledContexts.map((c) => c.id)),
|
|
3855
|
+
availableContextIds: new Set(availableContextsById.keys())
|
|
3856
|
+
}) : void 0;
|
|
3857
|
+
if (resolvedProject) {
|
|
3858
|
+
if (projectScope?.kbProfileDefaults) {
|
|
3859
|
+
for (const [ctxId, profile] of Object.entries(projectScope.kbProfileDefaults)) {
|
|
3860
|
+
if (!cfg.knowledgeBases[ctxId]) cfg.knowledgeBases[ctxId] = profile;
|
|
3861
|
+
}
|
|
3862
|
+
}
|
|
3863
|
+
enabledContexts = [
|
|
3864
|
+
...enabledContexts,
|
|
3865
|
+
...resolvedProject.addedContextIds.map((id) => availableContextsById.get(id)).filter((c) => Boolean(c))
|
|
3866
|
+
];
|
|
3867
|
+
}
|
|
3875
3868
|
const contextsById = new Map(enabledContexts.map((c) => [c.id, c]));
|
|
3876
3869
|
const kbKindById = new Map(
|
|
3877
3870
|
enabledContexts.map((c) => [
|
|
@@ -3882,7 +3875,12 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3882
3875
|
const documentContexts = enabledContexts.filter(
|
|
3883
3876
|
(c) => (cfg.knowledgeBases[c.id]?.kind ?? "documents") === "documents"
|
|
3884
3877
|
);
|
|
3885
|
-
const extraInstructions = [
|
|
3878
|
+
const extraInstructions = [
|
|
3879
|
+
cfg.instructions,
|
|
3880
|
+
adminInstructions,
|
|
3881
|
+
resolvedProject && projectScope?.customInstructions ? `Instructions for the attached project "${projectScope.name}":
|
|
3882
|
+
${projectScope.customInstructions}` : ""
|
|
3883
|
+
].filter(Boolean).join("\n");
|
|
3886
3884
|
const [memResult, routResult] = await Promise.all([
|
|
3887
3885
|
runMemoryPhase({
|
|
3888
3886
|
memoryChunks: memoryItems ?? [],
|
|
@@ -3928,6 +3926,30 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3928
3926
|
yield { result: "The user has requested to search in knowledge bases that are not part of the preselected knowledge bases: " + missing.join(", ") };
|
|
3929
3927
|
return;
|
|
3930
3928
|
}
|
|
3929
|
+
let effectiveMainContexts = mainContexts;
|
|
3930
|
+
if (resolvedProject) {
|
|
3931
|
+
const mainSet = new Set(mainContexts);
|
|
3932
|
+
const appended = resolvedProject.allProjectContextIds.filter(
|
|
3933
|
+
(id) => !mainSet.has(id) && contextsById.has(id)
|
|
3934
|
+
);
|
|
3935
|
+
if (appended.length > 0) {
|
|
3936
|
+
effectiveMainContexts = [...mainContexts, ...appended];
|
|
3937
|
+
result.steps.push({
|
|
3938
|
+
stepNumber: 1,
|
|
3939
|
+
text: `Including sources from project "${projectScope.name}": ${appended.join(", ")}`,
|
|
3940
|
+
toolCalls: [],
|
|
3941
|
+
chunks: [],
|
|
3942
|
+
tokens: 0
|
|
3943
|
+
});
|
|
3944
|
+
result.reasoning.push({
|
|
3945
|
+
text: `Including project sources: ${appended.join(", ")}`,
|
|
3946
|
+
tools: []
|
|
3947
|
+
});
|
|
3948
|
+
}
|
|
3949
|
+
}
|
|
3950
|
+
const fallbackContextsToSearch = fallbackContexts.filter(
|
|
3951
|
+
(id) => !effectiveMainContexts.includes(id)
|
|
3952
|
+
);
|
|
3931
3953
|
const {
|
|
3932
3954
|
updatedQuestion,
|
|
3933
3955
|
updatedKeywords,
|
|
@@ -3954,7 +3976,7 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3954
3976
|
}
|
|
3955
3977
|
const [mainSearch, speculativeFallbackSearch] = await Promise.all([
|
|
3956
3978
|
searchContexts({
|
|
3957
|
-
contextIds:
|
|
3979
|
+
contextIds: effectiveMainContexts,
|
|
3958
3980
|
contextsById,
|
|
3959
3981
|
kbProfiles: cfg.knowledgeBases,
|
|
3960
3982
|
question: updatedQuestion,
|
|
@@ -3967,13 +3989,14 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3967
3989
|
identifierPinsByContext,
|
|
3968
3990
|
memoryPinnedItemIds,
|
|
3969
3991
|
userPinnedItemIdsByContext,
|
|
3992
|
+
scopedItemsByContext: resolvedProject?.scopedItemsByContext,
|
|
3970
3993
|
rewrites: cfg.vocabulary.rewrites,
|
|
3971
3994
|
styleHint: cfg.vocabulary.styleHint,
|
|
3972
3995
|
maxQueries: cfg.tuning.maxQueriesPerContext,
|
|
3973
3996
|
skipPrefilter: false
|
|
3974
3997
|
}),
|
|
3975
|
-
|
|
3976
|
-
contextIds:
|
|
3998
|
+
fallbackContextsToSearch.length > 0 && !hasExplicitDocAndPage ? searchContexts({
|
|
3999
|
+
contextIds: fallbackContextsToSearch,
|
|
3977
4000
|
contextsById,
|
|
3978
4001
|
kbProfiles: cfg.knowledgeBases,
|
|
3979
4002
|
question: updatedQuestion,
|
|
@@ -3986,6 +4009,7 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3986
4009
|
identifierPinsByContext,
|
|
3987
4010
|
memoryPinnedItemIds,
|
|
3988
4011
|
userPinnedItemIdsByContext,
|
|
4012
|
+
scopedItemsByContext: resolvedProject?.scopedItemsByContext,
|
|
3989
4013
|
rewrites: cfg.vocabulary.rewrites,
|
|
3990
4014
|
styleHint: cfg.vocabulary.styleHint,
|
|
3991
4015
|
maxQueries: cfg.tuning.maxQueriesPerContext,
|
|
@@ -3999,6 +4023,9 @@ function createAgenticRetrievalTool(opts) {
|
|
|
3999
4023
|
})(),
|
|
4000
4024
|
...(function* () {
|
|
4001
4025
|
for (const s of userPinnedItemIdsByContext.values()) yield* s;
|
|
4026
|
+
})(),
|
|
4027
|
+
...(function* () {
|
|
4028
|
+
if (resolvedProject) for (const s of resolvedProject.pinsByContext.values()) yield* s;
|
|
4002
4029
|
})()
|
|
4003
4030
|
]);
|
|
4004
4031
|
const userPinnedItemIds = new Set(
|
|
@@ -4066,15 +4093,15 @@ function createAgenticRetrievalTool(opts) {
|
|
|
4066
4093
|
result.reasoning.push({ text: "Literal lookup satisfied; skipping fallback.", tools: [] });
|
|
4067
4094
|
yield { result: serializeOutput(result) };
|
|
4068
4095
|
}
|
|
4069
|
-
if (!literalLookupSatisfied &&
|
|
4096
|
+
if (!literalLookupSatisfied && fallbackContextsToSearch.length > 0 && (reranker ? mainRerank.rerank_score_max_genuine < cfg.tuning.fallbackThreshold : mainRerank.limited_results.length < cfg.tuning.topK)) {
|
|
4070
4097
|
result.steps.push({
|
|
4071
4098
|
stepNumber: 1,
|
|
4072
|
-
text: `Using fallback search in ${
|
|
4099
|
+
text: `Using fallback search in ${fallbackContextsToSearch.join(", ")}`,
|
|
4073
4100
|
toolCalls: [],
|
|
4074
4101
|
chunks: [],
|
|
4075
4102
|
tokens: 0
|
|
4076
4103
|
});
|
|
4077
|
-
result.reasoning.push({ text: `Fallback search in ${
|
|
4104
|
+
result.reasoning.push({ text: `Fallback search in ${fallbackContextsToSearch.join(", ")}`, tools: [] });
|
|
4078
4105
|
yield { result: serializeOutput(result) };
|
|
4079
4106
|
const fallbackRerank = await rerankResults({
|
|
4080
4107
|
chunks: speculativeFallbackSearch.chunks,
|
|
@@ -4160,7 +4187,7 @@ function sanitizeToolName(name) {
|
|
|
4160
4187
|
}
|
|
4161
4188
|
|
|
4162
4189
|
// src/templates/tools/convert-exulu-tools-to-ai-sdk-tools.ts
|
|
4163
|
-
import { randomUUID as
|
|
4190
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
4164
4191
|
|
|
4165
4192
|
// types/enums/statistics.ts
|
|
4166
4193
|
var STATISTICS_TYPE_ENUM = {
|
|
@@ -4176,12 +4203,12 @@ var STATISTICS_TYPE_ENUM = {
|
|
|
4176
4203
|
};
|
|
4177
4204
|
|
|
4178
4205
|
// src/templates/tools/memory-tool.ts
|
|
4179
|
-
import { z as
|
|
4206
|
+
import { z as z8 } from "zod";
|
|
4180
4207
|
var createNewMemoryItemTool = (agent, context) => {
|
|
4181
4208
|
const fields = {
|
|
4182
|
-
name:
|
|
4183
|
-
description:
|
|
4184
|
-
surroundingContext:
|
|
4209
|
+
name: z8.string().describe("The name of the item to create"),
|
|
4210
|
+
description: z8.string().describe("The description of the item to create"),
|
|
4211
|
+
surroundingContext: z8.string().describe("A description of the context surrounding this memory, for example if it relates to a question a user asked, a specific product, or entity etc...")
|
|
4185
4212
|
};
|
|
4186
4213
|
for (const field of context.fields) {
|
|
4187
4214
|
switch (field.type) {
|
|
@@ -4189,47 +4216,47 @@ var createNewMemoryItemTool = (agent, context) => {
|
|
|
4189
4216
|
case "longText":
|
|
4190
4217
|
case "shortText":
|
|
4191
4218
|
case "code":
|
|
4192
|
-
fields[field.name] =
|
|
4219
|
+
fields[field.name] = z8.string().describe("The " + field.name + " of the item to create");
|
|
4193
4220
|
break;
|
|
4194
4221
|
case "enum":
|
|
4195
4222
|
if (field.enumValues && field.enumValues.length > 0) {
|
|
4196
4223
|
const enumValues = field.enumValues;
|
|
4197
|
-
fields[field.name] =
|
|
4224
|
+
fields[field.name] = z8.preprocess(
|
|
4198
4225
|
(v) => typeof v === "string" ? v.toUpperCase() : v,
|
|
4199
|
-
|
|
4226
|
+
z8.enum(enumValues)
|
|
4200
4227
|
).describe(
|
|
4201
4228
|
"The " + field.name + " of the item to create. Must be one of: " + field.enumValues.join(", ")
|
|
4202
4229
|
);
|
|
4203
4230
|
} else {
|
|
4204
|
-
fields[field.name] =
|
|
4231
|
+
fields[field.name] = z8.string().describe("The " + field.name + " of the item to create");
|
|
4205
4232
|
}
|
|
4206
4233
|
break;
|
|
4207
4234
|
case "json":
|
|
4208
|
-
fields[field.name] =
|
|
4235
|
+
fields[field.name] = z8.string({}).describe(
|
|
4209
4236
|
"The " + field.name + " of the item to create, it should be a valid JSON string."
|
|
4210
4237
|
);
|
|
4211
4238
|
break;
|
|
4212
4239
|
case "markdown":
|
|
4213
|
-
fields[field.name] =
|
|
4240
|
+
fields[field.name] = z8.string().describe(
|
|
4214
4241
|
"The " + field.name + " of the item to create, it should be a valid Markdown string."
|
|
4215
4242
|
);
|
|
4216
4243
|
break;
|
|
4217
4244
|
case "number":
|
|
4218
|
-
fields[field.name] =
|
|
4245
|
+
fields[field.name] = z8.number().describe("The " + field.name + " of the item to create");
|
|
4219
4246
|
break;
|
|
4220
4247
|
case "boolean":
|
|
4221
|
-
fields[field.name] =
|
|
4248
|
+
fields[field.name] = z8.boolean().describe("The " + field.name + " of the item to create");
|
|
4222
4249
|
break;
|
|
4223
4250
|
case "file":
|
|
4224
4251
|
case "uuid":
|
|
4225
4252
|
case "date":
|
|
4226
4253
|
break;
|
|
4227
4254
|
default:
|
|
4228
|
-
fields[field.name] =
|
|
4255
|
+
fields[field.name] = z8.string().describe("The " + field.name + " of the item to create");
|
|
4229
4256
|
break;
|
|
4230
4257
|
}
|
|
4231
4258
|
}
|
|
4232
|
-
fields["visibility"] =
|
|
4259
|
+
fields["visibility"] = z8.enum(["private", "public"]).optional().describe(
|
|
4233
4260
|
"Whether this memory is private to the user or shared (public). Ask the user if unknown."
|
|
4234
4261
|
);
|
|
4235
4262
|
const toolName = "create_" + sanitizeName(context.name) + "_memory_item";
|
|
@@ -4239,7 +4266,7 @@ var createNewMemoryItemTool = (agent, context) => {
|
|
|
4239
4266
|
category: agent.name + "_memory",
|
|
4240
4267
|
description: "Create a new memory item in the " + agent.name + " memory context",
|
|
4241
4268
|
type: "function",
|
|
4242
|
-
inputSchema:
|
|
4269
|
+
inputSchema: z8.object(fields),
|
|
4243
4270
|
config: [],
|
|
4244
4271
|
execute: async (params) => {
|
|
4245
4272
|
const { name, description, surroundingContext, information, visibility, exuluConfig, user } = params;
|
|
@@ -4542,7 +4569,12 @@ function getS3Client(config) {
|
|
|
4542
4569
|
credentials: {
|
|
4543
4570
|
accessKeyId: config.fileUploads.s3key,
|
|
4544
4571
|
secretAccessKey: config.fileUploads.s3secret
|
|
4545
|
-
}
|
|
4572
|
+
},
|
|
4573
|
+
// AWS SDK >= 3.729 injects x-amz-checksum-crc32 (of an empty body) into
|
|
4574
|
+
// presigned PUT URLs, which S3-compatible stores like MinIO reject on
|
|
4575
|
+
// upload with a checksum mismatch. WHEN_REQUIRED disables that default.
|
|
4576
|
+
requestChecksumCalculation: "WHEN_REQUIRED",
|
|
4577
|
+
responseChecksumValidation: "WHEN_REQUIRED"
|
|
4546
4578
|
});
|
|
4547
4579
|
return s3Client;
|
|
4548
4580
|
}
|
|
@@ -5424,7 +5456,7 @@ ${body}`
|
|
|
5424
5456
|
// ee/invoke-skills/create-sandbox.ts
|
|
5425
5457
|
import { createBashTool } from "bash-tool";
|
|
5426
5458
|
import { tool as tool2 } from "ai";
|
|
5427
|
-
import { z as
|
|
5459
|
+
import { z as z9 } from "zod";
|
|
5428
5460
|
import CryptoJS4 from "crypto-js";
|
|
5429
5461
|
var getAllExuluVariables = async () => {
|
|
5430
5462
|
const { db: db2 } = await postgresClient();
|
|
@@ -5842,9 +5874,9 @@ Probe error: ${probe.reason ?? "(no detail)"}`
|
|
|
5842
5874
|
});
|
|
5843
5875
|
const writeFileTool = tool2({
|
|
5844
5876
|
description: 'Write content to a file in the sandbox. Creates parent directories if needed. Paths are always resolved against the session sandbox root \u2014 both relative paths ("skills/foo.md") and leading-slash paths ("/skills/foo.md") work and reach the same file. When the path is under the session artifact tree, the file is also uploaded to S3 and a short-lived presigned URL is returned in the tool output.',
|
|
5845
|
-
inputSchema:
|
|
5846
|
-
path:
|
|
5847
|
-
content:
|
|
5877
|
+
inputSchema: z9.object({
|
|
5878
|
+
path: z9.string().describe("The path where the file should be written. Relative paths and leading-slash paths are both resolved against the session sandbox root."),
|
|
5879
|
+
content: z9.string().describe("The content to write to the file")
|
|
5848
5880
|
}),
|
|
5849
5881
|
execute: async ({ path, content }) => {
|
|
5850
5882
|
const resolvedPath = resolveSessionPath(path, sessionDir);
|
|
@@ -5863,8 +5895,8 @@ Probe error: ${probe.reason ?? "(no detail)"}`
|
|
|
5863
5895
|
});
|
|
5864
5896
|
const readFileTool = tool2({
|
|
5865
5897
|
description: 'Read the contents of a file from the sandbox. Paths are always resolved against the session sandbox root \u2014 both relative paths ("skills/foo.md") and leading-slash paths ("/skills/foo.md") work and reach the same file. If the file does not exist, the error message is surfaced verbatim.',
|
|
5866
|
-
inputSchema:
|
|
5867
|
-
path:
|
|
5898
|
+
inputSchema: z9.object({
|
|
5899
|
+
path: z9.string().describe("The path of the file to read. Relative paths and leading-slash paths are both resolved against the session sandbox root.")
|
|
5868
5900
|
}),
|
|
5869
5901
|
execute: async ({ path }) => {
|
|
5870
5902
|
const resolvedPath = resolveSessionPath(path, sessionDir);
|
|
@@ -5875,8 +5907,8 @@ Probe error: ${probe.reason ?? "(no detail)"}`
|
|
|
5875
5907
|
const originalBashTool = tools.bash;
|
|
5876
5908
|
const bashTool = tool2({
|
|
5877
5909
|
description: originalBashTool.description ?? "",
|
|
5878
|
-
inputSchema:
|
|
5879
|
-
command:
|
|
5910
|
+
inputSchema: z9.object({
|
|
5911
|
+
command: z9.string().describe("The bash command to execute.")
|
|
5880
5912
|
}),
|
|
5881
5913
|
execute: async (args, opts) => {
|
|
5882
5914
|
const before = persistenceEnabled ? await snapshotSessionArtifacts() : null;
|
|
@@ -5948,9 +5980,9 @@ ${lines.join("\n")}`;
|
|
|
5948
5980
|
}
|
|
5949
5981
|
|
|
5950
5982
|
// src/utils/truncate-tool-output.ts
|
|
5951
|
-
var truncateToolOutput = (output, maxContextLength, toolName, tailFraction = 0.1) => {
|
|
5983
|
+
var truncateToolOutput = (output, maxContextLength, toolName, tailFraction = 0.1, charLimitOverride) => {
|
|
5952
5984
|
const effectiveCtx = maxContextLength != null && maxContextLength > 0 ? maxContextLength : 128e3;
|
|
5953
|
-
const charLimit = Math.floor(effectiveCtx * 0.25 * 4);
|
|
5985
|
+
const charLimit = charLimitOverride != null && charLimitOverride > 0 ? charLimitOverride : Math.floor(effectiveCtx * 0.25 * 4);
|
|
5954
5986
|
const clampedTail = Math.min(1, Math.max(0, tailFraction));
|
|
5955
5987
|
if (output.length <= charLimit) return output;
|
|
5956
5988
|
const headChars = Math.floor(charLimit * (1 - clampedTail));
|
|
@@ -5970,8 +6002,230 @@ var truncateToolOutput = (output, maxContextLength, toolName, tailFraction = 0.1
|
|
|
5970
6002
|
return head + marker + tail;
|
|
5971
6003
|
};
|
|
5972
6004
|
|
|
6005
|
+
// src/exulu/tool-output-offload.ts
|
|
6006
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
6007
|
+
|
|
6008
|
+
// src/exulu/context-budget.ts
|
|
6009
|
+
var DEFAULT_CONTEXT_WINDOW = 128e3;
|
|
6010
|
+
var deriveContextBudget = (contextWindowInput) => {
|
|
6011
|
+
const contextWindow = contextWindowInput != null && contextWindowInput > 0 ? contextWindowInput : DEFAULT_CONTEXT_WINDOW;
|
|
6012
|
+
const outputReserve = Math.min(32e3, Math.floor(contextWindow * 0.2));
|
|
6013
|
+
const usableWindow = contextWindow - outputReserve;
|
|
6014
|
+
return {
|
|
6015
|
+
contextWindow,
|
|
6016
|
+
outputReserve,
|
|
6017
|
+
usableWindow,
|
|
6018
|
+
warnThreshold: Math.floor(usableWindow * 0.8),
|
|
6019
|
+
blockThreshold: Math.floor(usableWindow * 0.95),
|
|
6020
|
+
toolOutputCapTokens: Math.min(25e3, Math.max(4e3, Math.floor(contextWindow * 0.1))),
|
|
6021
|
+
compactionTailTokens: Math.floor(usableWindow * 0.1),
|
|
6022
|
+
summaryBudgetTokens: Math.min(8e3, Math.floor(usableWindow * 0.05))
|
|
6023
|
+
};
|
|
6024
|
+
};
|
|
6025
|
+
var estimateTokens = (text) => text ? Math.ceil(text.length / 4) : 0;
|
|
6026
|
+
var estimateMessageTokens = (message) => estimateTokens(JSON.stringify(message));
|
|
6027
|
+
var getCompaction = (message) => message.metadata?.compaction;
|
|
6028
|
+
var sliceHistoryAtCheckpoint = (messages) => {
|
|
6029
|
+
let checkpointIdx = -1;
|
|
6030
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
6031
|
+
if (getCompaction(messages[i])) {
|
|
6032
|
+
checkpointIdx = i;
|
|
6033
|
+
break;
|
|
6034
|
+
}
|
|
6035
|
+
}
|
|
6036
|
+
if (checkpointIdx === -1) return messages;
|
|
6037
|
+
const checkpoint = messages[checkpointIdx];
|
|
6038
|
+
const coversUpTo = getCompaction(checkpoint).coversUpTo;
|
|
6039
|
+
const coversIdx = messages.findIndex((m) => m.id === coversUpTo);
|
|
6040
|
+
const boundary = coversIdx === -1 ? checkpointIdx : coversIdx;
|
|
6041
|
+
const after = messages.filter((m, i) => i > boundary && i !== checkpointIdx);
|
|
6042
|
+
return [checkpoint, ...after];
|
|
6043
|
+
};
|
|
6044
|
+
var contextOccupancy = (messages) => {
|
|
6045
|
+
let anchorIdx = -1;
|
|
6046
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
6047
|
+
const m = messages[i];
|
|
6048
|
+
const meta = m.metadata;
|
|
6049
|
+
if (getCompaction(m) || m.role === "assistant" && (typeof meta?.inputTokens === "number" || typeof meta?.lastStepInputTokens === "number")) {
|
|
6050
|
+
anchorIdx = i;
|
|
6051
|
+
break;
|
|
6052
|
+
}
|
|
6053
|
+
}
|
|
6054
|
+
let total = 0;
|
|
6055
|
+
let rest = messages;
|
|
6056
|
+
if (anchorIdx !== -1) {
|
|
6057
|
+
const anchor = messages[anchorIdx];
|
|
6058
|
+
const compaction = getCompaction(anchor);
|
|
6059
|
+
if (compaction) {
|
|
6060
|
+
total = compaction.occupancyEstimate;
|
|
6061
|
+
} else {
|
|
6062
|
+
const meta = anchor.metadata;
|
|
6063
|
+
total = typeof meta.lastStepInputTokens === "number" ? meta.lastStepInputTokens + (meta.lastStepOutputTokens ?? 0) : (meta.inputTokens ?? 0) + (meta.outputTokens ?? 0);
|
|
6064
|
+
}
|
|
6065
|
+
rest = messages.slice(anchorIdx + 1);
|
|
6066
|
+
}
|
|
6067
|
+
for (const m of rest) total += estimateMessageTokens(m);
|
|
6068
|
+
return total;
|
|
6069
|
+
};
|
|
6070
|
+
var CONTEXT_COMPACTION_REQUIRED = "CONTEXT_COMPACTION_REQUIRED";
|
|
6071
|
+
var COMPACTION_INSUFFICIENT = "COMPACTION_INSUFFICIENT";
|
|
6072
|
+
var ContextCompactionRequiredError = class extends Error {
|
|
6073
|
+
constructor(occupancy, budget) {
|
|
6074
|
+
super(
|
|
6075
|
+
JSON.stringify({
|
|
6076
|
+
code: CONTEXT_COMPACTION_REQUIRED,
|
|
6077
|
+
message: `This conversation no longer fits the model's context window (~${occupancy.toLocaleString("en-US")} of ${budget.usableWindow.toLocaleString("en-US")} usable tokens). Compact the conversation to continue.`,
|
|
6078
|
+
occupancy,
|
|
6079
|
+
usableWindow: budget.usableWindow,
|
|
6080
|
+
contextWindow: budget.contextWindow
|
|
6081
|
+
})
|
|
6082
|
+
);
|
|
6083
|
+
this.occupancy = occupancy;
|
|
6084
|
+
this.budget = budget;
|
|
6085
|
+
this.name = "ContextCompactionRequiredError";
|
|
6086
|
+
}
|
|
6087
|
+
};
|
|
6088
|
+
var PROVIDER_CONTEXT_ERROR_PATTERNS = [
|
|
6089
|
+
/ContextWindowExceededError/i,
|
|
6090
|
+
/context.?window/i,
|
|
6091
|
+
/context.?length/i,
|
|
6092
|
+
/maximum context/i,
|
|
6093
|
+
/prompt is too long/i,
|
|
6094
|
+
/input is too long/i,
|
|
6095
|
+
/token count exceeds/i,
|
|
6096
|
+
/too many tokens/i
|
|
6097
|
+
];
|
|
6098
|
+
var isProviderContextLengthError = (message) => PROVIDER_CONTEXT_ERROR_PATTERNS.some((re) => re.test(message));
|
|
6099
|
+
var mapStreamErrorMessage = (message) => isProviderContextLengthError(message) ? JSON.stringify({
|
|
6100
|
+
code: CONTEXT_COMPACTION_REQUIRED,
|
|
6101
|
+
message: "The model rejected the request because the conversation exceeds its context window. Compact the conversation to continue.",
|
|
6102
|
+
providerMessage: message.slice(0, 500)
|
|
6103
|
+
}) : message;
|
|
6104
|
+
|
|
6105
|
+
// src/exulu/tool-output-offload.ts
|
|
6106
|
+
var PREVIEW_CHARS = 4e3;
|
|
6107
|
+
var storeAsSessionFile = async (serialized, ctx) => {
|
|
6108
|
+
if (!ctx.sessionID || !ctx.exuluConfig?.fileUploads) return void 0;
|
|
6109
|
+
const safeTool = ctx.toolName.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 40);
|
|
6110
|
+
const name = `tool-output-${safeTool}-${randomUUID3().slice(0, 8)}.txt`;
|
|
6111
|
+
try {
|
|
6112
|
+
await uploadFile(
|
|
6113
|
+
Buffer.from(serialized, "utf-8"),
|
|
6114
|
+
`sessions/${ctx.sessionID}/${name}`,
|
|
6115
|
+
ctx.exuluConfig,
|
|
6116
|
+
{ contentType: "text/plain" },
|
|
6117
|
+
ctx.user?.id
|
|
6118
|
+
);
|
|
6119
|
+
return name;
|
|
6120
|
+
} catch (err) {
|
|
6121
|
+
console.error("[EXULU] Failed to offload oversized tool output to session files.", err);
|
|
6122
|
+
return void 0;
|
|
6123
|
+
}
|
|
6124
|
+
};
|
|
6125
|
+
var buildNotice = (tokens, capTokens, sessionFile) => sessionFile ? `Tool output truncated: ~${tokens.toLocaleString("en-US")} tokens (limit ${capTokens.toLocaleString("en-US")}). The FULL output is saved as session file "${sessionFile}" \u2014 call read_session_file with { filename: "${sessionFile}", offset, limit } to read specific line ranges.` : `Tool output truncated: ~${tokens.toLocaleString("en-US")} tokens (limit ${capTokens.toLocaleString("en-US")}). The remainder was discarded \u2014 re-run the tool with narrower arguments.`;
|
|
6126
|
+
var guardToolOutput = async (value, ctx) => {
|
|
6127
|
+
if (value == null) return value;
|
|
6128
|
+
let serialized;
|
|
6129
|
+
try {
|
|
6130
|
+
serialized = typeof value === "string" ? value : JSON.stringify(value);
|
|
6131
|
+
} catch {
|
|
6132
|
+
return value;
|
|
6133
|
+
}
|
|
6134
|
+
if (typeof serialized !== "string") return value;
|
|
6135
|
+
const budget = deriveContextBudget(ctx.contextWindow);
|
|
6136
|
+
const tokens = estimateTokens(serialized);
|
|
6137
|
+
if (tokens <= budget.toolOutputCapTokens) return value;
|
|
6138
|
+
const sessionFile = await storeAsSessionFile(serialized, ctx);
|
|
6139
|
+
const result = {
|
|
6140
|
+
truncated: true,
|
|
6141
|
+
notice: buildNotice(tokens, budget.toolOutputCapTokens, sessionFile),
|
|
6142
|
+
...sessionFile ? { sessionFile } : {},
|
|
6143
|
+
preview: serialized.slice(0, PREVIEW_CHARS)
|
|
6144
|
+
};
|
|
6145
|
+
return result;
|
|
6146
|
+
};
|
|
6147
|
+
var guardExtractedFileText = async (filename, text, ctx) => {
|
|
6148
|
+
const budget = deriveContextBudget(ctx.contextWindow);
|
|
6149
|
+
const tokens = estimateTokens(text);
|
|
6150
|
+
if (tokens <= budget.toolOutputCapTokens) return text;
|
|
6151
|
+
const sessionFile = await storeAsSessionFile(text, { ...ctx, toolName: `upload-${filename}` });
|
|
6152
|
+
const notice = sessionFile ? `[Document "${filename}" truncated: ~${tokens.toLocaleString("en-US")} tokens. The full extracted text is saved as session file "${sessionFile}" \u2014 read specific parts with read_session_file (offset/limit).]` : `[Document "${filename}" truncated: ~${tokens.toLocaleString("en-US")} tokens \u2014 the remainder is unavailable.]`;
|
|
6153
|
+
return `${text.slice(0, PREVIEW_CHARS)}
|
|
6154
|
+
|
|
6155
|
+
${notice}`;
|
|
6156
|
+
};
|
|
6157
|
+
|
|
6158
|
+
// src/templates/tools/session-file-read-tool.ts
|
|
6159
|
+
import { z as z10 } from "zod";
|
|
6160
|
+
var DEFAULT_LIMIT = 250;
|
|
6161
|
+
var MAX_CONTENT_CHARS = 16e3;
|
|
6162
|
+
var createSessionFileReadTool = ({
|
|
6163
|
+
sessionID,
|
|
6164
|
+
user,
|
|
6165
|
+
exuluConfig
|
|
6166
|
+
}) => {
|
|
6167
|
+
if (!sessionID || !exuluConfig?.fileUploads?.s3Bucket) return void 0;
|
|
6168
|
+
const readSessionFileExecute = async ({ filename, offset, limit }) => {
|
|
6169
|
+
const safeName = String(filename ?? "").trim();
|
|
6170
|
+
if (!safeName || safeName.includes("..") || safeName.includes("/") || safeName.includes("\\")) {
|
|
6171
|
+
return {
|
|
6172
|
+
error: "Invalid filename \u2014 pass the bare file name exactly as listed in the session files (no paths)."
|
|
6173
|
+
};
|
|
6174
|
+
}
|
|
6175
|
+
const uploads = exuluConfig.fileUploads;
|
|
6176
|
+
const generalPrefix = uploads.s3prefix ? `${uploads.s3prefix.replace(/\/$/, "")}/` : "";
|
|
6177
|
+
const key = `${generalPrefix}user_${user?.id ?? "api"}/sessions/${sessionID}/${safeName}`;
|
|
6178
|
+
try {
|
|
6179
|
+
const url = await getPresignedUrl(uploads.s3Bucket, key, exuluConfig);
|
|
6180
|
+
const res = await fetch(url);
|
|
6181
|
+
if (!res.ok) {
|
|
6182
|
+
return { error: `Could not read session file "${safeName}" (status ${res.status}). Check the exact file name.` };
|
|
6183
|
+
}
|
|
6184
|
+
const textBody = await res.text();
|
|
6185
|
+
const lines = textBody.split("\n");
|
|
6186
|
+
const start = (offset ?? 1) - 1;
|
|
6187
|
+
const requested = limit ?? DEFAULT_LIMIT;
|
|
6188
|
+
const sliced = lines.slice(start, start + requested);
|
|
6189
|
+
let content = sliced.join("\n");
|
|
6190
|
+
let linesReturned = sliced.length;
|
|
6191
|
+
if (content.length > MAX_CONTENT_CHARS) {
|
|
6192
|
+
content = content.slice(0, MAX_CONTENT_CHARS);
|
|
6193
|
+
linesReturned = Math.max(1, content.split("\n").length - 1);
|
|
6194
|
+
content = content + "\n[slice truncated \u2014 request fewer lines]";
|
|
6195
|
+
}
|
|
6196
|
+
return {
|
|
6197
|
+
content,
|
|
6198
|
+
totalLines: lines.length,
|
|
6199
|
+
offset: start + 1,
|
|
6200
|
+
linesReturned
|
|
6201
|
+
};
|
|
6202
|
+
} catch (err) {
|
|
6203
|
+
return { error: `Failed to read session file "${safeName}": ${err instanceof Error ? err.message : "unknown error"}` };
|
|
6204
|
+
}
|
|
6205
|
+
};
|
|
6206
|
+
return ExuluTool.internal({
|
|
6207
|
+
id: "read_session_file",
|
|
6208
|
+
name: "read_session_file",
|
|
6209
|
+
needsApproval: false,
|
|
6210
|
+
description: "Read a line range from a file stored in this session's files \u2014 including offloaded tool outputs (tool-output-*.txt) and uploaded documents. Use offset (1-based line number) and limit to page through large files instead of reading everything at once.",
|
|
6211
|
+
inputSchema: z10.object({
|
|
6212
|
+
filename: z10.string().describe('Exact session file name as referenced in a truncation notice, e.g. "tool-output-web_search-a1b2c3d4.txt"'),
|
|
6213
|
+
offset: z10.number().int().min(1).optional().describe("1-based first line to read (default 1)"),
|
|
6214
|
+
limit: z10.number().int().min(1).max(1e3).optional().describe(`Number of lines to read (default ${DEFAULT_LIMIT})`)
|
|
6215
|
+
}),
|
|
6216
|
+
type: "function",
|
|
6217
|
+
category: "session",
|
|
6218
|
+
config: [],
|
|
6219
|
+
// ExuluTool's execute type is modeled on retrieval tools ({result/job/items});
|
|
6220
|
+
// internal utility tools return richer shapes (memory-tool has the same
|
|
6221
|
+
// mismatch). The AI SDK passes the object through verbatim, so cast.
|
|
6222
|
+
execute: readSessionFileExecute
|
|
6223
|
+
});
|
|
6224
|
+
};
|
|
6225
|
+
|
|
5973
6226
|
// src/templates/tools/convert-exulu-tools-to-ai-sdk-tools.ts
|
|
5974
|
-
var
|
|
6227
|
+
var OUTPUT_OFFLOAD_EXEMPT_TOOL_IDS = /* @__PURE__ */ new Set(["agentic_context_search"]);
|
|
6228
|
+
var generateS3Key = (filename) => `${randomUUID4()}-${filename}`;
|
|
5975
6229
|
var s3Client2;
|
|
5976
6230
|
var getMimeType = (type) => {
|
|
5977
6231
|
switch (type) {
|
|
@@ -6069,7 +6323,7 @@ var hydrateVariables = async (tool3) => {
|
|
|
6069
6323
|
await Promise.all(promises);
|
|
6070
6324
|
return tool3;
|
|
6071
6325
|
};
|
|
6072
|
-
var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approvedTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, sessionID, req, project, sessionItems, model, agent, memoryItems) => {
|
|
6326
|
+
var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approvedTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, sessionID, req, project, sessionItems, model, agent, memoryItems, contextWindow, disabledTools) => {
|
|
6073
6327
|
if (!currentTools) return {};
|
|
6074
6328
|
if (!allExuluTools) {
|
|
6075
6329
|
allExuluTools = [];
|
|
@@ -6077,6 +6331,8 @@ var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approved
|
|
|
6077
6331
|
if (!contexts) {
|
|
6078
6332
|
contexts = [];
|
|
6079
6333
|
}
|
|
6334
|
+
const budget = deriveContextBudget(contextWindow);
|
|
6335
|
+
const toolOutputCharLimit = budget.toolOutputCapTokens * 4;
|
|
6080
6336
|
let sharedSessionSandbox;
|
|
6081
6337
|
if (sessionID && exuluConfig && agent?.sandbox_enabled === true) {
|
|
6082
6338
|
try {
|
|
@@ -6093,16 +6349,28 @@ var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approved
|
|
|
6093
6349
|
);
|
|
6094
6350
|
}
|
|
6095
6351
|
}
|
|
6096
|
-
|
|
6097
|
-
|
|
6098
|
-
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6352
|
+
const disabled = new Set(disabledTools ?? []);
|
|
6353
|
+
let projectScope;
|
|
6354
|
+
if (project && !disabled.has("agentic_context_search")) {
|
|
6355
|
+
const { db: db2 } = await postgresClient();
|
|
6356
|
+
const projectRow = await db2.from("projects").where("id", project).first();
|
|
6357
|
+
let rawItems = projectRow?.project_items;
|
|
6358
|
+
if (typeof rawItems === "string") {
|
|
6359
|
+
try {
|
|
6360
|
+
rawItems = JSON.parse(rawItems);
|
|
6361
|
+
} catch {
|
|
6362
|
+
rawItems = void 0;
|
|
6363
|
+
}
|
|
6364
|
+
}
|
|
6365
|
+
if (projectRow && Array.isArray(rawItems) && rawItems.length > 0) {
|
|
6366
|
+
projectScope = {
|
|
6367
|
+
id: projectRow.id,
|
|
6368
|
+
name: projectRow.name,
|
|
6369
|
+
description: projectRow.description ?? void 0,
|
|
6370
|
+
customInstructions: projectRow.custom_instructions ?? void 0,
|
|
6371
|
+
items: rawItems,
|
|
6372
|
+
kbProfileDefaults: buildProjectKbProfileDefaults(rawItems)
|
|
6373
|
+
};
|
|
6106
6374
|
}
|
|
6107
6375
|
}
|
|
6108
6376
|
if (agent?.memory && contexts?.length) {
|
|
@@ -6113,7 +6381,7 @@ var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approved
|
|
|
6113
6381
|
);
|
|
6114
6382
|
}
|
|
6115
6383
|
const createNewMemoryTool = createNewMemoryItemTool(agent, context);
|
|
6116
|
-
if (createNewMemoryTool) {
|
|
6384
|
+
if (createNewMemoryTool && !disabled.has(createNewMemoryTool.id)) {
|
|
6117
6385
|
if (!currentTools) {
|
|
6118
6386
|
currentTools = [];
|
|
6119
6387
|
}
|
|
@@ -6128,31 +6396,62 @@ var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approved
|
|
|
6128
6396
|
contexts,
|
|
6129
6397
|
items: sessionItems
|
|
6130
6398
|
});
|
|
6131
|
-
if (sessionItemsRetrievalTool) {
|
|
6399
|
+
if (sessionItemsRetrievalTool && !disabled.has(sessionItemsRetrievalTool.id)) {
|
|
6132
6400
|
currentTools.push(sessionItemsRetrievalTool);
|
|
6133
6401
|
}
|
|
6134
6402
|
}
|
|
6403
|
+
const sessionFileReadTool = createSessionFileReadTool({ sessionID, user, exuluConfig });
|
|
6404
|
+
if (sessionFileReadTool && !disabled.has(sessionFileReadTool.id)) {
|
|
6405
|
+
currentTools.push(sessionFileReadTool);
|
|
6406
|
+
}
|
|
6135
6407
|
console.log("[EXULU] Creating agentic search tool", contexts?.length, model);
|
|
6136
|
-
if (contexts?.length && model) {
|
|
6137
|
-
const
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6408
|
+
if (contexts?.length && model && !disabled.has("agentic_context_search")) {
|
|
6409
|
+
const index = currentTools.findIndex((tool3) => tool3.id === "agentic_context_search");
|
|
6410
|
+
const memoryContext = agent?.memory ? contexts.find((c) => c.id === agent.memory) : void 0;
|
|
6411
|
+
if (index !== -1) {
|
|
6412
|
+
const agenticSearchTool = createAgenticRetrievalTool({
|
|
6413
|
+
contexts: contexts.filter((context) => context.id !== agent?.memory),
|
|
6414
|
+
// memory is searched by the memory phase, not as a KB
|
|
6415
|
+
memoryContext,
|
|
6416
|
+
user,
|
|
6417
|
+
role: user?.role?.id,
|
|
6418
|
+
model,
|
|
6419
|
+
preselected: sessionItems,
|
|
6420
|
+
memoryItems,
|
|
6421
|
+
projectScope
|
|
6422
|
+
});
|
|
6423
|
+
if (agenticSearchTool) {
|
|
6150
6424
|
currentTools[index] = {
|
|
6151
6425
|
...currentTools[index],
|
|
6152
6426
|
// important to keep the original tool config
|
|
6153
6427
|
...agenticSearchTool
|
|
6154
6428
|
};
|
|
6155
6429
|
}
|
|
6430
|
+
} else if (projectScope) {
|
|
6431
|
+
const projectContextIds = new Set(
|
|
6432
|
+
projectScope.items.map((gid) => {
|
|
6433
|
+
const i = gid.indexOf("/");
|
|
6434
|
+
return i === -1 ? gid : gid.slice(0, i);
|
|
6435
|
+
})
|
|
6436
|
+
);
|
|
6437
|
+
const scopedContexts = contexts.filter(
|
|
6438
|
+
(c) => projectContextIds.has(c.id) && c.id !== agent?.memory
|
|
6439
|
+
);
|
|
6440
|
+
if (scopedContexts.length > 0) {
|
|
6441
|
+
const projectSearchTool = createAgenticRetrievalTool({
|
|
6442
|
+
contexts: scopedContexts,
|
|
6443
|
+
memoryContext,
|
|
6444
|
+
user,
|
|
6445
|
+
role: user?.role?.id,
|
|
6446
|
+
model,
|
|
6447
|
+
preselected: [...sessionItems ?? [], ...projectScope.items],
|
|
6448
|
+
memoryItems,
|
|
6449
|
+
projectScope
|
|
6450
|
+
});
|
|
6451
|
+
if (projectSearchTool) {
|
|
6452
|
+
currentTools.push(projectSearchTool);
|
|
6453
|
+
}
|
|
6454
|
+
}
|
|
6156
6455
|
}
|
|
6157
6456
|
} else {
|
|
6158
6457
|
const agenticSearchTool = currentTools.find((tool3) => tool3.id === "agentic_context_search");
|
|
@@ -6180,7 +6479,7 @@ var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approved
|
|
|
6180
6479
|
if (typeof result?.content === "string") {
|
|
6181
6480
|
return {
|
|
6182
6481
|
...result,
|
|
6183
|
-
content: truncateToolOutput(result.content,
|
|
6482
|
+
content: truncateToolOutput(result.content, budget.contextWindow, "readFile", 0.05, toolOutputCharLimit)
|
|
6184
6483
|
};
|
|
6185
6484
|
}
|
|
6186
6485
|
return result;
|
|
@@ -6197,10 +6496,10 @@ var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approved
|
|
|
6197
6496
|
return {
|
|
6198
6497
|
...result,
|
|
6199
6498
|
...typeof result?.stdout === "string" && {
|
|
6200
|
-
stdout: truncateToolOutput(result.stdout,
|
|
6499
|
+
stdout: truncateToolOutput(result.stdout, budget.contextWindow, "bash", 0.1, toolOutputCharLimit)
|
|
6201
6500
|
},
|
|
6202
6501
|
...typeof result?.stderr === "string" && {
|
|
6203
|
-
stderr: truncateToolOutput(result.stderr,
|
|
6502
|
+
stderr: truncateToolOutput(result.stderr, budget.contextWindow, "bash stderr", 0.4, toolOutputCharLimit)
|
|
6204
6503
|
}
|
|
6205
6504
|
};
|
|
6206
6505
|
}
|
|
@@ -6336,16 +6635,30 @@ var convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approved
|
|
|
6336
6635
|
user: user?.id,
|
|
6337
6636
|
role: user?.role?.id
|
|
6338
6637
|
});
|
|
6638
|
+
const guardCtx = {
|
|
6639
|
+
toolName: cur.name,
|
|
6640
|
+
contextWindow,
|
|
6641
|
+
sessionID,
|
|
6642
|
+
user,
|
|
6643
|
+
exuluConfig
|
|
6644
|
+
};
|
|
6645
|
+
const offloadExempt = OUTPUT_OFFLOAD_EXEMPT_TOOL_IDS.has(cur.id);
|
|
6339
6646
|
if (response && typeof response === "object" && Symbol.asyncIterator in response) {
|
|
6340
6647
|
let lastValue;
|
|
6341
6648
|
for await (const value of response) {
|
|
6342
6649
|
yield value;
|
|
6343
6650
|
lastValue = value;
|
|
6344
6651
|
}
|
|
6345
|
-
return lastValue;
|
|
6652
|
+
if (offloadExempt) return lastValue;
|
|
6653
|
+
const guarded = await guardToolOutput(lastValue, guardCtx);
|
|
6654
|
+
if (guarded !== lastValue) {
|
|
6655
|
+
yield guarded;
|
|
6656
|
+
}
|
|
6657
|
+
return guarded;
|
|
6346
6658
|
} else {
|
|
6347
|
-
|
|
6348
|
-
|
|
6659
|
+
const guarded = offloadExempt ? response : await guardToolOutput(response, guardCtx);
|
|
6660
|
+
yield guarded;
|
|
6661
|
+
return guarded;
|
|
6349
6662
|
}
|
|
6350
6663
|
}
|
|
6351
6664
|
}
|
|
@@ -6387,6 +6700,7 @@ export {
|
|
|
6387
6700
|
listTagsByPrefix,
|
|
6388
6701
|
getBudgetSettings,
|
|
6389
6702
|
setBudgetSettings,
|
|
6703
|
+
parseResetAt,
|
|
6390
6704
|
upsertBudget,
|
|
6391
6705
|
invalidateBudgetCaches,
|
|
6392
6706
|
getTagBudgetMap,
|
|
@@ -6403,10 +6717,20 @@ export {
|
|
|
6403
6717
|
OAUTH_CALLBACK_PATH,
|
|
6404
6718
|
decryptOauthState,
|
|
6405
6719
|
exchangeCodeForTokens,
|
|
6406
|
-
createProjectItemsRetrievalTool,
|
|
6407
6720
|
sanitizeToolName,
|
|
6408
6721
|
reportSystemDependencies,
|
|
6409
6722
|
downloadKeyIntoSandbox,
|
|
6723
|
+
truncateToolOutput,
|
|
6724
|
+
DEFAULT_CONTEXT_WINDOW,
|
|
6725
|
+
deriveContextBudget,
|
|
6726
|
+
estimateTokens,
|
|
6727
|
+
estimateMessageTokens,
|
|
6728
|
+
sliceHistoryAtCheckpoint,
|
|
6729
|
+
contextOccupancy,
|
|
6730
|
+
COMPACTION_INSUFFICIENT,
|
|
6731
|
+
ContextCompactionRequiredError,
|
|
6732
|
+
mapStreamErrorMessage,
|
|
6733
|
+
guardExtractedFileText,
|
|
6410
6734
|
hydrateVariables,
|
|
6411
6735
|
convertExuluToolsToAiSdkTools,
|
|
6412
6736
|
ExuluTool,
|