@exulu/backend 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-IJ4HNHOT.js → chunk-RVZWZNWG.js} +568 -274
- package/dist/{convert-exulu-tools-to-ai-sdk-tools-2PEDFZ2X.js → convert-exulu-tools-to-ai-sdk-tools-K3RHHLN6.js} +1 -1
- package/dist/index.cjs +1274 -437
- package/dist/index.d.cts +37 -2
- package/dist/index.d.ts +37 -2
- package/dist/index.js +720 -211
- package/ee/agentic-retrieval/pipeline/config.test.ts +15 -0
- package/ee/agentic-retrieval/pipeline/config.ts +6 -0
- package/ee/agentic-retrieval/pipeline/global-ids.ts +30 -0
- package/ee/agentic-retrieval/pipeline/index.test.ts +86 -1
- package/ee/agentic-retrieval/pipeline/index.ts +96 -46
- package/ee/agentic-retrieval/pipeline/project-scope.test.ts +73 -0
- package/ee/agentic-retrieval/pipeline/project-scope.ts +77 -0
- package/ee/agentic-retrieval/pipeline/search.test.ts +27 -0
- package/ee/agentic-retrieval/pipeline/search.ts +7 -0
- package/package.json +1 -1
|
@@ -79,3 +79,18 @@ describe("effectiveKbSettings", () => {
|
|
|
79
79
|
expect(s.kind).toBe("documents");
|
|
80
80
|
});
|
|
81
81
|
});
|
|
82
|
+
|
|
83
|
+
describe("project_search option", () => {
|
|
84
|
+
it("defaults to true when absent or empty (empty string = backend default)", () => {
|
|
85
|
+
expect(parsePipelineConfig({}).projectSearch).toBe(true);
|
|
86
|
+
expect(parsePipelineConfig(undefined).projectSearch).toBe(true);
|
|
87
|
+
expect(parsePipelineConfig({ project_search: "" }).projectSearch).toBe(true);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("parses explicit values", () => {
|
|
91
|
+
expect(parsePipelineConfig({ project_search: "false" }).projectSearch).toBe(false);
|
|
92
|
+
expect(parsePipelineConfig({ project_search: false }).projectSearch).toBe(false);
|
|
93
|
+
expect(parsePipelineConfig({ project_search: "true" }).projectSearch).toBe(true);
|
|
94
|
+
expect(parsePipelineConfig({ project_search: true }).projectSearch).toBe(true);
|
|
95
|
+
});
|
|
96
|
+
});
|
|
@@ -69,6 +69,8 @@ export type PipelineConfig = {
|
|
|
69
69
|
managedContext: boolean;
|
|
70
70
|
requirePreselectedContexts: boolean;
|
|
71
71
|
logging: boolean;
|
|
72
|
+
/** Search items attached to the chat's project as an additional source. Default true. */
|
|
73
|
+
projectSearch: boolean;
|
|
72
74
|
utilityModel: string;
|
|
73
75
|
knowledgeBases: Record<string, KbProfile>;
|
|
74
76
|
routing: z.infer<typeof routingSchema>;
|
|
@@ -132,6 +134,10 @@ export function parsePipelineConfig(raw?: Record<string, unknown>): PipelineConf
|
|
|
132
134
|
managedContext: boolVal(r["managed_context"]),
|
|
133
135
|
requirePreselectedContexts: boolVal(r["require_preselected_contexts"]),
|
|
134
136
|
logging: boolVal(r["logging"]),
|
|
137
|
+
projectSearch:
|
|
138
|
+
r["project_search"] === undefined || r["project_search"] === ""
|
|
139
|
+
? true
|
|
140
|
+
: boolVal(r["project_search"]),
|
|
135
141
|
utilityModel: strVal(r["utility_model"], ""),
|
|
136
142
|
knowledgeBases: jsonVal("knowledge_bases", knowledgeBasesSchema, r["knowledge_bases"]),
|
|
137
143
|
routing: jsonVal("routing", routingSchema, r["routing"]),
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse a list of global preselected IDs into a per-context map.
|
|
3
|
+
*
|
|
4
|
+
* Two supported formats:
|
|
5
|
+
* "<context_id>/<item_id>" → specific item; value is a non-empty string[]
|
|
6
|
+
* "<context_id>" → full context (no item filter); value is null
|
|
7
|
+
*
|
|
8
|
+
* If both a full-context entry and specific-item entries exist for the same
|
|
9
|
+
* context, full-context (null) wins.
|
|
10
|
+
*/
|
|
11
|
+
export function parsePreselectedItems(globalIds: string[]): Map<string, string[] | null> {
|
|
12
|
+
const map = new Map<string, string[] | null>();
|
|
13
|
+
for (const gid of globalIds) {
|
|
14
|
+
const slashIdx = gid.indexOf("/");
|
|
15
|
+
if (slashIdx === -1) {
|
|
16
|
+
// No slash → entire context selected
|
|
17
|
+
if (gid) map.set(gid, null);
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
const contextId = gid.slice(0, slashIdx);
|
|
21
|
+
const itemId = gid.slice(slashIdx + 1);
|
|
22
|
+
if (!contextId || !itemId) continue;
|
|
23
|
+
// Full-context entry already wins — don't downgrade to specific items
|
|
24
|
+
if (map.get(contextId) === null) continue;
|
|
25
|
+
const existing = map.get(contextId) ?? [];
|
|
26
|
+
existing.push(itemId);
|
|
27
|
+
map.set(contextId, existing);
|
|
28
|
+
}
|
|
29
|
+
return map;
|
|
30
|
+
}
|
|
@@ -37,7 +37,7 @@ describe("createAgenticRetrievalTool", () => {
|
|
|
37
37
|
const names = tool.config.map((c) => c.name).sort();
|
|
38
38
|
expect(names).toEqual([
|
|
39
39
|
"instructions", "knowledge_bases", "logging", "managed_context", "memory",
|
|
40
|
-
"max_steps", "require_preselected_contexts", "reranker", "routing", "tuning", "utility_model", "vocabulary",
|
|
40
|
+
"max_steps", "project_search", "require_preselected_contexts", "reranker", "routing", "tuning", "utility_model", "vocabulary",
|
|
41
41
|
].sort());
|
|
42
42
|
expect(tool.config.filter((c) => c.type === "json").map((c) => c.name).sort())
|
|
43
43
|
.toEqual(["knowledge_bases", "memory", "routing", "tuning", "vocabulary"].sort());
|
|
@@ -138,3 +138,88 @@ describe("parsePreselectedItems", () => {
|
|
|
138
138
|
expect(m.get("b")).toBeNull();
|
|
139
139
|
});
|
|
140
140
|
});
|
|
141
|
+
|
|
142
|
+
describe("projectScope factory surface", () => {
|
|
143
|
+
it("declares the project_search config option with default true", () => {
|
|
144
|
+
const tool = createAgenticRetrievalTool({ contexts: [], user: undefined, role: undefined, model: undefined });
|
|
145
|
+
const entry = tool!.config.find((c: { name: string }) => c.name === "project_search");
|
|
146
|
+
expect(entry).toBeDefined();
|
|
147
|
+
expect(entry!.type).toBe("boolean");
|
|
148
|
+
expect(entry!.default).toBe(true);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it("mentions the attached project in the tool description", () => {
|
|
152
|
+
const tool = createAgenticRetrievalTool({
|
|
153
|
+
contexts: [],
|
|
154
|
+
user: undefined,
|
|
155
|
+
role: undefined,
|
|
156
|
+
model: undefined,
|
|
157
|
+
projectScope: { id: "p1", name: "Modernization", items: ["docs/i1"] },
|
|
158
|
+
});
|
|
159
|
+
expect(tool!.description).toContain('project "Modernization"');
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
describe("projectScope execute-level wiring", () => {
|
|
164
|
+
beforeEach(() => {
|
|
165
|
+
jest.requireMock("./routing").runRoutingPhase.mockClear();
|
|
166
|
+
jest.requireMock("./search").searchContexts.mockClear();
|
|
167
|
+
jest.requireMock("./rerank").rerankResults.mockClear();
|
|
168
|
+
// Restore routing default (some tests override it with mockResolvedValueOnce)
|
|
169
|
+
jest.requireMock("./routing").runRoutingPhase.mockResolvedValue({
|
|
170
|
+
mainContexts: ["docs"], fallbackContexts: [], userPinnedItemIdsByContext: new Map(),
|
|
171
|
+
userRequestedPage: null, hasExplicitDocAndPage: false, steps: [{ text: "routed" }],
|
|
172
|
+
});
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it("gate-off: project_search:false suppresses project instructions, context append, and scopedItemsByContext", async () => {
|
|
176
|
+
const { runRoutingPhase } = jest.requireMock("./routing");
|
|
177
|
+
const { searchContexts } = jest.requireMock("./search");
|
|
178
|
+
const run = makeTool(
|
|
179
|
+
{ project_search: false },
|
|
180
|
+
{ projectScope: { id: "p1", name: "MyProject", customInstructions: "Do X", items: ["tickets/item1"] } },
|
|
181
|
+
);
|
|
182
|
+
const out = await drain(run(inputs));
|
|
183
|
+
// No "Including sources from project" step text in final output
|
|
184
|
+
const lastParsed = JSON.parse(out[out.length - 1].result);
|
|
185
|
+
expect(lastParsed.steps.every((s: any) => !s.text.includes("Including sources from project"))).toBe(true);
|
|
186
|
+
// Routing must NOT receive project custom instructions
|
|
187
|
+
const routingCall = runRoutingPhase.mock.calls[runRoutingPhase.mock.calls.length - 1][0];
|
|
188
|
+
expect(routingCall.extraInstructions ?? "").not.toContain("Instructions for the attached project");
|
|
189
|
+
// searchContexts must not receive any scopedItemsByContext entries
|
|
190
|
+
const mainSearchCall = searchContexts.mock.calls[0][0];
|
|
191
|
+
const scoped: Map<string, unknown> | undefined = mainSearchCall.scopedItemsByContext;
|
|
192
|
+
expect(scoped == null || scoped.size === 0).toBe(true);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it("case-2: enabled-context project items boost rerank pins; non-enabled context is appended and item-scoped", async () => {
|
|
196
|
+
const { searchContexts } = jest.requireMock("./search");
|
|
197
|
+
const { rerankResults } = jest.requireMock("./rerank");
|
|
198
|
+
// tickets disabled in agent config so the project adds it as a scoped source
|
|
199
|
+
const run = makeTool(
|
|
200
|
+
{ knowledge_bases: { tickets: { enabled: false } } },
|
|
201
|
+
{
|
|
202
|
+
projectScope: {
|
|
203
|
+
id: "p1",
|
|
204
|
+
name: "MyProject",
|
|
205
|
+
customInstructions: "Always cite sources",
|
|
206
|
+
items: ["docs/item1", "tickets/item2"],
|
|
207
|
+
},
|
|
208
|
+
},
|
|
209
|
+
);
|
|
210
|
+
const out = await drain(run(inputs));
|
|
211
|
+
const lastParsed = JSON.parse(out[out.length - 1].result);
|
|
212
|
+
// Step announces the appended project context
|
|
213
|
+
expect(lastParsed.steps.some((s: any) => s.text.includes("Including sources from project"))).toBe(true);
|
|
214
|
+
// Main searchContexts call gets the appended context id
|
|
215
|
+
const mainSearchCall = searchContexts.mock.calls[0][0];
|
|
216
|
+
expect(mainSearchCall.contextIds).toContain("tickets");
|
|
217
|
+
// scopedItemsByContext carries the non-enabled context's item ids
|
|
218
|
+
const scoped: Map<string, string[] | null> = mainSearchCall.scopedItemsByContext;
|
|
219
|
+
expect(scoped).toBeDefined();
|
|
220
|
+
expect(scoped.get("tickets")).toEqual(["item2"]);
|
|
221
|
+
// Rerank receives pinnedItemIds that include the enabled context's project item
|
|
222
|
+
const rerankCall = rerankResults.mock.calls[0][0];
|
|
223
|
+
expect(rerankCall.state.pinnedItemIds.has("item1")).toBe(true);
|
|
224
|
+
});
|
|
225
|
+
});
|
|
@@ -8,6 +8,7 @@ import { resolveReranker } from "@SRC/exulu/resolve-reranker";
|
|
|
8
8
|
import { resolveModel } from "@SRC/exulu/resolve-model";
|
|
9
9
|
import { exuluApp } from "@SRC/exulu/app/singleton";
|
|
10
10
|
import { parsePipelineConfig, effectiveKbSettings } from "./config";
|
|
11
|
+
import { resolveProjectScope, type ProjectScope } from "./project-scope";
|
|
11
12
|
import { runRoutingPhase } from "./routing";
|
|
12
13
|
import { runMemoryPhase } from "./memory";
|
|
13
14
|
import { resolveIdentifierPins } from "./prefilter";
|
|
@@ -15,42 +16,8 @@ import { searchContexts } from "./search";
|
|
|
15
16
|
import { rerankResults } from "./rerank";
|
|
16
17
|
import type { AgenticRetrievalOutput, RerankState, ChunkWithScore } from "./types";
|
|
17
18
|
import type { VectorSearchChunkResult } from "@SRC/graphql/resolvers/vector-search";
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
// parsePreselectedItems — verbatim copy of parseGlobalItemIds from v3/tools.ts
|
|
21
|
-
// (renamed for the pipeline public API)
|
|
22
|
-
// ---------------------------------------------------------------------------
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Parse a list of global preselected IDs into a per-context map.
|
|
26
|
-
*
|
|
27
|
-
* Two supported formats:
|
|
28
|
-
* "<context_id>/<item_id>" → specific item; value is a non-empty string[]
|
|
29
|
-
* "<context_id>" → full context (no item filter); value is null
|
|
30
|
-
*
|
|
31
|
-
* If both a full-context entry and specific-item entries exist for the same
|
|
32
|
-
* context, full-context (null) wins.
|
|
33
|
-
*/
|
|
34
|
-
export function parsePreselectedItems(globalIds: string[]): Map<string, string[] | null> {
|
|
35
|
-
const map = new Map<string, string[] | null>();
|
|
36
|
-
for (const gid of globalIds) {
|
|
37
|
-
const slashIdx = gid.indexOf("/");
|
|
38
|
-
if (slashIdx === -1) {
|
|
39
|
-
// No slash → entire context selected
|
|
40
|
-
if (gid) map.set(gid, null);
|
|
41
|
-
continue;
|
|
42
|
-
}
|
|
43
|
-
const contextId = gid.slice(0, slashIdx);
|
|
44
|
-
const itemId = gid.slice(slashIdx + 1);
|
|
45
|
-
if (!contextId || !itemId) continue;
|
|
46
|
-
// Full-context entry already wins — don't downgrade to specific items
|
|
47
|
-
if (map.get(contextId) === null) continue;
|
|
48
|
-
const existing = map.get(contextId) ?? [];
|
|
49
|
-
existing.push(itemId);
|
|
50
|
-
map.set(contextId, existing);
|
|
51
|
-
}
|
|
52
|
-
return map;
|
|
53
|
-
}
|
|
19
|
+
import { parsePreselectedItems } from "./global-ids";
|
|
20
|
+
export { parsePreselectedItems } from "./global-ids";
|
|
54
21
|
|
|
55
22
|
// ---------------------------------------------------------------------------
|
|
56
23
|
// Helpers
|
|
@@ -103,6 +70,7 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
103
70
|
instructions?: string;
|
|
104
71
|
preselected?: string[];
|
|
105
72
|
memoryItems?: VectorSearchChunkResult[];
|
|
73
|
+
projectScope?: ProjectScope;
|
|
106
74
|
}): ExuluTool | undefined {
|
|
107
75
|
const {
|
|
108
76
|
contexts,
|
|
@@ -113,6 +81,7 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
113
81
|
instructions: adminInstructions,
|
|
114
82
|
preselected,
|
|
115
83
|
memoryItems,
|
|
84
|
+
projectScope,
|
|
116
85
|
} = opts;
|
|
117
86
|
|
|
118
87
|
const license = checkLicense();
|
|
@@ -124,7 +93,11 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
124
93
|
return ExuluTool.internal({
|
|
125
94
|
id: "agentic_context_search",
|
|
126
95
|
name: "Context Search",
|
|
127
|
-
description:
|
|
96
|
+
description:
|
|
97
|
+
`Intelligent knowledge search across the available knowledge bases: ${contexts.map((c) => c.name || c.id).join(", ")}. Routes the question to the right sources, searches them with query expansion, and returns reranked passages. Results are exhaustive for the given query: do NOT repeat the call with a rephrased version of the same question — re-call only with genuinely new information (a different product or model, an explicitly named source or document, or new details from the user).` +
|
|
98
|
+
// Note: the description suffix intentionally remains even when the per-agent project_search
|
|
99
|
+
// config is off — the config is only known at execute time, not at factory time.
|
|
100
|
+
(projectScope ? ` Also searches the knowledge items attached to the project "${projectScope.name}".` : ""),
|
|
128
101
|
category: "contexts",
|
|
129
102
|
needsApproval: false,
|
|
130
103
|
type: "context",
|
|
@@ -167,10 +140,17 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
167
140
|
},
|
|
168
141
|
{
|
|
169
142
|
name: "max_steps",
|
|
170
|
-
description: "Maximum
|
|
143
|
+
description: "Maximum knowledge searches the agent may run for one message. Once spent, the search tool is disabled for the rest of the turn. 0 = no search-specific cap (the agent's overall tool-step budget still applies).",
|
|
171
144
|
type: "number",
|
|
172
145
|
default: 0,
|
|
173
146
|
},
|
|
147
|
+
{
|
|
148
|
+
name: "project_search",
|
|
149
|
+
description:
|
|
150
|
+
"Automatically include items attached to the chat's project as an additional knowledge source (boosts them in shared sources, adds scoped search for others).",
|
|
151
|
+
type: "boolean",
|
|
152
|
+
default: true,
|
|
153
|
+
},
|
|
174
154
|
{
|
|
175
155
|
name: "knowledge_bases",
|
|
176
156
|
description: "Per-knowledge-base profiles: enabled, kind (documents | conversations | records), instructions, and per-KB overrides (limit, expand, multiQuery, hyde). JSON object keyed by context id.",
|
|
@@ -309,6 +289,33 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
309
289
|
// ── Preselected items map ─────────────────────────────────────────────
|
|
310
290
|
const preselectedItems = parsePreselectedItems(preselected ?? []);
|
|
311
291
|
|
|
292
|
+
// ── Project scope: additional source, never narrows configured sources ──
|
|
293
|
+
const availableContextsById = new Map(contexts.map((c) => [c.id, c]));
|
|
294
|
+
const resolvedProject = cfg.projectSearch
|
|
295
|
+
? resolveProjectScope({
|
|
296
|
+
scope: projectScope,
|
|
297
|
+
enabledContextIds: new Set(enabledContexts.map((c) => c.id)),
|
|
298
|
+
availableContextIds: new Set(availableContextsById.keys()),
|
|
299
|
+
})
|
|
300
|
+
: undefined;
|
|
301
|
+
if (resolvedProject) {
|
|
302
|
+
// Synthesized profile defaults (e.g. transcriptions → conversations kind);
|
|
303
|
+
// a stored knowledge_bases profile always wins.
|
|
304
|
+
if (projectScope?.kbProfileDefaults) {
|
|
305
|
+
for (const [ctxId, profile] of Object.entries(projectScope.kbProfileDefaults)) {
|
|
306
|
+
if (!cfg.knowledgeBases[ctxId]) cfg.knowledgeBases[ctxId] = profile;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
// Copy-on-write: enabledContexts may alias the factory's contexts array
|
|
310
|
+
// (the restore-all branch above), so never push into it.
|
|
311
|
+
enabledContexts = [
|
|
312
|
+
...enabledContexts,
|
|
313
|
+
...resolvedProject.addedContextIds
|
|
314
|
+
.map((id) => availableContextsById.get(id))
|
|
315
|
+
.filter((c): c is NonNullable<typeof c> => Boolean(c)),
|
|
316
|
+
];
|
|
317
|
+
}
|
|
318
|
+
|
|
312
319
|
// ── Derived maps ──────────────────────────────────────────────────────
|
|
313
320
|
const contextsById = new Map(enabledContexts.map((c) => [c.id, c]));
|
|
314
321
|
const kbKindById = new Map(
|
|
@@ -322,7 +329,15 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
322
329
|
);
|
|
323
330
|
|
|
324
331
|
// ── Phase 1: memory + routing in parallel ─────────────────────────────
|
|
325
|
-
|
|
332
|
+
// Gate on resolvedProject: when project_search is off, resolvedProject is undefined
|
|
333
|
+
// and project custom instructions must NOT leak into routing/HyDE.
|
|
334
|
+
const extraInstructions = [
|
|
335
|
+
cfg.instructions,
|
|
336
|
+
adminInstructions,
|
|
337
|
+
resolvedProject && projectScope?.customInstructions
|
|
338
|
+
? `Instructions for the attached project "${projectScope.name}":\n${projectScope.customInstructions}`
|
|
339
|
+
: "",
|
|
340
|
+
]
|
|
326
341
|
.filter(Boolean)
|
|
327
342
|
.join("\n");
|
|
328
343
|
|
|
@@ -377,6 +392,36 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
377
392
|
return;
|
|
378
393
|
}
|
|
379
394
|
|
|
395
|
+
// ── Project sources are always-main (attaching a project is an explicit signal) ──
|
|
396
|
+
let effectiveMainContexts = mainContexts;
|
|
397
|
+
if (resolvedProject) {
|
|
398
|
+
const mainSet = new Set(mainContexts);
|
|
399
|
+
const appended = resolvedProject.allProjectContextIds.filter(
|
|
400
|
+
(id) => !mainSet.has(id) && contextsById.has(id),
|
|
401
|
+
);
|
|
402
|
+
if (appended.length > 0) {
|
|
403
|
+
effectiveMainContexts = [...mainContexts, ...appended];
|
|
404
|
+
result.steps.push({
|
|
405
|
+
stepNumber: 1,
|
|
406
|
+
text: `Including sources from project "${projectScope!.name}": ${appended.join(", ")}`,
|
|
407
|
+
toolCalls: [],
|
|
408
|
+
chunks: [],
|
|
409
|
+
tokens: 0,
|
|
410
|
+
});
|
|
411
|
+
result.reasoning.push({
|
|
412
|
+
text: `Including project sources: ${appended.join(", ")}`,
|
|
413
|
+
tools: [],
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// A project context appended to effectiveMainContexts can also appear in fallbackContexts
|
|
419
|
+
// (routing classified it as enabled before the project append). Deduplicate to avoid
|
|
420
|
+
// double-searching the same context in the speculative fallback pass.
|
|
421
|
+
const fallbackContextsToSearch = fallbackContexts.filter(
|
|
422
|
+
(id) => !effectiveMainContexts.includes(id),
|
|
423
|
+
);
|
|
424
|
+
|
|
380
425
|
const {
|
|
381
426
|
updatedQuestion,
|
|
382
427
|
updatedKeywords,
|
|
@@ -409,7 +454,7 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
409
454
|
// ── Phase 2: main + speculative fallback searchContexts in parallel ───
|
|
410
455
|
const [mainSearch, speculativeFallbackSearch] = await Promise.all([
|
|
411
456
|
searchContexts({
|
|
412
|
-
contextIds:
|
|
457
|
+
contextIds: effectiveMainContexts,
|
|
413
458
|
contextsById,
|
|
414
459
|
kbProfiles: cfg.knowledgeBases,
|
|
415
460
|
question: updatedQuestion,
|
|
@@ -422,14 +467,15 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
422
467
|
identifierPinsByContext,
|
|
423
468
|
memoryPinnedItemIds,
|
|
424
469
|
userPinnedItemIdsByContext,
|
|
470
|
+
scopedItemsByContext: resolvedProject?.scopedItemsByContext,
|
|
425
471
|
rewrites: cfg.vocabulary.rewrites,
|
|
426
472
|
styleHint: cfg.vocabulary.styleHint,
|
|
427
473
|
maxQueries: cfg.tuning.maxQueriesPerContext,
|
|
428
474
|
skipPrefilter: false,
|
|
429
475
|
}),
|
|
430
|
-
|
|
476
|
+
fallbackContextsToSearch.length > 0 && !hasExplicitDocAndPage
|
|
431
477
|
? searchContexts({
|
|
432
|
-
contextIds:
|
|
478
|
+
contextIds: fallbackContextsToSearch,
|
|
433
479
|
contextsById,
|
|
434
480
|
kbProfiles: cfg.knowledgeBases,
|
|
435
481
|
question: updatedQuestion,
|
|
@@ -442,6 +488,7 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
442
488
|
identifierPinsByContext,
|
|
443
489
|
memoryPinnedItemIds,
|
|
444
490
|
userPinnedItemIdsByContext,
|
|
491
|
+
scopedItemsByContext: resolvedProject?.scopedItemsByContext,
|
|
445
492
|
rewrites: cfg.vocabulary.rewrites,
|
|
446
493
|
styleHint: cfg.vocabulary.styleHint,
|
|
447
494
|
maxQueries: cfg.tuning.maxQueriesPerContext,
|
|
@@ -451,7 +498,7 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
451
498
|
]);
|
|
452
499
|
|
|
453
500
|
// ── Build rerank state ────────────────────────────────────────────────
|
|
454
|
-
// pinnedItemIds = memory ∪ exact identifier pins ∪ user pins
|
|
501
|
+
// pinnedItemIds = memory ∪ exact identifier pins ∪ user pins ∪ project pins
|
|
455
502
|
const pinnedItemIds = new Set<string>([
|
|
456
503
|
...memoryPinnedItemIds,
|
|
457
504
|
...(function* () {
|
|
@@ -460,6 +507,9 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
460
507
|
...(function* () {
|
|
461
508
|
for (const s of userPinnedItemIdsByContext.values()) yield* s;
|
|
462
509
|
})(),
|
|
510
|
+
...(function* () {
|
|
511
|
+
if (resolvedProject) for (const s of resolvedProject.pinsByContext.values()) yield* s;
|
|
512
|
+
})(),
|
|
463
513
|
]);
|
|
464
514
|
// userPinnedItemIds = user pins only
|
|
465
515
|
const userPinnedItemIds = new Set<string>(
|
|
@@ -548,19 +598,19 @@ export function createAgenticRetrievalTool(opts: {
|
|
|
548
598
|
// ── Fallback gate ─────────────────────────────────────────────────────
|
|
549
599
|
if (
|
|
550
600
|
!literalLookupSatisfied &&
|
|
551
|
-
|
|
601
|
+
fallbackContextsToSearch.length > 0 &&
|
|
552
602
|
(reranker
|
|
553
603
|
? mainRerank.rerank_score_max_genuine < cfg.tuning.fallbackThreshold
|
|
554
604
|
: mainRerank.limited_results.length < cfg.tuning.topK)
|
|
555
605
|
) {
|
|
556
606
|
result.steps.push({
|
|
557
607
|
stepNumber: 1,
|
|
558
|
-
text: `Using fallback search in ${
|
|
608
|
+
text: `Using fallback search in ${fallbackContextsToSearch.join(", ")}`,
|
|
559
609
|
toolCalls: [],
|
|
560
610
|
chunks: [],
|
|
561
611
|
tokens: 0,
|
|
562
612
|
});
|
|
563
|
-
result.reasoning.push({ text: `Fallback search in ${
|
|
613
|
+
result.reasoning.push({ text: `Fallback search in ${fallbackContextsToSearch.join(", ")}`, tools: [] });
|
|
564
614
|
yield { result: serializeOutput(result) };
|
|
565
615
|
|
|
566
616
|
const fallbackRerank = await rerankResults({
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { resolveProjectScope, buildProjectKbProfileDefaults } from "./project-scope";
|
|
2
|
+
|
|
3
|
+
const baseScope = {
|
|
4
|
+
id: "p1",
|
|
5
|
+
name: "Elevator Modernization",
|
|
6
|
+
items: ["docs/item-1", "docs/item-2", "tickets/item-9", "wiki"],
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
describe("resolveProjectScope", () => {
|
|
10
|
+
it("returns undefined for no scope or empty items", () => {
|
|
11
|
+
expect(resolveProjectScope({ scope: undefined, enabledContextIds: new Set(), availableContextIds: new Set() })).toBeUndefined();
|
|
12
|
+
expect(resolveProjectScope({ scope: { ...baseScope, items: [] }, enabledContextIds: new Set(), availableContextIds: new Set(["docs"]) })).toBeUndefined();
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it("enabled contexts get PINS (boost), never filters", () => {
|
|
16
|
+
const r = resolveProjectScope({
|
|
17
|
+
scope: baseScope,
|
|
18
|
+
enabledContextIds: new Set(["docs", "tickets", "wiki"]),
|
|
19
|
+
availableContextIds: new Set(["docs", "tickets", "wiki"]),
|
|
20
|
+
})!;
|
|
21
|
+
expect(r.pinsByContext.get("docs")).toEqual(new Set(["item-1", "item-2"]));
|
|
22
|
+
expect(r.pinsByContext.get("tickets")).toEqual(new Set(["item-9"]));
|
|
23
|
+
expect(r.scopedItemsByContext.size).toBe(0);
|
|
24
|
+
expect(r.addedContextIds).toEqual([]);
|
|
25
|
+
// bare-context "wiki" is already enabled in full → no pin entry either
|
|
26
|
+
expect(r.pinsByContext.has("wiki")).toBe(false);
|
|
27
|
+
expect(new Set(r.allProjectContextIds)).toEqual(new Set(["docs", "tickets", "wiki"]));
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it("non-enabled contexts get added item-scoped; bare-context entry scopes to whole context (null)", () => {
|
|
31
|
+
const r = resolveProjectScope({
|
|
32
|
+
scope: baseScope,
|
|
33
|
+
enabledContextIds: new Set(["docs"]),
|
|
34
|
+
availableContextIds: new Set(["docs", "tickets", "wiki"]),
|
|
35
|
+
})!;
|
|
36
|
+
expect(r.pinsByContext.get("docs")).toEqual(new Set(["item-1", "item-2"]));
|
|
37
|
+
expect(r.scopedItemsByContext.get("tickets")).toEqual(["item-9"]);
|
|
38
|
+
expect(r.scopedItemsByContext.get("wiki")).toBeNull();
|
|
39
|
+
expect(new Set(r.addedContextIds)).toEqual(new Set(["tickets", "wiki"]));
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("unknown contexts are dropped with a warning, not an error", () => {
|
|
43
|
+
const r = resolveProjectScope({
|
|
44
|
+
scope: { ...baseScope, items: ["ghost/item-1", "docs/item-1"] },
|
|
45
|
+
enabledContextIds: new Set(["docs"]),
|
|
46
|
+
availableContextIds: new Set(["docs"]),
|
|
47
|
+
})!;
|
|
48
|
+
expect(r.allProjectContextIds).toEqual(["docs"]);
|
|
49
|
+
expect(r.pinsByContext.get("docs")).toEqual(new Set(["item-1"]));
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("returns undefined when every referenced context is unknown", () => {
|
|
53
|
+
expect(
|
|
54
|
+
resolveProjectScope({
|
|
55
|
+
scope: { ...baseScope, items: ["ghost/x"] },
|
|
56
|
+
enabledContextIds: new Set(),
|
|
57
|
+
availableContextIds: new Set(["docs"]),
|
|
58
|
+
}),
|
|
59
|
+
).toBeUndefined();
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
describe("buildProjectKbProfileDefaults", () => {
|
|
64
|
+
it("maps the transcriptions context to conversations kind", () => {
|
|
65
|
+
expect(buildProjectKbProfileDefaults(["transcriptions/t1", "docs/d1"])).toEqual({
|
|
66
|
+
transcriptions: { enabled: true, kind: "conversations", instructions: "", overrides: {} },
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("returns {} when transcriptions is not referenced", () => {
|
|
71
|
+
expect(buildProjectKbProfileDefaults(["docs/d1", "wiki"])).toEqual({});
|
|
72
|
+
});
|
|
73
|
+
});
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { KbProfile } from "./config";
|
|
2
|
+
import { parsePreselectedItems } from "./global-ids";
|
|
3
|
+
|
|
4
|
+
/** Identity + items of the project attached to the current chat session. */
|
|
5
|
+
export type ProjectScope = {
|
|
6
|
+
id: string;
|
|
7
|
+
name: string;
|
|
8
|
+
description?: string;
|
|
9
|
+
customInstructions?: string;
|
|
10
|
+
/** Raw project_items gids: "<contextId>/<itemId>" or bare "<contextId>" (= whole context). */
|
|
11
|
+
items: string[];
|
|
12
|
+
/** Synthesized per-context profile defaults; a stored knowledge_bases profile always wins. */
|
|
13
|
+
kbProfileDefaults?: Record<string, KbProfile>;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export type ResolvedProjectScope = {
|
|
17
|
+
/** Contexts the instance already searches in full: project items only BOOST reranking. */
|
|
18
|
+
pinsByContext: Map<string, Set<string>>;
|
|
19
|
+
/** Contexts added for the project: hard item filter (null = whole context). */
|
|
20
|
+
scopedItemsByContext: Map<string, string[] | null>;
|
|
21
|
+
addedContextIds: string[];
|
|
22
|
+
allProjectContextIds: string[];
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Split a project's items into pin vs scoped-source treatment. The invariant:
|
|
27
|
+
* a project may ADD scope but must never NARROW what the agent already searches.
|
|
28
|
+
*/
|
|
29
|
+
export function resolveProjectScope(opts: {
|
|
30
|
+
scope: ProjectScope | undefined;
|
|
31
|
+
enabledContextIds: Set<string>;
|
|
32
|
+
availableContextIds: Set<string>;
|
|
33
|
+
}): ResolvedProjectScope | undefined {
|
|
34
|
+
const { scope, enabledContextIds, availableContextIds } = opts;
|
|
35
|
+
if (!scope || scope.items.length === 0) return undefined;
|
|
36
|
+
|
|
37
|
+
const itemsByContext = parsePreselectedItems(scope.items);
|
|
38
|
+
const pinsByContext = new Map<string, Set<string>>();
|
|
39
|
+
const scopedItemsByContext = new Map<string, string[] | null>();
|
|
40
|
+
const addedContextIds: string[] = [];
|
|
41
|
+
const allProjectContextIds: string[] = [];
|
|
42
|
+
|
|
43
|
+
for (const [ctxId, itemIds] of itemsByContext) {
|
|
44
|
+
if (!availableContextIds.has(ctxId)) {
|
|
45
|
+
console.warn(
|
|
46
|
+
`[EXULU pipeline] project "${scope.name}" references unknown context "${ctxId}" — skipping those items.`,
|
|
47
|
+
);
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
allProjectContextIds.push(ctxId);
|
|
51
|
+
if (enabledContextIds.has(ctxId)) {
|
|
52
|
+
if (itemIds && itemIds.length > 0) pinsByContext.set(ctxId, new Set(itemIds));
|
|
53
|
+
// bare-context entry on an already-enabled context adds nothing
|
|
54
|
+
} else {
|
|
55
|
+
scopedItemsByContext.set(ctxId, itemIds);
|
|
56
|
+
addedContextIds.push(ctxId);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (allProjectContextIds.length === 0) return undefined;
|
|
61
|
+
return { pinsByContext, scopedItemsByContext, addedContextIds, allProjectContextIds };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const TRANSCRIPTIONS_CONTEXT_ID = "transcriptions";
|
|
65
|
+
|
|
66
|
+
/** Kind heuristic for auto-configured project sources (design spec §7.3). */
|
|
67
|
+
export function buildProjectKbProfileDefaults(items: string[]): Record<string, KbProfile> {
|
|
68
|
+
const defaults: Record<string, KbProfile> = {};
|
|
69
|
+
for (const gid of items) {
|
|
70
|
+
const slashIdx = gid.indexOf("/");
|
|
71
|
+
const ctxId = slashIdx === -1 ? gid : gid.slice(0, slashIdx);
|
|
72
|
+
if (ctxId === TRANSCRIPTIONS_CONTEXT_ID && !defaults[ctxId]) {
|
|
73
|
+
defaults[ctxId] = { enabled: true, kind: "conversations", instructions: "", overrides: {} };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return defaults;
|
|
77
|
+
}
|
|
@@ -146,4 +146,31 @@ describe("searchContexts", () => {
|
|
|
146
146
|
const call = (multiQuerySearch as jest.Mock).mock.calls[0][0];
|
|
147
147
|
expect(call.queries).not.toContain("HYDE PASSAGE");
|
|
148
148
|
});
|
|
149
|
+
|
|
150
|
+
it("project-scoped contexts hard-filter to the project's items", async () => {
|
|
151
|
+
await searchContexts({
|
|
152
|
+
...base, contextIds: ["docs"],
|
|
153
|
+
scopedItemsByContext: new Map([["docs", ["pj1", "pj2"]]]),
|
|
154
|
+
identifierPinsByContext: new Map([["docs", new Set(["i1"])]]),
|
|
155
|
+
});
|
|
156
|
+
expect((multiQuerySearch as jest.Mock).mock.calls[0][0].pinnedItemIds).toEqual(["pj1", "pj2"]);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("project-scoped whole-context entry (null) searches the context unfiltered", async () => {
|
|
160
|
+
await searchContexts({
|
|
161
|
+
...base, contextIds: ["docs"],
|
|
162
|
+
scopedItemsByContext: new Map([["docs", null]]),
|
|
163
|
+
identifierPinsByContext: new Map([["docs", new Set(["i1"])]]),
|
|
164
|
+
});
|
|
165
|
+
expect((multiQuerySearch as jest.Mock).mock.calls[0][0].pinnedItemIds).toEqual([]);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("session preselection wins over project scoping", async () => {
|
|
169
|
+
await searchContexts({
|
|
170
|
+
...base, contextIds: ["docs"],
|
|
171
|
+
preselectedItems: new Map([["docs", ["s1"]]]),
|
|
172
|
+
scopedItemsByContext: new Map([["docs", ["pj1"]]]),
|
|
173
|
+
});
|
|
174
|
+
expect((multiQuerySearch as jest.Mock).mock.calls[0][0].pinnedItemIds).toEqual(["s1"]);
|
|
175
|
+
});
|
|
149
176
|
});
|
|
@@ -23,6 +23,7 @@ export async function searchContexts(opts: {
|
|
|
23
23
|
role: any;
|
|
24
24
|
model: any;
|
|
25
25
|
preselectedItems: Map<string, string[] | null>;
|
|
26
|
+
scopedItemsByContext?: Map<string, string[] | null>; // Project-added sources: hard item filter per context (null = whole context).
|
|
26
27
|
identifierPinsByContext: Map<string, Set<string>>; // from resolveIdentifierPins
|
|
27
28
|
memoryPinnedItemIds: Set<string>; // from memory phase (documents kind only)
|
|
28
29
|
userPinnedItemIdsByContext: Map<string, Set<string>>; // from routing phase
|
|
@@ -42,6 +43,7 @@ export async function searchContexts(opts: {
|
|
|
42
43
|
role,
|
|
43
44
|
model,
|
|
44
45
|
preselectedItems,
|
|
46
|
+
scopedItemsByContext,
|
|
45
47
|
identifierPinsByContext,
|
|
46
48
|
memoryPinnedItemIds,
|
|
47
49
|
userPinnedItemIdsByContext,
|
|
@@ -80,6 +82,11 @@ export async function searchContexts(opts: {
|
|
|
80
82
|
if (hasPreselection) {
|
|
81
83
|
// null value = whole context = no filter = []
|
|
82
84
|
pinnedItemIds = preselectedItems.get(ctxId) ?? [];
|
|
85
|
+
} else if (scopedItemsByContext?.has(ctxId)) {
|
|
86
|
+
// Project-scoped source: restrict to the project's items for this
|
|
87
|
+
// context (null = whole context). Deliberately NOT unioned with
|
|
88
|
+
// identifier/memory pins — those would widen a scoped source.
|
|
89
|
+
pinnedItemIds = scopedItemsByContext.get(ctxId) ?? [];
|
|
83
90
|
} else if (!skipPrefilter) {
|
|
84
91
|
// Rule 2: No preselection and !skipPrefilter
|
|
85
92
|
|