@exulu/backend 1.53.0 → 1.54.0

package/ee/agentic-retrieval/v3/agent-loop.ts

@@ -0,0 +1,242 @@
+import { generateText, stepCountIs, tool } from "ai";
+import type { LanguageModel, Tool as AITool, ModelMessage } from "ai";
+import { z } from "zod";
+import { withRetry } from "@SRC/utils/with-retry";
+import type { ExuluReranker } from "@SRC/exulu/reranker";
+import type { AgenticRetrievalOutput, ChunkResult, ClassificationResult } from "./types";
+import type { StrategyConfig } from "./strategies";
+import { createDynamicTools } from "./dynamic-tools";
+
+const FINISH_TOOL_NAME = "finish_retrieval";
+
+const finishRetrievalTool = tool({
+  description:
+    "Call this tool when you have retrieved sufficient information and no further searches are needed. " +
+    "You MUST call this tool to signal that retrieval is complete — do not write a text conclusion.",
+  inputSchema: z.object({
+    reasoning: z.string().describe("One sentence explaining why retrieval is complete"),
+  }),
+  execute: async ({ reasoning }) => JSON.stringify({ finished: true, reasoning }),
+});
+
+function extractChunksFromToolResults(toolResults: any[]): ChunkResult[] {
+  const chunks: ChunkResult[] = [];
+  for (const result of toolResults ?? []) {
+    // AI SDK v6 uses `output` (not `result`) for tool result values
+    const rawOutput = result.output ?? result.result;
+    let parsed: any;
+    try {
+      parsed = typeof rawOutput === "string" ? JSON.parse(rawOutput) : rawOutput;
+    } catch {
+      continue;
+    }
+
+    if (Array.isArray(parsed)) {
+      for (const item of parsed) {
+        if (item?.item_id && item?.context) {
+          chunks.push({
+            item_name: item.item_name,
+            item_id: item.item_id,
+            context: item.context?.id ?? item.context,
+            chunk_id: item.chunk_id,
+            chunk_index: item.chunk_index,
+            chunk_content: item.chunk_content,
+            metadata: item.metadata,
+          });
+        }
+      }
+    }
+  }
+  return chunks;
+}
+
+/**
+ * Core agent loop: one generateText call per step.
+ *
+ * Unlike v2 (which split each step into a reasoning call + a separate tool
+ * execution call), here a single call with toolChoice: "auto" lets the model
+ * reason and call tools in one pass. The model sees tool results from the
+ * previous step via the conversation history (messages array).
+ *
+ * The loop stops when:
+ * - The model makes no tool calls (it's satisfied), OR
+ * - The strategy's stepBudget is exhausted
+ */
+export async function* runAgentLoop(params: {
+  query: string;
+  strategy: StrategyConfig;
+  tools: Record<string, AITool>;
+  model: LanguageModel;
+  reranker?: ExuluReranker;
+  contextGuidance?: string;
+  customInstructions?: string;
+  classification: ClassificationResult;
+  onStepComplete?: (step: AgenticRetrievalOutput["steps"][0]) => void;
+}): AsyncGenerator<AgenticRetrievalOutput> {
+  const { query, strategy, tools, model, reranker, contextGuidance, customInstructions, onStepComplete } = params;
+
+  const output: AgenticRetrievalOutput = {
+    steps: [],
+    reasoning: [],
+    chunks: [],
+    usage: [],
+    totalTokens: 0,
+  };
+
+  const messages: ModelMessage[] = [{ role: "user", content: query }];
+  let dynamicTools: Record<string, AITool> = {};
+  let forceDepthExploration = false;
+  let forceContextCoverage = false;
+
+  // Track which suggested contexts have been searched to enforce coverage
+  const suggestedContextIds = params.classification.suggestedContextIds ?? [];
+  const searchedContextIds = new Set<string>();
+
+  const baseSystemPrompt = [
+    strategy.instructions,
+    contextGuidance ? `\nCONTEXT GUIDANCE:\n${contextGuidance}` : "",
+    customInstructions ? `\nCUSTOM INSTRUCTIONS (override context guidance above where they conflict):\n${customInstructions}` : "",
+  ]
+    .filter(Boolean)
+    .join("\n");
+
+  const SEARCH_TOOL_NAMES = new Set([
+    "search_content",
+    "save_search_results",
+    "count_items_or_chunks",
+    "search_items_by_name",
+  ]);
+
+  for (let step = 0; step < strategy.stepBudget; step++) {
+    console.log(`[EXULU] v3 agent loop — step ${step + 1}/${strategy.stepBudget}`);
+
+    // Build dynamic system prompt: add unsearched-context note after the first step
+    const unsearchedNow = suggestedContextIds.filter((id) => !searchedContextIds.has(id));
+    const contextCoverageNote =
+      unsearchedNow.length > 0 && step > 0
+        ? `\n\n⚠️ MANDATORY: The following suggested contexts have NOT been searched yet: [${unsearchedNow.join(", ")}]. You MUST include ALL of them in your next search call. Note: support/ticket contexts use document names like "Ticket #XXXX" — do NOT use item_names when searching them.`
+        : "";
+    const stepSystemPrompt = baseSystemPrompt + contextCoverageNote;
+
+    let result: Awaited<ReturnType<typeof generateText>>;
+    try {
+      const stepTools = forceDepthExploration || forceContextCoverage
+        ? { ...tools, ...dynamicTools } // finish_retrieval withheld — model must search/explore more
+        : { ...tools, ...dynamicTools, [FINISH_TOOL_NAME]: finishRetrievalTool };
+
+      result = await withRetry(() =>
+        generateText({
+          model,
+          temperature: 0,
+          system: stepSystemPrompt,
+          messages,
+          tools: stepTools,
+          toolChoice: "required",
+          stopWhen: stepCountIs(1),
+        }),
+      );
+    } catch (err) {
+      console.error("[EXULU] v3 generateText failed:", err);
+      throw err;
+    }
+
+    // Carry conversation forward: assistant message + tool results go into history
+    // so the model sees them on the next iteration.
+    messages.push(...(result.response.messages as ModelMessage[]));
+
+    // Extract chunks from tool results
+    let stepChunks: any[] = extractChunksFromToolResults(result.toolResults as any[]);
+
+    // Check if any search_content call excluded content (triggers page-load dynamic tools)
+    // AI SDK v6 uses `input` (not `args`) for tool call arguments
+    const hadExcludedContent = (result.toolCalls as any[])?.some(
+      (tc) =>
+        (tc.toolName === "search_content" && tc.input?.includeContent === false) ||
+        tc.toolName === "search_items_by_name",
+    );
+
+    // Rerank if reranker is available
+    if (reranker && stepChunks.length > 0) {
+      console.log(`[EXULU] v3 reranking ${stepChunks.length} chunks with ${reranker.name}`);
+      stepChunks = await reranker.run(query, stepChunks as any);
+    }
+
+    // Create dynamic tools (browse adjacent pages, load specific pages)
+    const newDynamic = await createDynamicTools(stepChunks as ChunkResult[], hadExcludedContent);
+    Object.assign(dynamicTools, newDynamic);
+
+    // If relevant content was found but fewer than 5 chunks, withhold finish_retrieval
+    // on the next step to force depth exploration via dynamic tools.
+    // Only applies when dynamic tools exist and there's budget remaining for both
+    // a depth step and a finish step.
+    forceDepthExploration =
+      stepChunks.length > 0 &&
+      stepChunks.length < 5 &&
+      Object.keys(newDynamic).length > 0 &&
+      step < strategy.stepBudget - 2;
+
+    // Track which suggested contexts have been searched this step
+    for (const tc of (result.toolCalls as any[]) ?? []) {
+      if (SEARCH_TOOL_NAMES.has(tc.toolName)) {
+        for (const id of (tc.input?.knowledge_base_ids ?? [])) {
+          searchedContextIds.add(id);
+        }
+      }
+    }
+
+    // Withhold finish_retrieval on the next step if suggested contexts remain unsearched
+    const unsearchedAfterStep = suggestedContextIds.filter((id) => !searchedContextIds.has(id));
+    forceContextCoverage = unsearchedAfterStep.length > 0 && step < strategy.stepBudget - 1;
+    if (forceContextCoverage) {
+      console.log(
+        `[EXULU] v3 forceContextCoverage — unsearched suggested: [${unsearchedAfterStep.join(", ")}]`,
+      );
+    }
+
+    // Record step
+    const stepRecord = {
+      stepNumber: step + 1,
+      text: result.text ?? "",
+      toolCalls: (result.toolCalls as any[])?.map((tc) => ({
+        name: tc.toolName,
+        id: tc.toolCallId,
+        input: tc.input,
+      })) ?? [],
+      chunks: stepChunks,
+      dynamicToolsCreated: Object.keys(newDynamic),
+      tokens: result.usage?.totalTokens ?? 0,
+    };
+
+    output.steps.push(stepRecord);
+    output.reasoning.push({
+      text: result.text ?? "",
+      tools: (result.toolCalls as any[])?.map((tc) => ({
+        name: tc.toolName,
+        id: tc.toolCallId,
+        input: tc.input,
+        output: stepChunks,
+      })) ?? [],
+    });
+    output.chunks.push(...stepChunks);
+    output.usage.push(result.usage);
+
+    onStepComplete?.(stepRecord);
+
+    yield { ...output };
+
+    // Stop if the model called finish_retrieval AND no forced continuation is needed
+    const calledFinish = (result.toolCalls as any[])?.some(
+      (tc) => tc.toolName === FINISH_TOOL_NAME,
+    );
+    if (calledFinish && !forceContextCoverage) {
+      console.log(`[EXULU] v3 model called finish_retrieval after step ${step + 1}`);
+      break;
+    } else if (calledFinish && forceContextCoverage) {
+      console.log(
+        `[EXULU] v3 model called finish_retrieval but overriding — unsearched suggested contexts remain`,
+      );
+    }
+  }
+
+  output.totalTokens = output.usage.reduce((sum, u) => sum + (u?.totalTokens ?? 0), 0);
+}

package/ee/agentic-retrieval/v3/classifier.ts

@@ -0,0 +1,73 @@
+import { generateText, Output } from "ai";
+import type { LanguageModel } from "ai";
+import { z } from "zod";
+import type { ExuluContext } from "@SRC/exulu/context";
+import type { ClassificationResult, ContextSample } from "./types";
+
+/**
+ * Classifies a query into one of four types and identifies which contexts are
+ * most relevant. This is a single fast LLM call that runs before the main
+ * agent loop, enabling strategy-based routing.
+ */
+export async function classifyQuery(
+  query: string,
+  contexts: ExuluContext[],
+  samples: ContextSample[],
+  model: LanguageModel,
+): Promise<ClassificationResult> {
+  const contextDescriptions = contexts
+    .map((ctx) => {
+      const sample = samples.find((s) => s.contextId === ctx.id);
+      const fieldList = sample?.fields.join(", ") ?? "name, external_id";
+      const exampleStr =
+        sample?.exampleItems.length
+          ? `\n    Example records: ${JSON.stringify(sample.exampleItems.slice(0, 2))}`
+          : "";
+      return `  - ${ctx.id}: ${ctx.name}\n    Description: ${ctx.description}\n    Fields: ${fieldList}${exampleStr}`;
+    })
+    .join("\n\n");
+
+  const result = await generateText({
+    model,
+    temperature: 0,
+    output: Output.object({
+      schema: z.object({
+        queryType: z
+          .enum(["aggregate", "list", "targeted", "exploratory"])
+          .describe(
+            "aggregate: ONLY use when the user explicitly asks to COUNT how many documents/items/tickets exist in the knowledge base (e.g. 'how many documents about X?', 'total number of tickets'). NEVER use for: real-world statistics stored in a document, intent statements, how-to questions, error/fault descriptions, configuration questions, or any query that does not explicitly ask for a count of knowledge base entries. When in doubt, choose targeted. " +
+              "list: user wants to enumerate matching items/documents (show me all, list documents about). " +
+              "targeted: use for almost everything — specific fact, answer, configuration, how-to, error/fault, feature/behavior question. Also use for intent statements and short commands describing a desired state (phrases that state what the user wants to do or achieve, even without an explicit question word). Real-world statistics stored in documents also go here. When in doubt, choose targeted over aggregate or exploratory. " +
+              "exploratory: only for broad conceptual questions needing multi-source synthesis (what is the process for Z, explain how X works, general overview of topic Y).",
+          ),
+        language: z
+          .string()
+          .describe("ISO 639-3 language code of the query (e.g. eng, deu, fra)"),
+        suggestedContextIds: z
+          .array(z.string())
+          .describe(
+            "IDs of knowledge bases most likely to contain the answer. Return empty array to search all contexts.",
+          ),
+      }),
+    }),
+    toolChoice: "none",
+    system: `You are a query classifier for a multi-knowledge-base retrieval system.
+Classify the query and identify which knowledge bases are most relevant.
+
+Available knowledge bases:
+${contextDescriptions}
+
+Guidelines for queryType:
+- Use "aggregate" ONLY when the query contains explicit counting language (e.g., "how many", "count", "total number", "wie viele"). Short statements, commands, or phrases without a question word are NEVER aggregate — classify them as targeted.
+- When in doubt between aggregate and targeted: always choose targeted.
+
+Guidelines for suggestedContextIds:
+- Be conservative: only suggest contexts that are genuinely likely to contain the answer.
+  Aim for 2–3 focused suggestions rather than listing everything.
+- Use each knowledge base's name and description (shown above) to judge relevance.
+- Return an empty array only if you truly cannot determine which contexts are relevant.`,
+    prompt: `Query: ${query}`,
+  });
+
+  return result.output as ClassificationResult;
+}

package/ee/agentic-retrieval/v3/context-sampler.ts

@@ -0,0 +1,70 @@
+import { ExuluContext, getTableName } from "@SRC/exulu/context";
+import { postgresClient } from "@SRC/postgres/client";
+import { applyAccessControl } from "@SRC/graphql/utilities/access-control";
+import { convertContextToTableDefinition } from "@SRC/graphql/utilities/convert-context-to-table-definition";
+import type { User } from "@EXULU_TYPES/models/user";
+import type { ContextSample } from "./types";
+
+const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
+
+/**
+ * Pulls 1–2 example item records per context at agent initialization and caches
+ * them in memory. These samples are injected into the classifier prompt so the
+ * model understands what data is actually stored (not just field names).
+ */
+export class ContextSampler {
+  private cache = new Map<string, ContextSample>();
+
+  async getSamples(
+    contexts: ExuluContext[],
+    user?: User,
+    role?: string,
+  ): Promise<ContextSample[]> {
+    return Promise.all(contexts.map((ctx) => this.getSample(ctx, user, role)));
+  }
+
+  private async getSample(
+    ctx: ExuluContext,
+    user?: User,
+    role?: string,
+  ): Promise<ContextSample> {
+    const cached = this.cache.get(ctx.id);
+    if (cached && Date.now() - cached.sampledAt < CACHE_TTL_MS) {
+      return cached;
+    }
+
+    const { db } = await postgresClient();
+    const tableName = getTableName(ctx.id);
+    const tableDefinition = convertContextToTableDefinition(ctx);
+
+    const customFieldNames = ctx.fields.map((f) => f.name);
+    const selectFields = ["id", "name", "external_id", ...customFieldNames];
+
+    let exampleItems: Record<string, any>[] = [];
+    try {
+      let query = db(tableName).select(selectFields).whereNull("archived").limit(2);
+      query = applyAccessControl(tableDefinition, query, user, tableName);
+      exampleItems = await query;
+    } catch {
+      // If table doesn't exist yet or column mismatch, return empty samples
+    }
+
+    const sample: ContextSample = {
+      contextId: ctx.id,
+      contextName: ctx.name,
+      fields: ["name", "external_id", ...customFieldNames],
+      exampleItems,
+      sampledAt: Date.now(),
+    };
+
+    this.cache.set(ctx.id, sample);
+
+    // Refresh in background after TTL without blocking the caller
+    return sample;
+  }
+
+  /** Evict a context from cache so it's re-sampled on next use */
+  invalidate(contextId: string): void {
+    this.cache.delete(contextId);
+  }
+}

package/ee/agentic-retrieval/v3/dynamic-tools.ts

@@ -0,0 +1,115 @@
+import { z } from "zod";
+import { tool } from "ai";
+import type { Tool as AITool } from "ai";
+import { postgresClient } from "@SRC/postgres/client";
+import { getChunksTableName } from "@SRC/exulu/context";
+import { sanitizeToolName } from "@SRC/utils/sanitize-tool-name.ts";
+import type { ChunkResult } from "./types";
+
+/**
+ * Creates per-chunk navigation tools from the results of a search step.
+ *
+ * Two types of dynamic tools are created:
+ * - get_more_content_from_{item}: Browse adjacent chunks of a multi-chunk item
+ * - get_{item}_page_{n}_content: Load the full text of a specific page/chunk
+ *   (created when includeContent was false in the original search)
+ */
+export async function createDynamicTools(
+  chunks: ChunkResult[],
+  hadExcludedContent: boolean,
+): Promise<Record<string, AITool>> {
+  const { db } = await postgresClient();
+  const tools: Record<string, AITool> = {};
+  const seenItems = new Set<string>();
+
+  for (const chunk of chunks) {
+    if (!chunk.item_id || !chunk.context) continue;
+
+    // ── get_more_content_from_{item} ──────────────────────────
+    const browseToolName = sanitizeToolName(`get_more_content_from_${chunk.item_name}`);
+    if (!seenItems.has(chunk.item_id) && !tools[browseToolName]) {
+      seenItems.add(chunk.item_id);
+      const chunksTable = getChunksTableName(chunk.context);
+
+      try {
+        const countResult = await db(chunksTable)
+          .count("id as count")
+          .where("source", chunk.item_id)
+          .first();
+        const total = Number(countResult?.count ?? 0);
+
+        if (total > 1) {
+          const capturedChunk = chunk;
+          tools[browseToolName] = tool({
+            description: `"${chunk.item_name}" has ${total} pages/chunks. Use this to read a range of pages from it.`,
+            inputSchema: z.object({
+              from_index: z.number().min(1).default(1).describe("Starting chunk index (1-based)"),
+              to_index: z
+                .number()
+                .max(total)
+                .describe(`Ending chunk index (max ${total})`),
+            }),
+            execute: async ({ from_index, to_index }) => {
+              const { db: db2 } = await postgresClient();
+              const rows = await db2(chunksTable)
+                .select("*")
+                .where("source", capturedChunk.item_id)
+                .whereBetween("chunk_index", [from_index, to_index])
+                .orderBy("chunk_index", "asc");
+
+              return JSON.stringify(
+                rows.map((r) => ({
+                  chunk_content: r.content,
+                  chunk_index: r.chunk_index,
+                  chunk_id: r.id,
+                  item_id: capturedChunk.item_id,
+                  item_name: capturedChunk.item_name,
+                  context: capturedChunk.context,
+                })),
+              );
+            },
+          });
+        }
+      } catch {
+        // Skip if table not accessible
+      }
+    }
+
+    // ── get_{item}_page_{n}_content ───────────────────────────
+    if (hadExcludedContent && chunk.chunk_id) {
+      const pageToolName = sanitizeToolName(
+        `get_${chunk.item_name}_page_${chunk.chunk_index}_content`,
+      );
+      if (!tools[pageToolName]) {
+        const capturedChunk = chunk;
+        tools[pageToolName] = tool({
+          description: `Load the full text of page ${chunk.chunk_index} from "${chunk.item_name}"`,
+          inputSchema: z.object({
+            reasoning: z.string().describe("Why you need this specific page's content"),
+          }),
+          execute: async () => {
+            const { db: db2 } = await postgresClient();
+            const chunksTable = getChunksTableName(capturedChunk.context!);
+            const rows = await db2(chunksTable)
+              .select("*")
+              .where("id", capturedChunk.chunk_id!)
+              .limit(1);
+
+            if (!rows[0]) return JSON.stringify({ error: "Chunk not found" });
+
+            return JSON.stringify({
+              chunk_content: rows[0].content,
+              chunk_index: rows[0].chunk_index,
+              chunk_id: rows[0].id,
+              item_id: capturedChunk.item_id,
+              item_name: capturedChunk.item_name,
+              context: capturedChunk.context ?? "",
+            });
+          },
+        });
+      }
+    }
+  }
+
+  return tools;
+}