@exulu/backend 1.54.0 → 1.56.0

package/ee/agentic-retrieval/v3/trajectory.ts

@@ -1,20 +1,31 @@
 import * as fs from "fs/promises";
 import * as path from "path";
-import type { AgenticRetrievalOutput, ClassificationResult } from "./types";
+import type { AgenticRetrievalOutput, ClassificationResult, ChunkResult } from "./types";
 
-/**
- * Module-level registry so external callers (e.g. test scripts) can read
- * the path of the most recently saved trajectory file.
- * Works because both the trajectory logger and the test run in the same process.
- */
 export const trajectoryRegistry = {
   lastFile: undefined as string | undefined,
 };
 
+export interface TrajectoryStepData {
+  stepNumber: number;
+  systemPrompt: string;
+  text: string;
+  toolCalls: Array<{
+    name: string;
+    id: string;
+    input: any;
+    output?: any;
+  }>;
+  chunks: ChunkResult[];
+  dynamicToolsCreated: string[];
+  tokens: number;
+}
+
 interface TrajectoryData {
   timestamp: string;
   query: string;
   classification: ClassificationResult;
+  preselectedItemIds?: string[];
   steps: {
     step_number: number;
     text: string;
@@ -35,6 +46,7 @@ interface TrajectoryData {
 
 export class TrajectoryLogger {
   private data: TrajectoryData;
+  private richSteps: TrajectoryStepData[] = [];
   private startTime = Date.now();
   private logDir: string;
 
@@ -42,12 +54,14 @@ export class TrajectoryLogger {
     query: string,
     classification: ClassificationResult,
     logDir = path.join(process.cwd(), "ee/agentic-retrieval/logs"),
+    preselectedItemIds?: string[],
   ) {
     this.logDir = logDir;
     this.data = {
       timestamp: new Date().toISOString(),
       query,
       classification,
+      preselectedItemIds: preselectedItemIds?.length ? preselectedItemIds : undefined,
       steps: [],
       final: {
         total_chunks: 0,
@@ -70,24 +84,223 @@ export class TrajectoryLogger {
     });
   }
 
-  async finalize(output: AgenticRetrievalOutput, success: boolean, error?: Error): Promise<string | undefined> {
+  recordRichStep(data: TrajectoryStepData): void {
+    this.richSteps.push(data);
+  }
+
+  private toMarkdown(durationMs: number, success: boolean, error?: Error): string {
+    const totalTokens = this.richSteps.reduce((sum, s) => sum + s.tokens, 0);
+    const totalChunks = this.richSteps.reduce((sum, s) => sum + s.chunks.length, 0);
+    const status = success ? "✓ Success" : `✗ Failed${error ? `: ${error.message}` : ""}`;
+    const lines: string[] = [];
+
+    // ── Header ──────────────────────────────────────────────────────────────
+    lines.push(`# Agentic Retrieval — ${this.data.timestamp}`);
+    lines.push("");
+    lines.push(`**Query:** ${this.data.query}  `);
+    lines.push(
+      `**Duration:** ${(durationMs / 1000).toFixed(1)}s | **Tokens:** ${totalTokens} | **Status:** ${status}`,
+    );
+    lines.push("");
+
+    // ── Classification ───────────────────────────────────────────────────────
+    lines.push("## Classification");
+    lines.push("");
+    lines.push(`- **Type:** \`${this.data.classification.queryType}\``);
+    lines.push(`- **Language:** \`${this.data.classification.language}\``);
+    const suggested = this.data.classification.suggestedContextIds;
+    lines.push(
+      `- **Suggested contexts:** ${suggested.length > 0 ? suggested.map((id) => `\`${id}\``).join(", ") : "*(all)*"}`,
+    );
+    if (this.data.preselectedItemIds?.length) {
+      lines.push(
+        `- **Preselected item IDs:** ${this.data.preselectedItemIds.map((id) => `\`${id}\``).join(", ")}`,
+      );
+    }
+    lines.push("");
+    lines.push("---");
+    lines.push("");
+
+    // ── System prompt (from step 1, collapsed) ───────────────────────────────
+    const firstStep = this.richSteps[0];
+    if (firstStep) {
+      lines.push("## System Prompt");
+      lines.push("");
+      lines.push("<details>");
+      lines.push("<summary>View system prompt</summary>");
+      lines.push("");
+      lines.push("```");
+      lines.push(firstStep.systemPrompt);
+      lines.push("```");
+      lines.push("");
+      lines.push("</details>");
+      lines.push("");
+      lines.push("---");
+      lines.push("");
+    }
+
+    // ── Steps ────────────────────────────────────────────────────────────────
+    for (const step of this.richSteps) {
+      const toolLabel =
+        step.toolCalls.map((tc) => `\`${tc.name}\``).join(", ") || "*(no tool calls)*";
+      lines.push(`## Step ${step.stepNumber} — ${toolLabel}`);
+      lines.push("");
+      const dynLabel =
+        step.dynamicToolsCreated.length > 0
+          ? step.dynamicToolsCreated.map((t) => `\`${t}\``).join(", ")
+          : "none";
+      lines.push(
+        `**Tokens:** ${step.tokens} | **Chunks retrieved:** ${step.chunks.length} | **Dynamic tools created:** ${dynLabel}`,
+      );
+      lines.push("");
+
+      // Reasoning
+      if (step.text) {
+        lines.push("### Reasoning");
+        lines.push("");
+        lines.push(step.text);
+        lines.push("");
+      }
+
+      // Tool calls
+      if (step.toolCalls.length > 0) {
+        lines.push("### Tool Calls");
+        lines.push("");
+        for (const [i, tc] of step.toolCalls.entries()) {
+          lines.push(`#### ${i + 1}. \`${tc.name}\``);
+          lines.push("");
+          lines.push("**Input:**");
+          lines.push("```json");
+          lines.push(JSON.stringify(tc.input, null, 2));
+          lines.push("```");
+          lines.push("");
+
+          if (tc.output !== undefined) {
+            let parsedOutput: any;
+            try {
+              parsedOutput =
+                typeof tc.output === "string" ? JSON.parse(tc.output) : tc.output;
+            } catch {
+              parsedOutput = tc.output;
+            }
+            const outputStr = JSON.stringify(parsedOutput, null, 2);
+            const truncated = outputStr.length > 2000;
+            lines.push("**Output:**");
+            lines.push("```json");
+            lines.push(truncated ? `${outputStr.slice(0, 2000)}\n… (truncated)` : outputStr);
+            lines.push("```");
+            lines.push("");
+          }
+        }
+      }
+
+      // Chunks table
+      if (step.chunks.length > 0) {
+        lines.push("### Chunks Retrieved");
+        lines.push("");
+        lines.push("| # | Item | Context | Chunk | Score |");
+        lines.push("|---|------|---------|-------|-------|");
+        for (const [i, c] of step.chunks.entries()) {
+          const score =
+            c.metadata?.hybrid_score ??
+            c.metadata?.cosine_distance ??
+            c.metadata?.fts_rank ??
+            "—";
+          const scoreStr = typeof score === "number" ? score.toFixed(4) : String(score);
+          lines.push(
+            `| ${i + 1} | ${c.item_name ?? "—"} | \`${c.context}\` | ${c.chunk_index ?? "—"} | ${scoreStr} |`,
+          );
+        }
+        lines.push("");
+
+        const withContent = step.chunks.filter((c) => c.chunk_content);
+        if (withContent.length > 0) {
+          lines.push("<details>");
+          lines.push("<summary>View chunk content</summary>");
+          lines.push("");
+          for (const c of withContent) {
+            lines.push(`**${c.item_name} (chunk ${c.chunk_index}):**`);
+            lines.push("");
+            const content = (c.chunk_content ?? "").trim();
+            lines.push(`> ${content.split("\n").join("\n> ")}`);
+            lines.push("");
+          }
+          lines.push("</details>");
+          lines.push("");
+        }
+      }
+
+      // Per-step system prompt addendum (only when it differs from step 1)
+      if (firstStep && step.stepNumber > 1 && step.systemPrompt !== firstStep.systemPrompt) {
+        const addendum = step.systemPrompt.slice(firstStep.systemPrompt.length).trim();
+        if (addendum) {
+          lines.push("<details>");
+          lines.push("<summary>System prompt addendum (this step only)</summary>");
+          lines.push("");
+          lines.push("```");
+          lines.push(addendum);
+          lines.push("```");
+          lines.push("");
+          lines.push("</details>");
+          lines.push("");
+        }
+      }
+
+      lines.push("---");
+      lines.push("");
+    }
+
+    // ── Summary ──────────────────────────────────────────────────────────────
+    lines.push("## Summary");
+    lines.push("");
+    lines.push("| Metric | Value |");
+    lines.push("|--------|-------|");
+    lines.push(`| Steps | ${this.richSteps.length} |`);
+    lines.push(`| Total chunks | ${totalChunks} |`);
+    lines.push(`| Total tokens | ${totalTokens} |`);
+    lines.push(`| Duration | ${(durationMs / 1000).toFixed(1)}s |`);
+    lines.push(`| Status | ${status} |`);
+    if (error) {
+      lines.push(`| Error | ${error.message} |`);
+    }
+    lines.push("");
+
+    return lines.join("\n");
+  }
+
+  async finalize(
+    output: AgenticRetrievalOutput,
+    success: boolean,
+    error?: Error,
+    writeFiles = false,
+  ): Promise<string | undefined> {
+    const durationMs = Date.now() - this.startTime;
+
     this.data.final = {
       total_chunks: output.chunks.length,
       total_steps: output.steps.length,
       total_tokens: output.totalTokens,
-      duration_ms: Date.now() - this.startTime,
+      duration_ms: durationMs,
       success,
       error: error?.message,
     };
 
+    if (!writeFiles) return undefined;
+
     try {
       await fs.mkdir(this.logDir, { recursive: true });
-      const filename = `trajectory_${Date.now()}.json`;
-      const fullPath = path.join(this.logDir, filename);
-      await fs.writeFile(fullPath, JSON.stringify(this.data, null, 2), "utf-8");
-      console.log(`[EXULU] v3 trajectory saved: ${filename}`);
-      trajectoryRegistry.lastFile = fullPath;
-      return fullPath;
+      const ts = Date.now();
+      const jsonPath = path.join(this.logDir, `trajectory_${ts}.json`);
+      const mdPath = path.join(this.logDir, `trajectory_${ts}.md`);
+
+      await Promise.all([
+        fs.writeFile(jsonPath, JSON.stringify(this.data, null, 2), "utf-8"),
+        fs.writeFile(mdPath, this.toMarkdown(durationMs, success, error), "utf-8"),
+      ]);
+
+      console.log(`[EXULU] v3 trajectory saved: trajectory_${ts}.json + trajectory_${ts}.md`);
+      trajectoryRegistry.lastFile = jsonPath;
+      return jsonPath;
     } catch (e) {
       console.error("[EXULU] v3 failed to write trajectory:", e);
       return undefined;

package/ee/agentic-retrieval/v4/agent-loop.ts

@@ -1,32 +1,82 @@
-import { generateText, stepCountIs } from "ai";
+import { generateText, stepCountIs, tool } from "ai";
 import type { LanguageModel, Tool as AITool, ModelMessage } from "ai";
+import { z } from "zod";
 import { withRetry } from "@SRC/utils/with-retry";
-import { harvestChunks } from "./tools";
+import type { ExuluReranker } from "@SRC/exulu/reranker";
 import type { AgenticRetrievalOutput, ChunkResult } from "./types";
+import { DEFAULT_MAX_STEPS, type AgenticRetrievalLog, type ContextRetrievalConfig } from ".";
 
-const MAX_STEPS = 10;
+const FINISH_TOOL_NAME = "finish_retrieval";
+
+const finishRetrievalTool = tool({
+  description:
+    "Call this tool when you have retrieved sufficient information and no further searches are needed. " +
+    "You MUST call this tool to signal that retrieval is complete — do not write a text conclusion.",
+  inputSchema: z.object({
+    reasoning: z.string().describe("One sentence explaining why retrieval is complete"),
+  }),
+  execute: async ({ reasoning }) => JSON.stringify({ finished: true, reasoning }),
+});
+
+function extractChunksFromToolResults(toolResults: any[]): ChunkResult[] {
+  const chunks: ChunkResult[] = [];
+  for (const result of toolResults ?? []) {
+    // AI SDK v6 uses `output` (not `result`) for tool result values
+    const rawOutput = result.output ?? result.result;
+    let parsed: any;
+    try {
+      parsed = typeof rawOutput === "string" ? JSON.parse(rawOutput) : rawOutput;
+    } catch {
+      continue;
+    }
+
+    if (Array.isArray(parsed)) {
+      for (const item of parsed) {
+        if (item?.item_id && item?.context) {
+          chunks.push({
+            item_name: item.item_name,
+            item_id: item.item_id,
+            context: item.context?.id ?? item.context,
+            chunk_id: item.chunk_id,
+            chunk_index: item.chunk_index,
+            chunk_content: item.chunk_content,
+            metadata: item.metadata,
+          });
+        }
+      }
+    }
+  }
+  return chunks;
+}
 
 /**
- * Observe → Infer → Act loop for V4 agentic retrieval.
+ * Core agent loop: one generateText call per step.
  *
- * Unlike V3 (which pre-classifies, routes to strategies, and forces tool calls),
- * this loop simply:
- *  1. Calls the model with toolChoice "auto"
- *  2. Executes whatever tools the model picks
- *  3. Harvests any chunk-shaped rows from query results
- *  4. Repeats until the model produces a text response (no tool calls) or
- *     the MAX_STEPS budget is exhausted
+ * Unlike v2 (which split each step into a reasoning call + a separate tool
+ * execution call), here a single call with toolChoice: "auto" lets the model
+ * reason and call tools in one pass. The model sees tool results from the
+ * previous step via the conversation history (messages array).
  *
- * The model decides when it has enough information — no finish_retrieval tool needed.
+ * The loop stops when:
+ * - The model makes no tool calls (it's satisfied), OR
+ * - The strategy's stepBudget is exhausted
  */
 export async function* runAgentLoop(params: {
-  query: string;
-  systemPrompt: string;
+  config: ContextRetrievalConfig;
+  userQuery: string;
+  log: AgenticRetrievalLog;
+  todos: {
+    status: "planned" | "completed";
+    description: string;
+    current: boolean;
+  }[];
   tools: Record<string, AITool>;
   model: LanguageModel;
+  reranker?: ExuluReranker;
+  sessionID?: string;
   onStepComplete?: (step: AgenticRetrievalOutput["steps"][0]) => void;
 }): AsyncGenerator<AgenticRetrievalOutput> {
-  const { query, systemPrompt, tools, model, onStepComplete } = params;
+  const { userQuery, tools, model, reranker, sessionID, onStepComplete, config, log, todos } = params;
 
   const output: AgenticRetrievalOutput = {
     steps: [],
@@ -36,83 +86,120 @@ export async function* runAgentLoop(params: {
     totalTokens: 0,
   };
 
-  // Deduplicate chunks by chunk_id across all steps
-  const seenChunkIds = new Set<string>();
+  const messages: ModelMessage[] = [{ role: "user", content: userQuery }];
+
+  const stepBudget = config.maxSteps || DEFAULT_MAX_STEPS
+
+  const SYSTEM_PROMPT = `
+  You are a helpful assistant that can search the knowledge base and retrieve information.
+
+  You are searching for information that is relevant to the following question:
+  <user_query>
+  ${userQuery}
+  </user_query>
 
-  const messages: ModelMessage[] = [{ role: "user", content: query }];
+  You have the following instructions for this knowledge base:
+  <instructions>
+  ${config.instructions}
+  </instructions>
 
-  for (let step = 0; step < MAX_STEPS; step++) {
-    console.log(`[EXULU] v4 agent loop — step ${step + 1}/${MAX_STEPS}`);
+  A first search strategy was drafted as a todo list:
+  <todo_list>
+  ${todos.map((todo, index) => `${index + 1}. ${todo.status} - ${todo.description}`).join("\n")}
+  </todo_list>
+
+  `;
+
+  for (let step = 0; step < stepBudget; step++) {
+
+    log.entries.push({
+      label: "Agent loop step",
+      timestamp: new Date().toISOString(),
+      message: `[EXULU] v3 agent loop — step ${step + 1}/${stepBudget}`,
+    });
 
     let result: Awaited<ReturnType<typeof generateText>>;
+
+    const stepTools = { ...tools, [FINISH_TOOL_NAME]: finishRetrievalTool };
+    
     try {
       result = await withRetry(() =>
         generateText({
           model,
           temperature: 0,
-          system: systemPrompt,
+          system: SYSTEM_PROMPT,
           messages,
-          tools,
-          toolChoice: "auto",
+          tools: stepTools,
+          toolChoice: "required",
           stopWhen: stepCountIs(1),
         }),
       );
     } catch (err) {
-      console.error("[EXULU] v4 generateText failed:", err);
+      console.error("[EXULU] v3 generateText failed:", err);
       throw err;
     }
 
-    // Append assistant turn + tool results to conversation history
+    // Carry conversation forward: assistant message + tool results go into history
+    // so the model sees them on the next iteration.
     messages.push(...(result.response.messages as ModelMessage[]));
 
-    // Harvest chunks from any execute_query tool results
-    const rawToolResults = (result.toolResults as any[]) ?? [];
-    const stepChunks: ChunkResult[] = [];
-    for (const chunk of harvestChunks(rawToolResults)) {
-      if (!chunk.chunk_id || !seenChunkIds.has(chunk.chunk_id)) {
-        if (chunk.chunk_id) seenChunkIds.add(chunk.chunk_id);
-        stepChunks.push(chunk);
-      }
-    }
+    // Extract chunks from tool results
+    let stepChunks: any[] = extractChunksFromToolResults(result.toolResults as any[]);
+
+    // Deduplicate by chunk_id within this step (parallel tool calls can return the same chunk
+    // if the agent searches the same context twice, or the same chunk is indexed in two contexts).
+    const seenChunkIds = new Set<string>();
+    stepChunks = stepChunks.filter((c) => {
+      if (!c.chunk_id) return true;
+      if (seenChunkIds.has(c.chunk_id)) return false;
+      seenChunkIds.add(c.chunk_id);
+      return true;
+    });
 
     // Record step
-    const stepRecord: AgenticRetrievalOutput["steps"][0] = {
+    const stepRecord = {
       stepNumber: step + 1,
       text: result.text ?? "",
-      toolCalls:
-        (result.toolCalls as any[])?.map((tc) => ({
-          name: tc.toolName,
-          id: tc.toolCallId,
-          input: tc.input,
-        })) ?? [],
+      toolCalls: (result.toolCalls as any[])?.map((tc) => ({
+        name: tc.toolName,
+        id: tc.toolCallId,
+        input: tc.input,
+      })) ?? [],
       chunks: stepChunks,
       tokens: result.usage?.totalTokens ?? 0,
     };
 
+    log.entries.push({
+      label: "Step completed",
+      timestamp: new Date().toISOString(),
+      message: JSON.stringify(stepRecord),
+    });
+
     output.steps.push(stepRecord);
     output.reasoning.push({
       text: result.text ?? "",
-      tools:
-        (result.toolCalls as any[])?.map((tc) => ({
-          name: tc.toolName,
-          id: tc.toolCallId,
-          input: tc.input,
-          output: rawToolResults.find(
-            (r: any) => (r.toolCallId ?? r.id) === tc.toolCallId,
-          )?.output,
-        })) ?? [],
+      tools: (result.toolCalls as any[])?.map((tc) => ({
+        name: tc.toolName,
+        id: tc.toolCallId,
+        input: tc.input,
+        output: stepChunks,
+      })) ?? [],
     });
-    output.chunks.push(...stepChunks);
+    // Deduplicate against chunks already accumulated from prior steps
+    const existingChunkIds = new Set(output.chunks.map((c) => c.chunk_id).filter(Boolean));
+    output.chunks.push(...stepChunks.filter((c) => !c.chunk_id || !existingChunkIds.has(c.chunk_id)));
     output.usage.push(result.usage);
 
     onStepComplete?.(stepRecord);
 
     yield { ...output };
 
-    // Stop when the model wrote a text response without calling any tools
-    const calledTools = (result.toolCalls as any[])?.length > 0;
-    if (!calledTools) {
-      console.log(`[EXULU] v4 — model finished after step ${step + 1} (no tool calls)`);
+    // Stop if the model called finish_retrieval AND no forced continuation is needed
+    const calledFinish = (result.toolCalls as any[])?.some(
+      (tc) => tc.toolName === FINISH_TOOL_NAME,
+    );
+    if (calledFinish) {
+      console.log(`[EXULU] v3 model called finish_retrieval after step ${step + 1}`);
       break;
     }
   }

package/ee/agentic-retrieval/v4/context-sampler.ts

@@ -0,0 +1,79 @@
+import { ExuluContext, getTableName } from "@SRC/exulu/context";
+import { postgresClient } from "@SRC/postgres/client";
+import { applyAccessControl } from "@SRC/graphql/utilities/access-control";
+import { convertContextToTableDefinition } from "@SRC/graphql/utilities/convert-context-to-table-definition";
+import type { User } from "@EXULU_TYPES/models/user";
+
+const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
+
+export interface ContextSample {
+  contextId: string;
+  contextName: string;
+  /** All field names available on items (standard + custom) */
+  fields: string[];
+  /** Up to 2 example item records */
+  exampleItems: Array<Record<string, any>>;
+  sampledAt: number;
+}
+
+/**
+ * Pulls 1–2 example item records per context at agent initialization and caches
+ * them in memory. These samples are injected into the classifier prompt so the
+ * model understands what data is actually stored (not just field names).
+ */
+export class ContextSampler {
+  private cache = new Map<string, ContextSample>();
+
+  async getSamples(
+    contexts: ExuluContext[],
+    user?: User,
+    role?: string,
+  ): Promise<ContextSample[]> {
+    return Promise.all(contexts.map((ctx) => this.getSample(ctx, user, role)));
+  }
+
+  private async getSample(
+    ctx: ExuluContext,
+    user?: User,
+    role?: string,
+  ): Promise<ContextSample> {
+    const cached = this.cache.get(ctx.id);
+    if (cached && Date.now() - cached.sampledAt < CACHE_TTL_MS) {
+      return cached;
+    }
+
+    const { db } = await postgresClient();
+    const tableName = getTableName(ctx.id);
+    const tableDefinition = convertContextToTableDefinition(ctx);
+
+    const customFieldNames = ctx.fields.map((f) => f.name);
+    const selectFields = ["id", "name", "external_id", ...customFieldNames];
+
+    let exampleItems: Record<string, any>[] = [];
+    try {
+      let query = db(tableName).select(selectFields).whereNull("archived").limit(2);
+      query = applyAccessControl(tableDefinition, query, user, tableName);
+      exampleItems = await query;
+    } catch {
+      // If table doesn't exist yet or column mismatch, return empty samples
+    }
+
+    const sample: ContextSample = {
+      contextId: ctx.id,
+      contextName: ctx.name,
+      fields: ["name", "external_id", ...customFieldNames],
+      exampleItems,
+      sampledAt: Date.now(),
+    };
+
+    this.cache.set(ctx.id, sample);
+
+    // Refresh in background after TTL without blocking the caller
+    return sample;
+  }
+
+  /** Evict a context from cache so it's re-sampled on next use */
+  invalidate(contextId: string): void {
+    this.cache.delete(contextId);
+  }
+}