@exulu/backend 1.53.1 → 1.55.0

package/ee/agentic-retrieval/v3/trajectory.ts

@@ -0,0 +1,309 @@
+import * as fs from "fs/promises";
+import * as path from "path";
+import type { AgenticRetrievalOutput, ClassificationResult, ChunkResult } from "./types";
+
+export const trajectoryRegistry = {
+  lastFile: undefined as string | undefined,
+};
+
+export interface TrajectoryStepData {
+  stepNumber: number;
+  systemPrompt: string;
+  text: string;
+  toolCalls: Array<{
+    name: string;
+    id: string;
+    input: any;
+    output?: any;
+  }>;
+  chunks: ChunkResult[];
+  dynamicToolsCreated: string[];
+  tokens: number;
+}
+
+interface TrajectoryData {
+  timestamp: string;
+  query: string;
+  classification: ClassificationResult;
+  preselectedItemIds?: string[];
+  steps: {
+    step_number: number;
+    text: string;
+    tool_calls: { name: string; id: string; input: any }[];
+    chunks_retrieved: number;
+    dynamic_tools_created: string[];
+    tokens: number;
+  }[];
+  final: {
+    total_chunks: number;
+    total_steps: number;
+    total_tokens: number;
+    duration_ms: number;
+    success: boolean;
+    error?: string;
+  };
+}
+
+export class TrajectoryLogger {
+  private data: TrajectoryData;
+  private richSteps: TrajectoryStepData[] = [];
+  private startTime = Date.now();
+  private logDir: string;
+
+  constructor(
+    query: string,
+    classification: ClassificationResult,
+    logDir = path.join(process.cwd(), "ee/agentic-retrieval/logs"),
+    preselectedItemIds?: string[],
+  ) {
+    this.logDir = logDir;
+    this.data = {
+      timestamp: new Date().toISOString(),
+      query,
+      classification,
+      preselectedItemIds: preselectedItemIds?.length ? preselectedItemIds : undefined,
+      steps: [],
+      final: {
+        total_chunks: 0,
+        total_steps: 0,
+        total_tokens: 0,
+        duration_ms: 0,
+        success: false,
+      },
+    };
+  }
+
+  recordStep(step: AgenticRetrievalOutput["steps"][0]): void {
+    this.data.steps.push({
+      step_number: step.stepNumber,
+      text: step.text,
+      tool_calls: step.toolCalls,
+      chunks_retrieved: step.chunks.length,
+      dynamic_tools_created: step.dynamicToolsCreated,
+      tokens: step.tokens,
+    });
+  }
+
+  recordRichStep(data: TrajectoryStepData): void {
+    this.richSteps.push(data);
+  }
+
+  private toMarkdown(durationMs: number, success: boolean, error?: Error): string {
+    const totalTokens = this.richSteps.reduce((sum, s) => sum + s.tokens, 0);
+    const totalChunks = this.richSteps.reduce((sum, s) => sum + s.chunks.length, 0);
+    const status = success ? "✓ Success" : `✗ Failed${error ? `: ${error.message}` : ""}`;
+    const lines: string[] = [];
+
+    // ── Header ──────────────────────────────────────────────────────────────
+    lines.push(`# Agentic Retrieval — ${this.data.timestamp}`);
+    lines.push("");
+    lines.push(`**Query:** ${this.data.query}  `);
+    lines.push(
+      `**Duration:** ${(durationMs / 1000).toFixed(1)}s | **Tokens:** ${totalTokens} | **Status:** ${status}`,
+    );
+    lines.push("");
+
+    // ── Classification ───────────────────────────────────────────────────────
+    lines.push("## Classification");
+    lines.push("");
+    lines.push(`- **Type:** \`${this.data.classification.queryType}\``);
+    lines.push(`- **Language:** \`${this.data.classification.language}\``);
+    const suggested = this.data.classification.suggestedContextIds;
+    lines.push(
+      `- **Suggested contexts:** ${suggested.length > 0 ? suggested.map((id) => `\`${id}\``).join(", ") : "*(all)*"}`,
+    );
+    if (this.data.preselectedItemIds?.length) {
+      lines.push(
+        `- **Preselected item IDs:** ${this.data.preselectedItemIds.map((id) => `\`${id}\``).join(", ")}`,
+      );
+    }
+    lines.push("");
+    lines.push("---");
+    lines.push("");
+
+    // ── System prompt (from step 1, collapsed) ───────────────────────────────
+    const firstStep = this.richSteps[0];
+    if (firstStep) {
+      lines.push("## System Prompt");
+      lines.push("");
+      lines.push("<details>");
+      lines.push("<summary>View system prompt</summary>");
+      lines.push("");
+      lines.push("```");
+      lines.push(firstStep.systemPrompt);
+      lines.push("```");
+      lines.push("");
+      lines.push("</details>");
+      lines.push("");
+      lines.push("---");
+      lines.push("");
+    }
+
+    // ── Steps ────────────────────────────────────────────────────────────────
+    for (const step of this.richSteps) {
+      const toolLabel =
+        step.toolCalls.map((tc) => `\`${tc.name}\``).join(", ") || "*(no tool calls)*";
+      lines.push(`## Step ${step.stepNumber} — ${toolLabel}`);
+      lines.push("");
+      const dynLabel =
+        step.dynamicToolsCreated.length > 0
+          ? step.dynamicToolsCreated.map((t) => `\`${t}\``).join(", ")
+          : "none";
+      lines.push(
+        `**Tokens:** ${step.tokens} | **Chunks retrieved:** ${step.chunks.length} | **Dynamic tools created:** ${dynLabel}`,
+      );
+      lines.push("");
+
+      // Reasoning
+      if (step.text) {
+        lines.push("### Reasoning");
+        lines.push("");
+        lines.push(step.text);
+        lines.push("");
+      }
+
+      // Tool calls
+      if (step.toolCalls.length > 0) {
+        lines.push("### Tool Calls");
+        lines.push("");
+        for (const [i, tc] of step.toolCalls.entries()) {
+          lines.push(`#### ${i + 1}. \`${tc.name}\``);
+          lines.push("");
+          lines.push("**Input:**");
+          lines.push("```json");
+          lines.push(JSON.stringify(tc.input, null, 2));
+          lines.push("```");
+          lines.push("");
+
+          if (tc.output !== undefined) {
+            let parsedOutput: any;
+            try {
+              parsedOutput =
+                typeof tc.output === "string" ? JSON.parse(tc.output) : tc.output;
+            } catch {
+              parsedOutput = tc.output;
+            }
+            const outputStr = JSON.stringify(parsedOutput, null, 2);
+            const truncated = outputStr.length > 2000;
+            lines.push("**Output:**");
+            lines.push("```json");
+            lines.push(truncated ? `${outputStr.slice(0, 2000)}\n… (truncated)` : outputStr);
+            lines.push("```");
+            lines.push("");
+          }
+        }
+      }
+
+      // Chunks table
+      if (step.chunks.length > 0) {
+        lines.push("### Chunks Retrieved");
+        lines.push("");
+        lines.push("| # | Item | Context | Chunk | Score |");
+        lines.push("|---|------|---------|-------|-------|");
+        for (const [i, c] of step.chunks.entries()) {
+          const score =
+            c.metadata?.hybrid_score ??
+            c.metadata?.cosine_distance ??
+            c.metadata?.fts_rank ??
+            "—";
+          const scoreStr = typeof score === "number" ? score.toFixed(4) : String(score);
+          lines.push(
+            `| ${i + 1} | ${c.item_name ?? "—"} | \`${c.context}\` | ${c.chunk_index ?? "—"} | ${scoreStr} |`,
+          );
+        }
+        lines.push("");
+
+        const withContent = step.chunks.filter((c) => c.chunk_content);
+        if (withContent.length > 0) {
+          lines.push("<details>");
+          lines.push("<summary>View chunk content</summary>");
+          lines.push("");
+          for (const c of withContent) {
+            lines.push(`**${c.item_name} (chunk ${c.chunk_index}):**`);
+            lines.push("");
+            const content = (c.chunk_content ?? "").trim();
+            lines.push(`> ${content.split("\n").join("\n> ")}`);
+            lines.push("");
+          }
+          lines.push("</details>");
+          lines.push("");
+        }
+      }
+
+      // Per-step system prompt addendum (only when it differs from step 1)
+      if (firstStep && step.stepNumber > 1 && step.systemPrompt !== firstStep.systemPrompt) {
+        const addendum = step.systemPrompt.slice(firstStep.systemPrompt.length).trim();
+        if (addendum) {
+          lines.push("<details>");
+          lines.push("<summary>System prompt addendum (this step only)</summary>");
+          lines.push("");
+          lines.push("```");
+          lines.push(addendum);
+          lines.push("```");
+          lines.push("");
+          lines.push("</details>");
+          lines.push("");
+        }
+      }
+
+      lines.push("---");
+      lines.push("");
+    }
+
+    // ── Summary ──────────────────────────────────────────────────────────────
+    lines.push("## Summary");
+    lines.push("");
+    lines.push("| Metric | Value |");
+    lines.push("|--------|-------|");
+    lines.push(`| Steps | ${this.richSteps.length} |`);
+    lines.push(`| Total chunks | ${totalChunks} |`);
+    lines.push(`| Total tokens | ${totalTokens} |`);
+    lines.push(`| Duration | ${(durationMs / 1000).toFixed(1)}s |`);
+    lines.push(`| Status | ${status} |`);
+    if (error) {
+      lines.push(`| Error | ${error.message} |`);
+    }
+    lines.push("");
+
+    return lines.join("\n");
+  }
+
+  async finalize(
+    output: AgenticRetrievalOutput,
+    success: boolean,
+    error?: Error,
+    writeFiles = false,
+  ): Promise<string | undefined> {
+    const durationMs = Date.now() - this.startTime;
+
+    this.data.final = {
+      total_chunks: output.chunks.length,
+      total_steps: output.steps.length,
+      total_tokens: output.totalTokens,
+      duration_ms: durationMs,
+      success,
+      error: error?.message,
+    };
+
+    if (!writeFiles) return undefined;
+
+    try {
+      await fs.mkdir(this.logDir, { recursive: true });
+      const ts = Date.now();
+      const jsonPath = path.join(this.logDir, `trajectory_${ts}.json`);
+      const mdPath = path.join(this.logDir, `trajectory_${ts}.md`);
+
+      await Promise.all([
+        fs.writeFile(jsonPath, JSON.stringify(this.data, null, 2), "utf-8"),
+        fs.writeFile(mdPath, this.toMarkdown(durationMs, success, error), "utf-8"),
+      ]);
+
+      console.log(`[EXULU] v3 trajectory saved: trajectory_${ts}.json + trajectory_${ts}.md`);
+      trajectoryRegistry.lastFile = jsonPath;
+      return jsonPath;
+    } catch (e) {
+      console.error("[EXULU] v3 failed to write trajectory:", e);
+      return undefined;
+    }
+  }
+}

package/ee/agentic-retrieval/v3/types.ts

@@ -0,0 +1,59 @@
+export type QueryType = "aggregate" | "list" | "targeted" | "exploratory";
+
+export interface ClassificationResult {
+  queryType: QueryType;
+  language: string;
+  /** IDs of contexts most likely relevant. Empty means search all. */
+  suggestedContextIds: string[];
+}
+
+export interface ContextSample {
+  contextId: string;
+  contextName: string;
+  /** All field names available on items (standard + custom) */
+  fields: string[];
+  /** Up to 2 example item records */
+  exampleItems: Array<Record<string, any>>;
+  sampledAt: number;
+}
+
+export interface ChunkResult {
+  item_name: string;
+  item_id: string;
+  context: string;
+  chunk_id?: string;
+  chunk_index?: number;
+  chunk_content?: string;
+  metadata?: Record<string, any>;
+}
+
+export interface RetrievalStep {
+  stepNumber: number;
+  /** Text the model output during this step (reasoning) */
+  text: string;
+  toolCalls: Array<{ name: string; id: string; input: any }>;
+  chunks: ChunkResult[];
+  dynamicToolsCreated: string[];
+  tokens: number;
+}
+
+interface Reasoning {
+  text: string;
+  tools: {
+    name: string;
+    id: string;
+    input: any;
+    output: any;
+  }[]
+}
+
+export interface AgenticRetrievalOutput {
+  steps: RetrievalStep[];
+  reasoning: Reasoning[];
+  /** All chunks collected across all steps */
+  chunks: ChunkResult[];
+  usage: any[];
+  totalTokens: number;
+  /** Path to the trajectory JSON file written to disk, if any */
+  trajectoryFile?: string;
+}

package/ee/chunking/markdown.ts

@@ -516,7 +516,9 @@ export class MarkdownChunker {
         return newHeaders;
     }
 
-    public async chunk(text: string, chunkSize: number, prefix?: string): Promise<{
+    public async chunk(text: string, chunkSize: number, prefix?: string, config?: {
+        pageBreakTags?: boolean;
+    }): Promise<{
         text: string;
         page: number;
     }[]> {
@@ -684,7 +686,7 @@ export class MarkdownChunker {
                     finalText = headerPrefixText + '\n\n' + currentSlice;
                 }
 
-                if (currentPage) {
+                if (currentPage && config?.pageBreakTags) {
                     finalText = `<!-- Current page: ${currentPage} -->\n\n` + finalText;
                 }
 

package/ee/invoke-skills/create-sandbox.ts

@@ -0,0 +1,119 @@
+import {
+    SandboxManager,
+    type SandboxRuntimeConfig,
+} from '@anthropic-ai/sandbox-runtime'
+import { mkdir, rm, writeFile } from 'node:fs/promises'
+import { join, dirname } from 'node:path'
+import { listS3ObjectsByPrefix, getS3ObjectContent } from '@SRC/uppy/index.ts'
+import type { ExuluConfig } from '@SRC/exulu/app/index.ts'
+
+import { tool } from 'ai'
+import { z } from 'zod'
+
+// This is called on every session where a skill is enabled
+// each sandbox setup includes the skill files from the enabled
+// skills, and uses the Anthropic Sandbox Runtime (SRT) to
+// limit read and write scopes.
+
+export interface SkillRef {
+    id: string
+    name: string
+    s3folder: string
+    current_version: number
+}
+
+export interface SkillSandboxHandle {
+    /** Absolute path to the session's temporary directory, containing all downloaded skill files. */
+    sessionDir: string
+    /** Wraps a shell command string so it runs inside the sandbox. */
+    wrapCommand: (command: string) => Promise<string>
+    /** Tears down the sandbox and deletes the session directory. */
+    cleanup: () => Promise<void>
+}
+
+/**
+ * Creates a sandboxed environment for a session:
+ * 1. Creates a temp directory at /tmp/exulu-sessions/<sessionId>
+ * 2. Downloads all files for each enabled skill into <sessionDir>/skills/<skillName>/
+ * 3. Initialises the SRT SandboxManager with filesystem access scoped to sessionDir only
+ *    and no network access
+ */
+export async function createSkillSandbox(
+    sessionId: string,
+    skills: SkillRef[],
+    config: ExuluConfig,
+): Promise<SkillSandboxHandle> {
+    const sessionDir = join('/tmp', 'exulu-sessions', sessionId)
+
+    await mkdir(sessionDir, { recursive: true })
+
+    const skillsDirectory = join(sessionDir, 'skills')
+
+    // Download each skill's files from S3 into the session directory
+    for (const skill of skills) {
+        const versionPrefix = `skills/${skill.id}/v${skill.current_version}/`
+        const files = await listS3ObjectsByPrefix(versionPrefix, config)
+
+        for (const file of files) {
+            // Extract the path relative to the version prefix, accounting for any S3 general prefix
+            const prefixIndex = file.key.indexOf(versionPrefix)
+            const relativePath = prefixIndex >= 0
+                ? file.key.slice(prefixIndex + versionPrefix.length)
+                : file.key
+
+            if (!relativePath) continue // skip directory markers
+
+            const localPath = join(skillsDirectory, skill.name, relativePath)
+            await mkdir(dirname(localPath), { recursive: true })
+
+            const content = await getS3ObjectContent(file.key, config)
+            await writeFile(localPath, content, 'utf-8')
+        }
+    }
+
+    const sandboxConfig: SandboxRuntimeConfig = {
+        network: {
+            allowedDomains: [], // todo
+            deniedDomains: [], // todo
+        },
+        filesystem: {
+            // Deny reads to home directory but re-allow only the session folder.
+            // System paths (/usr, /lib, etc.) remain readable for process execution.
+            denyRead: ['~'],
+            allowRead: [sessionDir],
+            // Write access is scoped exclusively to the session folder.
+            allowWrite: [sessionDir],
+            denyWrite: [],
+        },
+    }
+
+    await SandboxManager.initialize(sandboxConfig)
+
+    // Todo proper instructions to use skills
+
+    /* const bashTool = function createBashTool() {
+        return tool({
+            description: `
+            Execute bash commands inside the sandbox.
+            Examples (not exhaustive): ls, cat, less, head, tail, grep
+            `,
+            inputSchema: z.object({
+                command: z.string().describe('The bash command to execute'),
+                args: z.array(z.string()).describe('Arguments to pass to the command')
+            }),
+            execute: async ({ command, args }) => {
+                // code that executes when the tool is called
+                return await SandboxManager.wrapWithSandbox(command)
+            }
+        });
+    } */
+
+    return {
+        sessionDir,
+        wrapCommand: (command: string) => SandboxManager.wrapWithSandbox(command),
+        cleanup: async () => {
+            await SandboxManager.reset()
+            await rm(sessionDir, { recursive: true, force: true })
+        },
+    }
+}

package/ee/python/documents/processing/doc_processor.ts

@@ -94,6 +94,84 @@ async function processWord(file: Buffer): Promise<ProcessorOutput> {
   }
 }
 
+/**
+ * Processes a standalone image file by optionally extracting content using VLM
+ */
+async function processImage(
+  buffer: Buffer,
+  paths: ProcessingPaths,
+  config?: DocumentProcessorConfig,
+  verbose: boolean = false,
+): Promise<ProcessorOutput> {
+  try {
+    // Create images directory
+    await fs.promises.mkdir(paths.images, { recursive: true });
+
+    // Save the image
+    const imagePath = path.join(paths.images, '1.png');
+    await fs.promises.writeFile(imagePath, buffer);
+
+    console.log(`[EXULU] Image saved to: ${imagePath}`);
+
+    // Create initial ProcessedDocument with minimal content
+    let json: ProcessedDocument = [{
+      page: 1,
+      content: '', // Empty initially, will be populated by VLM if enabled
+      image: imagePath,
+      headings: [],
+    }];
+
+    // If VLM is enabled, use it to extract content from the image
+    if (config?.vlm?.model) {
+      console.log('[EXULU] Extracting content from image using VLM...');
+
+      json = await validateWithVLM(
+        json,
+        config.vlm.model,
+        verbose,
+        config.vlm.concurrency
+      );
+
+      // Save the processed result
+      await fs.promises.writeFile(
+        paths.json,
+        JSON.stringify(json, null, 2),
+        'utf-8'
+      );
+
+      console.log('[EXULU] VLM content extraction complete');
+
+      const correctedCount = json.filter(p => p.vlm_corrected_text).length;
+      console.log(`[EXULU] Content extracted: ${correctedCount > 0 ? 'Yes' : 'No'}`);
+    } else {
+      console.log('[EXULU] No VLM configured, image saved without content extraction');
+      console.log('[EXULU] Note: Enable VLM in config to extract text/content from images');
+
+      // Save empty result
+      await fs.promises.writeFile(
+        paths.json,
+        JSON.stringify(json, null, 2),
+        'utf-8'
+      );
+    }
+
+    // Build markdown from content
+    const markdown = json.map(p => p.vlm_corrected_text ?? p.content).join('\n\n\n<!-- END_OF_PAGE -->\n\n\n');
+
+    // Save markdown
+    await fs.promises.writeFile(paths.markdown, markdown, 'utf-8');
+
+    return {
+      markdown: markdown,
+      json: json,
+    };
+
+  } catch (error) {
+    console.error('[EXULU] Error processing image:', error);
+    throw error;
+  }
+}
+
 /**
  * Normalizes markdown content by removing excessive whitespace,
  * especially in table formatting.
@@ -307,6 +385,8 @@ If the page contains a flow-chart, schematic, technical drawing or control board
     reasoning: parsedOutput.reasoning,
   };
 
+  console.log(`[EXULU] VLM validation result: ${JSON.stringify(validation)}`);
+
   return validation;
 }
 
@@ -413,31 +493,36 @@ async function validateWithVLM(
 
       const imagePath = page.image;
 
-      if (!page.content) {
-        console.warn(`[EXULU] Page ${page.page}: No content found, skipping validation`);
-        return;
-      }
-
       if (!imagePath) {
         console.warn(`[EXULU] Page ${page.page}: No image found, skipping validation`);
         return;
       }
 
-      // Check if page.content has a .jpeg, .jpg, .png, .gif, .webp image
-      const hasImage = page.content.match(/\.(jpeg|jpg|png|gif|webp)/i);
-      // Check if the content has multiple occurences of |
-      const hasTable = (page.content.match(/\|/g)?.length || 0) > 1;
+      // For standalone images, page.content will be empty initially
+      // For PDFs/documents, we check if VLM validation is needed based on content
+      if (page.content) {
+        // Check if page.content has a .jpeg, .jpg, .png, .gif, .webp image
+        const hasImage = page.content.match(/\.(jpeg|jpg|png|gif|webp)/i);
+        // Check if the content has multiple occurences of |
+        const hasTable = (page.content.match(/\|/g)?.length || 0) > 1;
 
-      if (!hasImage && !hasTable) {
+        if (!hasImage && !hasTable) {
+          if (verbose) {
+            console.log(`[EXULU] Page ${page.page}: No image or table found, SKIPPING VLM validation`);
+          }
+          return;
+        }
+      } else {
+        // Empty content means this is likely a standalone image that needs content extraction
         if (verbose) {
-          console.log(`[EXULU] Page ${page.page}: No image or table found, SKIPPING VLM validation`);
+          console.log(`[EXULU] Page ${page.page}: Standalone image, proceeding with VLM content extraction`);
         }
-        return;
       }
 
       // Validate the page
       let validation: VLMValidationResult;
       try {
+        console.log(`[EXULU] Validating page ${page.page} with VLM`);
         validation = await withRetry(async () => {
           return await validatePageWithVLM(page, imagePath, model);
         }, 3);
@@ -573,6 +658,13 @@ async function processDocument(
     case 'doc':
       result = await processWord(buffer);
       break;
+    case 'jpg':
+    case 'jpeg':
+    case 'png':
+    case 'gif':
+    case 'webp':
+      result = await processImage(buffer, paths, config, verbose);
+      break;
 
     // Todo other file types with docx and officeparser
     default:
@@ -922,7 +1014,7 @@ export async function documentProcessor({
     let supportedTypes: string[] = [];
     switch (config?.processor.name) {
       case "docling":
-        supportedTypes = ['pdf', 'docx', 'doc', 'txt', 'md'];
+        supportedTypes = ['pdf', 'docx', 'doc', 'txt', 'md', 'jpg', 'jpeg', 'png', 'gif', 'webp'];
         break;
       case "officeparser":
         supportedTypes = [];
@@ -931,7 +1023,7 @@ export async function documentProcessor({
         supportedTypes = ['pdf', 'doc', 'docx', 'docm', 'odt', 'rtf', 'ppt', 'pptx', 'pptm', 'odp', 'xls', 'xlsx', 'xlsm', 'ods', 'csv', 'tsv'];
         break;
       case "mistral":
-        supportedTypes = ['pdf', 'docx', 'doc', 'txt', 'md'];
+        supportedTypes = ['pdf', 'docx', 'doc', 'txt', 'md', 'jpg', 'jpeg', 'png', 'gif', 'webp'];
         break;
     }
 

package/ee/workers.ts

@@ -1311,7 +1311,7 @@ const pollJobResult = async ({
       break;
     }
     // Wait for 2 seconds before polling again
-    await new Promise((resolve) => setTimeout((resolve) => resolve(true), 2000));
+    await new Promise((resolve) => setTimeout(() => resolve(true), 2000));
   }
   return result;
 };