@exulu/backend 3.7.3 → 4.0.0

package/ee/invoke-skills/artifact-filter.test.ts

@@ -0,0 +1,49 @@
+import { isIgnoredArtifactPath, capArtifacts, needsDownload } from "./artifact-filter";
+
+describe("isIgnoredArtifactPath — dependency and cache trees are not session artifacts", () => {
+  it("ignores virtualenvs, node_modules, caches and VCS metadata at any depth", () => {
+    for (const p of [
+      "venv/pyvenv.cfg",
+      "venv/lib/python3.11/site-packages/docx/api.py",
+      ".venv/bin/python",
+      "node_modules/lodash/index.js",
+      "scripts/__pycache__/create.cpython-311.pyc",
+      ".cache/pip/http/x",
+      ".git/objects/ab/cd",
+      "project/env/lib/python3.11/site-packages/x.py",
+    ]) {
+      expect(isIgnoredArtifactPath(p)).toBe(true);
+    }
+  });
+
+  it("keeps the files the agent actually produces", () => {
+    for (const p of ["Besprechungsprotokoll.docx", "create_minutes.py", "out/report.pdf", "Angebot/Angebot_107426.docx"]) {
+      expect(isIgnoredArtifactPath(p)).toBe(false);
+    }
+  });
+});
+
+describe("capArtifacts — a bash command that touches thousands of files must not flood the tool result", () => {
+  const many = Array.from({ length: 250 }, (_, i) => ({ relativePath: `out/file_${i}.txt`, url: `https://s3/${i}` }));
+
+  it("returns everything below the cap untouched", () => {
+    const { kept, omitted } = capArtifacts(many.slice(0, 10), 50);
+    expect(kept).toHaveLength(10);
+    expect(omitted).toBe(0);
+  });
+
+  it("keeps only the first N and reports how many were omitted", () => {
+    const { kept, omitted } = capArtifacts(many, 50);
+    expect(kept).toHaveLength(50);
+    expect(kept[0]!.relativePath).toBe("out/file_0.txt");
+    expect(omitted).toBe(200);
+  });
+});
+
+describe("needsDownload — resyncing S3 session files into a live sandbox", () => {
+  it("downloads when the local file is missing or differs in size, skips identical files", () => {
+    expect(needsDownload(undefined, 100)).toBe(true);
+    expect(needsDownload(99, 100)).toBe(true);
+    expect(needsDownload(100, 100)).toBe(false);
+  });
+});

package/ee/invoke-skills/artifact-filter.ts

@@ -0,0 +1,38 @@
+/**
+ * The sandbox mirrors every file a shell command creates under the session directory to
+ * S3 and lists each one (with a presigned URL) in the tool result. A single
+ * `python -m venv venv && pip install python-docx` creates thousands of files, which
+ * turned one tool result into ~1.8M characters and killed the session's context window.
+ */
+
+const IGNORED_SEGMENTS = new Set([
+  "venv",
+  ".venv",
+  "env",
+  "node_modules",
+  "__pycache__",
+  "site-packages",
+  "dist-packages",
+  ".cache",
+  ".git",
+  ".pytest_cache",
+  ".mypy_cache",
+  ".ipynb_checkpoints",
+]);
+
+/** True when any path segment is a dependency, cache or VCS directory. */
+export function isIgnoredArtifactPath(relativePath: string): boolean {
+  return relativePath.split(/[\\/]+/).some((segment) => IGNORED_SEGMENTS.has(segment));
+}
+
+export const DEFAULT_ARTIFACT_CAP = 50;
+
+export function capArtifacts<T>(artifacts: T[], max: number = DEFAULT_ARTIFACT_CAP): { kept: T[]; omitted: number } {
+  if (artifacts.length <= max) return { kept: artifacts, omitted: 0 };
+  return { kept: artifacts.slice(0, max), omitted: artifacts.length - max };
+}
+
+/** Whether an S3 session file must be (re)downloaded into the live sandbox directory. */
+export function needsDownload(localSize: number | undefined, remoteSize: number): boolean {
+  return localSize === undefined || localSize !== remoteSize;
+}

package/ee/invoke-skills/create-sandbox.ts

@@ -8,6 +8,7 @@ import { join, dirname, resolve, relative, posix } from 'node:path'
 import { exec, spawn } from 'node:child_process'
 import { promisify } from 'node:util'
 import { listS3ObjectsByPrefix, getS3ObjectBytes, uploadFile, getPresignedUrl, type S3FileObject } from '@SRC/uppy/index.ts'
+import { isIgnoredArtifactPath, capArtifacts, needsDownload } from './artifact-filter'
 import { getNpmGlobalRoot } from '@SRC/exulu/system-dependencies.ts'
 import type { ExuluConfig } from '@SRC/exulu/app/index.ts'
 import { createBashTool, type Sandbox } from "bash-tool";
@@ -270,6 +271,7 @@ async function restoreArtifactsFromS3(
     sessionId: string,
     userId: number | string,
     config: ExuluConfig,
+    opts: { onlyMissing?: boolean } = {},
 ): Promise<void> {
     const userPrefix = `user_${userId}/sessions/${sessionId}/`
     let objects: S3FileObject[]
@@ -297,8 +299,18 @@ async function restoreArtifactsFromS3(
         const idx = obj.key.indexOf(userPrefix)
         const relativePath = idx >= 0 ? obj.key.slice(idx + userPrefix.length) : ''
         if (!relativePath) continue // directory marker or unexpected key shape
+        if (isIgnoredArtifactPath(relativePath)) continue // never pull dependency trees back
 
         const localPath = join(sessionDir, relativePath)
+        if (opts.onlyMissing) {
+            let localSize: number | undefined
+            try {
+                localSize = (await stat(localPath)).size
+            } catch {
+                localSize = undefined
+            }
+            if (!needsDownload(localSize, obj.size)) continue
+        }
         try {
             // Use binary-safe fetch — session artifacts now include PDFs, .docx
             // and other binary formats (from user uploads as well as agent
@@ -380,6 +392,20 @@ export async function downloadKeyIntoSandbox(opts: {
  * cache AND no session directory on disk), previously persisted artifacts for
  * the session are restored from S3 into the fresh session directory.
  */
+/**
+ * Loaded lazily: python-setup resolves the package root via import.meta.url, which
+ * must not be evaluated when this module is merely imported (jest runs CJS).
+ */
+async function resolvePythonVenvPath(): Promise<string | undefined> {
+    try {
+        const { getPythonVenvPath } = await import('@SRC/utils/python-setup.ts')
+        return getPythonVenvPath()
+    } catch (err) {
+        console.warn('[SKILLS] Could not resolve the Python venv for the session sandbox; skill scripts fall back to the system python.', err)
+        return undefined
+    }
+}
+
 export async function createSessionSandbox(
     sessionId: string,
     skills: SkillRef[],
@@ -404,6 +430,16 @@ export async function createSessionSandbox(
             cached.installedSkills.set(skill.id, skill.current_version)
         }
 
+        // Files uploaded through the side panel (or by an email routine) after the
+        // sandbox was created live only in S3 until re-synced; without this the agent's
+        // `ls` never sees them although the system prompt promises it will.
+        if (userId && config.fileUploads) {
+            try {
+                await restoreArtifactsFromS3(cached.handle.sessionDir, sessionId, userId, config, { onlyMissing: true })
+            } catch (err) {
+                console.error(`[SKILLS] Failed to re-sync S3 session files for session ${sessionId}; continuing.`, err)
+            }
+        }
         return cached.handle
     }
 
@@ -439,8 +475,9 @@ export async function createSessionSandbox(
     // Restore artifacts from S3 only on a true cold start. If the dir already
     // existed, the local files are at least as new as S3 and may contain
     // unsaved-to-S3 writes from a prior process.
-    if (userId && config.fileUploads && !dirExisted) {
-        await restoreArtifactsFromS3(sessionDir, sessionId, userId, config)
+    if (userId && config.fileUploads) {
+        // Cold start restores everything; a surviving on-disk dir only fetches what changed.
+        await restoreArtifactsFromS3(sessionDir, sessionId, userId, config, { onlyMissing: dirExisted })
     }
 
     // Probe whether bwrap can actually create namespaces on this host. On a
@@ -492,6 +529,9 @@ export async function createSessionSandbox(
     // fail with a clear MODULE_NOT_FOUND, matching what the user would see
     // outside the sandbox).
     const npmGlobalRoot = await getNpmGlobalRoot()
+    // The @exulu/backend Python venv carries the skill runtime deps (python-docx …).
+    // Put its bin dir first on PATH so `python3` inside the sandbox resolves to it.
+    const pythonVenvPath = await resolvePythonVenvPath()
 
     // Per-session policy. Passed to every wrapWithSandbox() invocation made
     // from within this closure. customConfig wins over the singleton's
@@ -510,6 +550,7 @@ export async function createSessionSandbox(
                 // the sandbox. Without this, `require('docx')` fails with
                 // EPERM even when NODE_PATH points the resolver here.
                 ...(npmGlobalRoot ? [npmGlobalRoot] : []),
+                ...(pythonVenvPath ? [pythonVenvPath] : []),
             ],
             allowWrite: [sessionDir],
             denyWrite: [],
@@ -563,6 +604,9 @@ export async function createSessionSandbox(
         ...configuredVariables,
         ...process.env,
         ...(npmGlobalRoot ? { NODE_PATH: npmGlobalRoot } : {}),
+        ...(pythonVenvPath
+            ? { PATH: `${join(pythonVenvPath, 'bin')}:${process.env.PATH ?? ''}`, VIRTUAL_ENV: pythonVenvPath }
+            : {}),
     }
 
     // Either wrap a command with bwrap/sandbox-exec or return it unchanged when
@@ -757,6 +801,7 @@ export async function createSessionSandbox(
             for (const entry of entries) {
                 const full = join(dir, entry.name)
                 if (full === skillsDir) continue
+                if (isIgnoredArtifactPath(relative(sessionDir, full))) continue
                 if (entry.isDirectory()) {
                     await walk(full)
                 } else if (entry.isFile()) {
@@ -917,19 +962,26 @@ export async function createSessionSandbox(
             // marker block isn't truncated. Only files with a presigned URL
             // appear here; locally-only entries would just confuse the user.
             let stdout = result?.stdout ?? ''
-            const withUrls = artifacts.filter((a) => a.url)
+            // Cap the listing: one command can touch thousands of files, and every
+            // entry carries a presigned URL. The full set is still mirrored to S3.
+            const { kept, omitted } = capArtifacts(artifacts)
+            const withUrls = kept.filter((a) => a.url)
             if (withUrls.length > 0) {
                 const lines = ['', '[exulu-artifacts]']
                 for (const a of withUrls) {
                     lines.push(`  ${a.relativePath}: ${a.url}`)
                 }
+                if (omitted > 0) {
+                    lines.push(`  … ${omitted} more file(s) were created and mirrored but are not listed here.`)
+                }
                 stdout = `${stdout}\n${lines.join('\n')}`
             }
 
             return {
                 ...result,
                 stdout,
-                artifacts,
+                artifacts: kept,
+                ...(omitted > 0 ? { artifactsOmitted: omitted } : {}),
             }
         },
     })

package/ee/python/documents/processing/README.md

@@ -126,14 +126,13 @@ IMAGE_RESOLUTION_SCALE = 2.0  # Image resolution multiplier
 
 This script requires the following Python packages (installed via `npm run python:setup`):
 
-- `docling` - Document conversion
-- `docling-hierarchical-pdf` - Hierarchical heading processing
+- `pypdf` - PDF page-range splitting for the `mistral` OCR processor
 - `transformers` - ML-based text processing
 - `PIL` - Image handling
 
 ### Troubleshooting
 
-**Issue: ImportError for docling**
+**Issue: ImportError for pypdf**
 ```bash
 npm run python:install
 ```

package/ee/python/documents/processing/doc_processor.ts

@@ -12,7 +12,6 @@ import WordExtractor from 'word-extractor';
 import { parseOfficeAsync } from "officeparser";
 import { checkLicense } from '@EE/entitlements';
 import { executePythonScript } from '@SRC/utils/python-executor';
-import { setupPythonEnvironment, validatePythonEnvironment } from '@SRC/utils/python-setup';
 import { LiteParse } from '@llamaindex/liteparse';
 import { resolveOcr } from '@SRC/exulu/resolve-ocr';
 import type { ResolveOcrInput } from '@SRC/exulu/resolve-ocr';
@@ -31,7 +30,7 @@ type DocumentProcessorConfig = {
     concurrency: number;
   },
   processor: {
-    name: "docling" | "liteparse" | "mistral" | "officeparser"
+    name: "liteparse" | "mistral" | "officeparser"
     /**
      * LiteLLM model_name for the "mistral" OCR processor (declared in
      * config.litellm.yaml). Defaults to "mistral-ocr". OCR is routed through
@@ -534,7 +533,7 @@ async function validateWithVLM(
   verbose: boolean = false,
   concurrency: number = 10
 ): Promise<ProcessedDocument> {
-  console.log(`[EXULU] Starting VLM validation for docling output, ${document.length} pages...`);
+  console.log(`[EXULU] Starting VLM validation for processor output, ${document.length} pages...`);
   console.log(`[EXULU] Concurrency limit: ${concurrency}`);
 
   // Create a concurrency limiter
@@ -683,7 +682,7 @@ async function processDocument(
   /* 
   tempDir/
     uuid/
-      docling.json
+      processed.json
       images/
   */
   const paths: ProcessingPaths = {
@@ -751,56 +750,7 @@ async function processPdf(
   try {
     let json: ProcessedDocument = [];
     // Call the PDF processor script
-    if (config?.processor.name === "docling") {
-
-      // Validate Python environment and setup if needed
-      console.log(`[EXULU] Validating Python environment...`);
-      const validation = await validatePythonEnvironment(undefined, true);
-
-      if (!validation.valid) {
-        console.log(`[EXULU] Python environment not ready, setting up automatically...`);
-        console.log(`[EXULU] Reason: ${validation.message}`);
-
-        const setupResult = await setupPythonEnvironment({
-          verbose: true,
-          force: false, // Only setup if not already done
-        });
-
-        if (!setupResult.success) {
-          throw new Error(`Failed to setup Python environment: ${setupResult.message}\n\n${setupResult.output || ''}`);
-        }
-
-        console.log(`[EXULU] Python environment setup completed successfully`);
-      } else {
-        console.log(`[EXULU] Python environment is valid`);
-      }
-
-      console.log(`[EXULU] Processing document with document_to_markdown.py`);
-
-      const result = await executePythonScript({
-        scriptPath: 'ee/python/documents/processing/document_to_markdown.py',
-        args: [
-          paths.source,
-          '-o', paths.json,
-          '--images-dir', paths.images
-        ],
-        timeout: 30 * 60 * 1000, // 30 minutes for large documents
-      });
-
-      // Log processing info from stderr
-      if (result.stderr) {
-        console.log('Processing info:', result.stderr.trim());
-      }
-
-      if (!result.success) {
-        throw new Error(`Document processing failed: ${result.stderr}`);
-      }
-
-      // Read the generated JSON file
-      const jsonContent = await fs.promises.readFile(paths.json, 'utf-8');
-      json = JSON.parse(jsonContent);
-
-    } else if (config?.processor.name === "officeparser") {
+    if (config?.processor.name === "officeparser") {
       const text = await parseOfficeAsync(buffer, {
         outputErrorToConsole: false,
         newlineDelimiter: "\n",
@@ -934,16 +884,29 @@ async function processPdf(
       }));
 
       fs.writeFileSync(paths.json, JSON.stringify(json, null, 2));
+    } else {
+      // Without this the if/else chain fell through leaving `json` empty, and
+      // the document was stored as zero pages with no error anywhere — a silent
+      // data-loss path. "docling" in particular used to be a valid value here.
+      // Every member of the union is handled above, so TypeScript narrows
+      // `processor.name` to `never` here. At runtime a caller can still pass
+      // anything, because these configs are routinely built from plain JS or
+      // from database rows. String() widens it back for the message.
+      const configured = String(config?.processor?.name ?? '');
+      throw new Error(
+        configured === ''
+          ? '[EXULU] No document processor configured. Set processor.name to one of: mistral, liteparse, officeparser.'
+          : `[EXULU] Unknown document processor "${configured}". Supported processors are: mistral, liteparse, officeparser.` +
+            (configured === 'docling'
+              ? ' The "docling" processor was removed: it depended on PyMuPDF, which is AGPL-licensed. Use "mistral" for PDF OCR.'
+              : '')
+      );
     }
 
     console.log(`[EXULU] \n✓ Document processing completed successfully`);
     console.log(`[EXULU] Total pages: ${json.length}`);
     console.log(`[EXULU] Output file: ${paths.json}`);
 
-    if (config?.vlm?.model) {
-      console.error('[EXULU] VLM validation is only supported when docling is enabled, skipping validation.');
-    }
-
     // Apply VLM validation if enabled
     const vlmModel = config?.vlm?.model ? await resolveVlmModel(config) : undefined;
     if (vlmModel && json.length > 0) {
@@ -1121,9 +1084,6 @@ export async function documentProcessor({
 
     let supportedTypes: string[] = [];
     switch (config?.processor.name) {
-      case "docling":
-        supportedTypes = ['pdf', 'docx', 'doc', 'txt', 'md', 'jpg', 'jpeg', 'png', 'gif', 'webp'];
-        break;
       case "officeparser":
         supportedTypes = ['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'rtf', 'csv', 'md', 'html'];
         break;

package/ee/python/documents/processing/split_pdf.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-PDF Splitter — splits a PDF into fixed-size page chunks using PyMuPDF.
+PDF Splitter — splits a PDF into fixed-size page chunks using pypdf.
 
 Outputs a JSON array to stdout, each element:
   { "path": "<absolute-path>", "start_page": <int>, "end_page": <int> }
@@ -22,26 +22,20 @@ import argparse
 
 # stdout is this script's result channel and the caller does JSON.parse() on it,
 # so nothing else may write there. Our own prints all pass file=sys.stderr, but
-# dependencies do not honour that: PyMuPDF sends its messages to sys.stdout by
-# default, and importing the legacy `fitz` alias emits
-# "warning: The `fitz` API is deprecated ..." — which lands ahead of the payload
-# and fails the caller with "Unexpected token 'w'". PyMuPDF arrives unpinned as
-# a docling transitive dependency, so a routine rebuild is enough to introduce a
-# banner like that. Point sys.stdout at stderr before importing anything and
-# keep a private handle for the result, so any library that prints — now or
-# after a future dependency bump — is shunted to the log channel instead of
-# corrupting the payload.
+# a dependency need not: any library is free to emit a deprecation banner or a
+# progress line on stdout, and it would land ahead of the payload and fail the
+# caller with "Unexpected token 'w'". Point sys.stdout at stderr before
+# importing anything and keep a private handle for the result, so a library
+# that prints — now or after a future dependency bump — is shunted to the log
+# channel instead of corrupting the payload.
 _stdout = sys.stdout
 sys.stdout = sys.stderr
 
-try:
-    import pymupdf as fitz  # PyMuPDF >= 1.24.3, where the module was renamed
-except ImportError:  # older releases only ship the legacy `fitz` module
-    import fitz
+from pypdf import PdfReader, PdfWriter
 
 
 def _write_chunk(
-    doc: fitz.Document,
+    reader: PdfReader,
     output_dir: str,
     chunk_start: int,
     chunk_end: int,
@@ -56,10 +50,11 @@ def _write_chunk(
     """
     chunk_path = os.path.join(output_dir, f"chunk_{chunk_start}_{chunk_end - 1}.pdf")
 
-    sub = fitz.open()
-    sub.insert_pdf(doc, from_page=chunk_start, to_page=chunk_end - 1)
-    sub.save(chunk_path)
-    sub.close()
+    writer = PdfWriter()
+    writer.append(reader, pages=(chunk_start, chunk_end))
+    with open(chunk_path, "wb") as fh:
+        writer.write(fh)
+    writer.close()
 
     chunk_bytes = os.path.getsize(chunk_path)
     n_pages = chunk_end - chunk_start
@@ -68,8 +63,8 @@ def _write_chunk(
         os.remove(chunk_path)
         mid = chunk_start + n_pages // 2
         return (
-            _write_chunk(doc, output_dir, chunk_start, mid, max_size_bytes)
-            + _write_chunk(doc, output_dir, mid, chunk_end, max_size_bytes)
+            _write_chunk(reader, output_dir, chunk_start, mid, max_size_bytes)
+            + _write_chunk(reader, output_dir, mid, chunk_end, max_size_bytes)
         )
 
     if max_size_bytes and chunk_bytes > max_size_bytes:
@@ -88,21 +83,21 @@ def split_pdf(
     chunk_size: int,
     max_size_bytes: int | None = None,
 ) -> list[dict]:
-    doc = fitz.open(input_path)
+    reader = PdfReader(input_path)
 
     # Some PDFs are saved with an empty owner/user password by certain writers
     # (e.g. older Adobe Acrobat exports). The OS opens them transparently by
     # trying "" first, but most libraries raise immediately. We replicate that
-    # OS-level behaviour here.
-    if doc.needs_pass:
-        authenticated = doc.authenticate("")
-        if not authenticated:
+    # OS-level behaviour here. pypdf's decrypt() returns a PasswordType enum
+    # whose NOT_DECRYPTED member is falsy, so a plain truth test is enough.
+    if reader.is_encrypted:
+        if not reader.decrypt(""):
             raise ValueError(
                 "PDF requires a non-empty password and cannot be opened automatically."
             )
         print("[split_pdf] Authenticated with empty password (phantom-password PDF)", file=sys.stderr)
 
-    total_pages = len(doc)
+    total_pages = len(reader.pages)
     file_size = os.path.getsize(input_path)
     print(
         f"[split_pdf] Total pages: {total_pages}, chunk size: {chunk_size}, "
@@ -115,7 +110,7 @@ def split_pdf(
 
     if not needs_split:
         print("[split_pdf] No split needed — returning original path", file=sys.stderr)
-        doc.close()
+        reader.close()
         return [{
             "path": os.path.abspath(input_path),
             "start_page": 0,
@@ -127,7 +122,7 @@ def split_pdf(
     chunks = []
     for start_page in range(0, total_pages, chunk_size):
         end_page = min(start_page + chunk_size, total_pages)
-        sub_chunks = _write_chunk(doc, output_dir, start_page, end_page, max_size_bytes)
+        sub_chunks = _write_chunk(reader, output_dir, start_page, end_page, max_size_bytes)
         for c in sub_chunks:
             print(
                 f"[split_pdf] Chunk {len(chunks) + 1}: pages {c['start_page']}–{c['end_page'] - 1} "
@@ -136,7 +131,7 @@ def split_pdf(
             )
         chunks.extend(sub_chunks)
 
-    doc.close()
+    reader.close()
     return chunks