@exulu/backend 3.7.4 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/{chunk-27K2CO47.js → chunk-QMN6MVHQ.js} +1 -1
  2. package/dist/{chunk-AWMU6QXB.js → chunk-RBEWHG7I.js} +297 -39
  3. package/dist/cli/start-whisper.js +1 -1
  4. package/dist/{convert-exulu-tools-to-ai-sdk-tools-XNQ6Q3X6.js → convert-exulu-tools-to-ai-sdk-tools-6RU4IZMI.js} +1 -1
  5. package/dist/index.cjs +566 -217
  6. package/dist/index.d.cts +3 -1
  7. package/dist/index.d.ts +3 -1
  8. package/dist/index.js +250 -180
  9. package/dist/{python-setup-JZGHWQCG.js → python-setup-DRJ3QX5F.js} +1 -1
  10. package/ee/LICENSE.md +2 -2
  11. package/ee/agentic-retrieval/pipeline/config.test.ts +18 -1
  12. package/ee/agentic-retrieval/pipeline/config.ts +15 -0
  13. package/ee/agentic-retrieval/pipeline/index.test.ts +73 -0
  14. package/ee/agentic-retrieval/pipeline/index.ts +67 -13
  15. package/ee/agentic-retrieval/pipeline/memory.test.ts +59 -0
  16. package/ee/agentic-retrieval/pipeline/memory.ts +181 -11
  17. package/ee/agentic-retrieval/pipeline/pin-rerun.test.ts +17 -0
  18. package/ee/agentic-retrieval/pipeline/pin-rerun.ts +29 -0
  19. package/ee/agentic-retrieval/pipeline/routing.test.ts +34 -0
  20. package/ee/agentic-retrieval/pipeline/routing.ts +96 -5
  21. package/ee/agentic-retrieval/pipeline/search.ts +9 -6
  22. package/ee/agentic-retrieval/pipeline/timing.test.ts +24 -0
  23. package/ee/agentic-retrieval/pipeline/timing.ts +26 -0
  24. package/ee/agentic-retrieval/pipeline/types.ts +2 -0
  25. package/ee/python/documents/processing/README.md +2 -3
  26. package/ee/python/documents/processing/doc_processor.ts +21 -61
  27. package/ee/python/documents/processing/split_pdf.py +25 -30
  28. package/ee/python/documents/processing/tests/__init__.py +0 -0
  29. package/ee/python/documents/processing/tests/test_split_pdf.py +230 -0
  30. package/ee/python/requirements.txt +12 -2
  31. package/ee/python/setup.sh +40 -1
  32. package/ee/python/transcription/pipeline.py +109 -15
  33. package/ee/python/transcription/tests/test_align_model_licensing.py +184 -0
  34. package/ee/workers.ts +2 -7
  35. package/license.md +2 -2
  36. package/package.json +3 -4
  37. package/scripts/postinstall.cjs +52 -1
  38. package/ee/python/documents/processing/document_to_markdown.py +0 -413
@@ -126,14 +126,13 @@ IMAGE_RESOLUTION_SCALE = 2.0 # Image resolution multiplier
126
126
 
127
127
  This script requires the following Python packages (installed via `npm run python:setup`):
128
128
 
129
- - `docling` - Document conversion
130
- - `docling-hierarchical-pdf` - Hierarchical heading processing
129
+ - `pypdf` - PDF page-range splitting for the `mistral` OCR processor
131
130
  - `transformers` - ML-based text processing
132
131
  - `PIL` - Image handling
133
132
 
134
133
  ### Troubleshooting
135
134
 
136
- **Issue: ImportError for docling**
135
+ **Issue: ImportError for pypdf**
137
136
  ```bash
138
137
  npm run python:install
139
138
  ```
@@ -12,7 +12,6 @@ import WordExtractor from 'word-extractor';
12
12
  import { parseOfficeAsync } from "officeparser";
13
13
  import { checkLicense } from '@EE/entitlements';
14
14
  import { executePythonScript } from '@SRC/utils/python-executor';
15
- import { setupPythonEnvironment, validatePythonEnvironment } from '@SRC/utils/python-setup';
16
15
  import { LiteParse } from '@llamaindex/liteparse';
17
16
  import { resolveOcr } from '@SRC/exulu/resolve-ocr';
18
17
  import type { ResolveOcrInput } from '@SRC/exulu/resolve-ocr';
@@ -31,7 +30,7 @@ type DocumentProcessorConfig = {
31
30
  concurrency: number;
32
31
  },
33
32
  processor: {
34
- name: "docling" | "liteparse" | "mistral" | "officeparser"
33
+ name: "liteparse" | "mistral" | "officeparser"
35
34
  /**
36
35
  * LiteLLM model_name for the "mistral" OCR processor (declared in
37
36
  * config.litellm.yaml). Defaults to "mistral-ocr". OCR is routed through
@@ -534,7 +533,7 @@ async function validateWithVLM(
534
533
  verbose: boolean = false,
535
534
  concurrency: number = 10
536
535
  ): Promise<ProcessedDocument> {
537
- console.log(`[EXULU] Starting VLM validation for docling output, ${document.length} pages...`);
536
+ console.log(`[EXULU] Starting VLM validation for processor output, ${document.length} pages...`);
538
537
  console.log(`[EXULU] Concurrency limit: ${concurrency}`);
539
538
 
540
539
  // Create a concurrency limiter
@@ -683,7 +682,7 @@ async function processDocument(
683
682
  /*
684
683
  tempDir/
685
684
  uuid/
686
- docling.json
685
+ processed.json
687
686
  images/
688
687
  */
689
688
  const paths: ProcessingPaths = {
@@ -751,56 +750,7 @@ async function processPdf(
751
750
  try {
752
751
  let json: ProcessedDocument = [];
753
752
  // Call the PDF processor script
754
- if (config?.processor.name === "docling") {
755
-
756
- // Validate Python environment and setup if needed
757
- console.log(`[EXULU] Validating Python environment...`);
758
- const validation = await validatePythonEnvironment(undefined, true);
759
-
760
- if (!validation.valid) {
761
- console.log(`[EXULU] Python environment not ready, setting up automatically...`);
762
- console.log(`[EXULU] Reason: ${validation.message}`);
763
-
764
- const setupResult = await setupPythonEnvironment({
765
- verbose: true,
766
- force: false, // Only setup if not already done
767
- });
768
-
769
- if (!setupResult.success) {
770
- throw new Error(`Failed to setup Python environment: ${setupResult.message}\n\n${setupResult.output || ''}`);
771
- }
772
-
773
- console.log(`[EXULU] Python environment setup completed successfully`);
774
- } else {
775
- console.log(`[EXULU] Python environment is valid`);
776
- }
777
-
778
- console.log(`[EXULU] Processing document with document_to_markdown.py`);
779
-
780
- const result = await executePythonScript({
781
- scriptPath: 'ee/python/documents/processing/document_to_markdown.py',
782
- args: [
783
- paths.source,
784
- '-o', paths.json,
785
- '--images-dir', paths.images
786
- ],
787
- timeout: 30 * 60 * 1000, // 30 minutes for large documents
788
- });
789
-
790
- // Log processing info from stderr
791
- if (result.stderr) {
792
- console.log('Processing info:', result.stderr.trim());
793
- }
794
-
795
- if (!result.success) {
796
- throw new Error(`Document processing failed: ${result.stderr}`);
797
- }
798
-
799
- // Read the generated JSON file
800
- const jsonContent = await fs.promises.readFile(paths.json, 'utf-8');
801
- json = JSON.parse(jsonContent);
802
-
803
- } else if (config?.processor.name === "officeparser") {
753
+ if (config?.processor.name === "officeparser") {
804
754
  const text = await parseOfficeAsync(buffer, {
805
755
  outputErrorToConsole: false,
806
756
  newlineDelimiter: "\n",
@@ -934,16 +884,29 @@ async function processPdf(
934
884
  }));
935
885
 
936
886
  fs.writeFileSync(paths.json, JSON.stringify(json, null, 2));
887
+ } else {
888
+ // Without this the if/else chain fell through leaving `json` empty, and
889
+ // the document was stored as zero pages with no error anywhere — a silent
890
+ // data-loss path. "docling" in particular used to be a valid value here.
891
+ // Every member of the union is handled above, so TypeScript narrows
892
+ // `processor.name` to `never` here. At runtime a caller can still pass
893
+ // anything, because these configs are routinely built from plain JS or
894
+ // from database rows. String() widens it back for the message.
895
+ const configured = String(config?.processor?.name ?? '');
896
+ throw new Error(
897
+ configured === ''
898
+ ? '[EXULU] No document processor configured. Set processor.name to one of: mistral, liteparse, officeparser.'
899
+ : `[EXULU] Unknown document processor "${configured}". Supported processors are: mistral, liteparse, officeparser.` +
900
+ (configured === 'docling'
901
+ ? ' The "docling" processor was removed: it depended on PyMuPDF, which is AGPL-licensed. Use "mistral" for PDF OCR.'
902
+ : '')
903
+ );
937
904
  }
938
905
 
939
906
  console.log(`[EXULU] \n✓ Document processing completed successfully`);
940
907
  console.log(`[EXULU] Total pages: ${json.length}`);
941
908
  console.log(`[EXULU] Output file: ${paths.json}`);
942
909
 
943
- if (config?.vlm?.model) {
944
- console.error('[EXULU] VLM validation is only supported when docling is enabled, skipping validation.');
945
- }
946
-
947
910
  // Apply VLM validation if enabled
948
911
  const vlmModel = config?.vlm?.model ? await resolveVlmModel(config) : undefined;
949
912
  if (vlmModel && json.length > 0) {
@@ -1121,9 +1084,6 @@ export async function documentProcessor({
1121
1084
 
1122
1085
  let supportedTypes: string[] = [];
1123
1086
  switch (config?.processor.name) {
1124
- case "docling":
1125
- supportedTypes = ['pdf', 'docx', 'doc', 'txt', 'md', 'jpg', 'jpeg', 'png', 'gif', 'webp'];
1126
- break;
1127
1087
  case "officeparser":
1128
1088
  supportedTypes = ['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'rtf', 'csv', 'md', 'html'];
1129
1089
  break;
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env python3
2
2
  """
3
- PDF Splitter — splits a PDF into fixed-size page chunks using PyMuPDF.
3
+ PDF Splitter — splits a PDF into fixed-size page chunks using pypdf.
4
4
 
5
5
  Outputs a JSON array to stdout, each element:
6
6
  { "path": "<absolute-path>", "start_page": <int>, "end_page": <int> }
@@ -22,26 +22,20 @@ import argparse
22
22
 
23
23
  # stdout is this script's result channel and the caller does JSON.parse() on it,
24
24
  # so nothing else may write there. Our own prints all pass file=sys.stderr, but
25
- # dependencies do not honour that: PyMuPDF sends its messages to sys.stdout by
26
- # default, and importing the legacy `fitz` alias emits
27
- # "warning: The `fitz` API is deprecated ..." which lands ahead of the payload
28
- # and fails the caller with "Unexpected token 'w'". PyMuPDF arrives unpinned as
29
- # a docling transitive dependency, so a routine rebuild is enough to introduce a
30
- # banner like that. Point sys.stdout at stderr before importing anything and
31
- # keep a private handle for the result, so any library that prints — now or
32
- # after a future dependency bump — is shunted to the log channel instead of
33
- # corrupting the payload.
25
+ # a dependency need not: any library is free to emit a deprecation banner or a
26
+ # progress line on stdout, and it would land ahead of the payload and fail the
27
+ # caller with "Unexpected token 'w'". Point sys.stdout at stderr before
28
+ # importing anything and keep a private handle for the result, so a library
29
+ # that prints now or after a future dependency bump — is shunted to the log
30
+ # channel instead of corrupting the payload.
34
31
  _stdout = sys.stdout
35
32
  sys.stdout = sys.stderr
36
33
 
37
- try:
38
- import pymupdf as fitz # PyMuPDF >= 1.24.3, where the module was renamed
39
- except ImportError: # older releases only ship the legacy `fitz` module
40
- import fitz
34
+ from pypdf import PdfReader, PdfWriter
41
35
 
42
36
 
43
37
  def _write_chunk(
44
- doc: fitz.Document,
38
+ reader: PdfReader,
45
39
  output_dir: str,
46
40
  chunk_start: int,
47
41
  chunk_end: int,
@@ -56,10 +50,11 @@ def _write_chunk(
56
50
  """
57
51
  chunk_path = os.path.join(output_dir, f"chunk_{chunk_start}_{chunk_end - 1}.pdf")
58
52
 
59
- sub = fitz.open()
60
- sub.insert_pdf(doc, from_page=chunk_start, to_page=chunk_end - 1)
61
- sub.save(chunk_path)
62
- sub.close()
53
+ writer = PdfWriter()
54
+ writer.append(reader, pages=(chunk_start, chunk_end))
55
+ with open(chunk_path, "wb") as fh:
56
+ writer.write(fh)
57
+ writer.close()
63
58
 
64
59
  chunk_bytes = os.path.getsize(chunk_path)
65
60
  n_pages = chunk_end - chunk_start
@@ -68,8 +63,8 @@ def _write_chunk(
68
63
  os.remove(chunk_path)
69
64
  mid = chunk_start + n_pages // 2
70
65
  return (
71
- _write_chunk(doc, output_dir, chunk_start, mid, max_size_bytes)
72
- + _write_chunk(doc, output_dir, mid, chunk_end, max_size_bytes)
66
+ _write_chunk(reader, output_dir, chunk_start, mid, max_size_bytes)
67
+ + _write_chunk(reader, output_dir, mid, chunk_end, max_size_bytes)
73
68
  )
74
69
 
75
70
  if max_size_bytes and chunk_bytes > max_size_bytes:
@@ -88,21 +83,21 @@ def split_pdf(
88
83
  chunk_size: int,
89
84
  max_size_bytes: int | None = None,
90
85
  ) -> list[dict]:
91
- doc = fitz.open(input_path)
86
+ reader = PdfReader(input_path)
92
87
 
93
88
  # Some PDFs are saved with an empty owner/user password by certain writers
94
89
  # (e.g. older Adobe Acrobat exports). The OS opens them transparently by
95
90
  # trying "" first, but most libraries raise immediately. We replicate that
96
- # OS-level behaviour here.
97
- if doc.needs_pass:
98
- authenticated = doc.authenticate("")
99
- if not authenticated:
91
+ # OS-level behaviour here. pypdf's decrypt() returns a PasswordType enum
92
+ # whose NOT_DECRYPTED member is falsy, so a plain truth test is enough.
93
+ if reader.is_encrypted:
94
+ if not reader.decrypt(""):
100
95
  raise ValueError(
101
96
  "PDF requires a non-empty password and cannot be opened automatically."
102
97
  )
103
98
  print("[split_pdf] Authenticated with empty password (phantom-password PDF)", file=sys.stderr)
104
99
 
105
- total_pages = len(doc)
100
+ total_pages = len(reader.pages)
106
101
  file_size = os.path.getsize(input_path)
107
102
  print(
108
103
  f"[split_pdf] Total pages: {total_pages}, chunk size: {chunk_size}, "
@@ -115,7 +110,7 @@ def split_pdf(
115
110
 
116
111
  if not needs_split:
117
112
  print("[split_pdf] No split needed — returning original path", file=sys.stderr)
118
- doc.close()
113
+ reader.close()
119
114
  return [{
120
115
  "path": os.path.abspath(input_path),
121
116
  "start_page": 0,
@@ -127,7 +122,7 @@ def split_pdf(
127
122
  chunks = []
128
123
  for start_page in range(0, total_pages, chunk_size):
129
124
  end_page = min(start_page + chunk_size, total_pages)
130
- sub_chunks = _write_chunk(doc, output_dir, start_page, end_page, max_size_bytes)
125
+ sub_chunks = _write_chunk(reader, output_dir, start_page, end_page, max_size_bytes)
131
126
  for c in sub_chunks:
132
127
  print(
133
128
  f"[split_pdf] Chunk {len(chunks) + 1}: pages {c['start_page']}–{c['end_page'] - 1} "
@@ -136,7 +131,7 @@ def split_pdf(
136
131
  )
137
132
  chunks.extend(sub_chunks)
138
133
 
139
- doc.close()
134
+ reader.close()
140
135
  return chunks
141
136
 
142
137
 
@@ -0,0 +1,230 @@
1
+ """Tests for split_pdf.py.
2
+
3
+ Builds real PDFs with pypdf rather than using fixtures, so the suite is
4
+ self-contained and the page counts/byte sizes are known exactly.
5
+
6
+ Run from the repo root with the venv active:
7
+ cd ee/python/documents/processing && ../../.venv/bin/python -m pytest tests
8
+ """
9
+
10
+ import json
11
+ import os
12
+ import subprocess
13
+ import sys
14
+ from pathlib import Path
15
+
16
+ import pytest
17
+ from pypdf import PdfReader, PdfWriter
18
+ from pypdf.generic import DecodedStreamObject, NameObject
19
+
20
+ # Make ee/python/documents/processing importable.
21
+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
22
+
23
+ import split_pdf as sp # noqa: E402
24
+
25
+ SCRIPT = Path(__file__).resolve().parent.parent / "split_pdf.py"
26
+
27
+
28
+ def _make_pdf(path: Path, pages: int, page_bytes: int = 0) -> Path:
29
+ """Write a PDF whose pages carry a real content stream.
30
+
31
+ Each page's stream holds a `% exulu-page-N` marker so tests can assert the
32
+ page order survived, plus `page_bytes` of incompressible padding so the
33
+ byte-size bisection tests get a file whose size scales with page count.
34
+ Page content is used rather than custom dictionary keys because a PDF
35
+ library is free to drop unknown keys when it rebuilds pages, and PyMuPDF
36
+ does exactly that.
37
+ """
38
+ writer = PdfWriter()
39
+ for i in range(pages):
40
+ writer.add_blank_page(width=200, height=200)
41
+ body = f"% exulu-page-{i}\n".encode()
42
+ if page_bytes:
43
+ body += b"% " + os.urandom(page_bytes).hex().encode() + b"\n"
44
+ stream = DecodedStreamObject()
45
+ stream.set_data(body)
46
+ writer.pages[i][NameObject("/Contents")] = writer._add_object(stream)
47
+ with open(path, "wb") as fh:
48
+ writer.write(fh)
49
+ return path
50
+
51
+
52
+ def _page_markers(pdf_path: str) -> list[str]:
53
+ """Read back the `% exulu-page-N` marker from every page of a chunk."""
54
+ markers = []
55
+ for page in PdfReader(pdf_path).pages:
56
+ data = page["/Contents"].get_data().decode("latin-1")
57
+ markers.append(data.splitlines()[0].removeprefix("% "))
58
+ return markers
59
+
60
+
61
+ def _run_script(*args: str) -> tuple[int, str, str]:
62
+ proc = subprocess.run(
63
+ [sys.executable, str(SCRIPT), *args],
64
+ capture_output=True,
65
+ text=True,
66
+ )
67
+ return proc.returncode, proc.stdout, proc.stderr
68
+
69
+
70
+ # --- no-split path -----------------------------------------------------------
71
+
72
+
73
+ def test_returns_original_when_within_limits(tmp_path):
74
+ src = _make_pdf(tmp_path / "small.pdf", pages=3)
75
+ chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=25)
76
+
77
+ assert len(chunks) == 1
78
+ assert chunks[0]["path"] == str(src.resolve())
79
+ assert chunks[0]["start_page"] == 0
80
+ assert chunks[0]["end_page"] == 3
81
+ # No copy is made, so the output dir is never created.
82
+ assert not (tmp_path / "out").exists()
83
+
84
+
85
+ # --- page-count splitting ----------------------------------------------------
86
+
87
+
88
+ def test_splits_on_page_count(tmp_path):
89
+ src = _make_pdf(tmp_path / "big.pdf", pages=10)
90
+ out = tmp_path / "out"
91
+ chunks = sp.split_pdf(str(src), str(out), chunk_size=4)
92
+
93
+ assert [(c["start_page"], c["end_page"]) for c in chunks] == [(0, 4), (4, 8), (8, 10)]
94
+ for c in chunks:
95
+ assert Path(c["path"]).exists()
96
+ assert [len(PdfReader(c["path"]).pages) for c in chunks] == [4, 4, 2]
97
+
98
+
99
+ def test_chunk_ranges_are_contiguous_and_cover_every_page(tmp_path):
100
+ src = _make_pdf(tmp_path / "doc.pdf", pages=17)
101
+ chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=5)
102
+
103
+ assert chunks[0]["start_page"] == 0
104
+ assert chunks[-1]["end_page"] == 17
105
+ for prev, nxt in zip(chunks, chunks[1:]):
106
+ assert prev["end_page"] == nxt["start_page"]
107
+ assert sum(len(PdfReader(c["path"]).pages) for c in chunks) == 17
108
+
109
+
110
+ def test_exact_multiple_of_chunk_size_produces_no_empty_trailing_chunk(tmp_path):
111
+ src = _make_pdf(tmp_path / "doc.pdf", pages=8)
112
+ chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=4)
113
+
114
+ assert [(c["start_page"], c["end_page"]) for c in chunks] == [(0, 4), (4, 8)]
115
+
116
+
117
+ def test_page_content_is_preserved_in_order(tmp_path):
118
+ """The Nth page of the source must land at the right offset in its chunk."""
119
+ src = _make_pdf(tmp_path / "marked.pdf", pages=6)
120
+
121
+ chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=2)
122
+
123
+ seen = []
124
+ for c in chunks:
125
+ seen.extend(_page_markers(c["path"]))
126
+ assert seen == [f"exulu-page-{i}" for i in range(6)]
127
+
128
+
129
+ # --- byte-size bisection -----------------------------------------------------
130
+
131
+
132
+ def test_oversized_chunk_is_bisected_by_page_count(tmp_path):
133
+ # 4 pages, ~40 KB each. A 4-page chunk exceeds 100 KB and must bisect.
134
+ src = _make_pdf(tmp_path / "heavy.pdf", pages=4, page_bytes=20_000)
135
+ chunks = sp.split_pdf(
136
+ str(src), str(tmp_path / "out"), chunk_size=4, max_size_bytes=100_000
137
+ )
138
+
139
+ assert len(chunks) > 1
140
+ assert [(c["start_page"], c["end_page"]) for c in chunks][0][0] == 0
141
+ assert chunks[-1]["end_page"] == 4
142
+ for prev, nxt in zip(chunks, chunks[1:]):
143
+ assert prev["end_page"] == nxt["start_page"]
144
+
145
+
146
+ def test_single_page_over_limit_is_kept_with_a_warning(tmp_path, capsys):
147
+ src = _make_pdf(tmp_path / "huge-page.pdf", pages=1, page_bytes=60_000)
148
+ # Force the split path: max_size_bytes below the file size.
149
+ chunks = sp.split_pdf(
150
+ str(src), str(tmp_path / "out"), chunk_size=1, max_size_bytes=1_000
151
+ )
152
+
153
+ assert len(chunks) == 1
154
+ assert Path(chunks[0]["path"]).exists()
155
+ assert "cannot be split further" in capsys.readouterr().err
156
+
157
+
158
+ def test_file_over_size_limit_splits_even_when_page_count_fits(tmp_path):
159
+ src = _make_pdf(tmp_path / "wide.pdf", pages=4, page_bytes=20_000)
160
+ chunks = sp.split_pdf(
161
+ str(src), str(tmp_path / "out"), chunk_size=25, max_size_bytes=50_000
162
+ )
163
+
164
+ # chunk_size alone would have returned the original untouched.
165
+ assert len(chunks) > 1
166
+
167
+
168
+ # --- encryption --------------------------------------------------------------
169
+
170
+
171
+ def test_opens_phantom_password_pdf(tmp_path):
172
+ """A PDF encrypted with an empty password must open, as the OS does."""
173
+ writer = PdfWriter()
174
+ for _ in range(6):
175
+ writer.add_blank_page(width=200, height=200)
176
+ writer.encrypt("")
177
+ src = tmp_path / "phantom.pdf"
178
+ with open(src, "wb") as fh:
179
+ writer.write(fh)
180
+
181
+ assert PdfReader(src).is_encrypted
182
+
183
+ chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=2)
184
+ assert len(chunks) == 3
185
+ assert sum(len(PdfReader(c["path"]).pages) for c in chunks) == 6
186
+
187
+
188
+ def test_rejects_pdf_with_a_real_password(tmp_path):
189
+ writer = PdfWriter()
190
+ writer.add_blank_page(width=200, height=200)
191
+ writer.encrypt("hunter2")
192
+ src = tmp_path / "locked.pdf"
193
+ with open(src, "wb") as fh:
194
+ writer.write(fh)
195
+
196
+ with pytest.raises(ValueError, match="non-empty password"):
197
+ sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=1)
198
+
199
+
200
+ # --- CLI contract ------------------------------------------------------------
201
+
202
+
203
+ def test_stdout_is_pure_json(tmp_path):
204
+ src = _make_pdf(tmp_path / "doc.pdf", pages=6)
205
+ code, out, err = _run_script(str(src), str(tmp_path / "out"), "--chunk-size", "2")
206
+
207
+ assert code == 0
208
+ parsed = json.loads(out) # must not raise: nothing but JSON on stdout
209
+ assert len(parsed) == 3
210
+ # Diagnostics go to stderr, not stdout.
211
+ assert "[split_pdf]" in err
212
+ assert "[split_pdf]" not in out
213
+
214
+
215
+ def test_cli_max_size_mb_is_converted_to_bytes(tmp_path):
216
+ src = _make_pdf(tmp_path / "heavy.pdf", pages=4, page_bytes=20_000)
217
+ code, out, _ = _run_script(
218
+ str(src), str(tmp_path / "out"), "--chunk-size", "4", "--max-size-mb", "0.05"
219
+ )
220
+
221
+ assert code == 0
222
+ assert len(json.loads(out)) > 1
223
+
224
+
225
+ def test_exits_nonzero_on_failure(tmp_path):
226
+ code, out, err = _run_script(str(tmp_path / "missing.pdf"), str(tmp_path / "out"))
227
+
228
+ assert code == 1
229
+ assert "[split_pdf] ERROR" in err
230
+ assert out.strip() == ""
@@ -1,12 +1,15 @@
1
- docling
2
1
  # transformers <5: the 5.x line requires huggingface_hub>=1.0, which removed the
3
2
  # `use_auth_token` kwarg that pyannote.audio 3.x still passes to hf_hub_download()
4
3
  # (→ "unexpected keyword argument 'use_auth_token'", diarization silently
5
4
  # disabled). whisperx only needs transformers>=4.48, so the 4.x line is fine.
6
5
  transformers>=4.48,<5
7
6
  pyinstaller
8
- docling-hierarchical-pdf
9
7
  defusedxml
8
+ # PDF page-range splitting for the "mistral" OCR processor (split_pdf.py).
9
+ # BSD-3-Clause, no dependencies. Replaced PyMuPDF, which is AGPL-3.0 or a paid
10
+ # Artifex commercial licence; see split_pdf.py. Do not reintroduce a PDF library
11
+ # without checking its licence.
12
+ pypdf>=6.1
10
13
  # Whisper transcription server. Used by `npx @exulu/backend exulu-start-whisper`.
11
14
  #
12
15
  # Notes on the pins:
@@ -30,6 +33,11 @@ requests
30
33
  # LiteLLM proxy — only used when EXULU_USE_LITELLM=true. Always installed so
31
34
  # the dep is ready when the env var is flipped. Pinned to a tested version;
32
35
  # upgrade deliberately.
36
+ # NOTE: the [proxy] extra hard-requires litellm-enterprise, which is NOT MIT —
37
+ # it carries the proprietary BerriAI Enterprise License. setup.sh step 6.4
38
+ # uninstalls it right after this file is installed; see the comment there. Keep
39
+ # the extra (we need its other 29 packages) and keep that uninstall step
40
+ # together.
33
41
  litellm[proxy]==1.97.0
34
42
  # FastAPI ceiling — NOT cosmetic. litellm 1.97.0 declares fastapi>=0.136.3,<1.0
35
43
  # but imports the private `get_flat_dependant` from fastapi.dependencies.utils
@@ -57,3 +65,5 @@ google-cloud-aiplatform>=1.38
57
65
  # skill scripts (e.g. the built-in docx-manipulation skill) find their libraries without
58
66
  # creating a venv per session (which mirrored thousands of files into the tool result).
59
67
  python-docx>=1.1
68
+ python-pptx>=1.0
69
+ openpyxl>=3.1
@@ -240,6 +240,39 @@ pip install -r "$REQUIREMENTS_FILE"
240
240
 
241
241
  print_success "All dependencies installed successfully"
242
242
 
243
+ # Step 6.4: Remove the proprietary litellm-enterprise package.
244
+ #
245
+ # requirements.txt pins `litellm[proxy]`, and that extra hard-requires
246
+ # litellm-enterprise. We need the other 29 packages the extra brings in
247
+ # (gunicorn, fastapi, apscheduler, boto3, mcp, litellm-proxy-extras, …) but not
248
+ # this one: it is NOT MIT like litellm itself. It carries the BerriAI Enterprise
249
+ # License, which permits production use only with a paid per-seat subscription
250
+ # and forbids redistributing the package. Exulu ships ee/ (including this script
251
+ # and requirements.txt) inside the public npm package, so leaving it installed
252
+ # would put a proprietary dependency on every customer's machine.
253
+ #
254
+ # Removing it is safe. Every one of litellm's 36 `litellm_enterprise` import
255
+ # sites is wrapped in `try/except ImportError`; with the package absent the
256
+ # proxy imports cleanly, `enterprise_proxy_config` falls back to None and the
257
+ # route table is unchanged. Exulu never sets LITELLM_LICENSE, so `premium_user`
258
+ # is False and no enterprise feature was reachable in the first place.
259
+ #
260
+ # If you DO hold a BerriAI Enterprise subscription and want the enterprise
261
+ # callbacks, set EXULU_KEEP_LITELLM_ENTERPRISE=true to skip this step.
262
+ echo ""
263
+ if [ "${EXULU_KEEP_LITELLM_ENTERPRISE:-false}" = "true" ]; then
264
+ print_warning "EXULU_KEEP_LITELLM_ENTERPRISE=true — keeping litellm-enterprise (proprietary; requires a BerriAI subscription for production use)."
265
+ elif pip show litellm-enterprise > /dev/null 2>&1; then
266
+ print_info "Removing litellm-enterprise (proprietary BerriAI package pulled in by litellm[proxy])..."
267
+ if pip uninstall -y litellm-enterprise > /dev/null 2>&1; then
268
+ print_success "litellm-enterprise removed; LiteLLM proxy runs on the MIT-licensed core"
269
+ else
270
+ print_warning "Could not remove litellm-enterprise. It is proprietary (BerriAI Enterprise License) and production use requires a paid subscription — remove it manually with 'pip uninstall -y litellm-enterprise' or set EXULU_KEEP_LITELLM_ENTERPRISE=true if you hold one."
271
+ fi
272
+ else
273
+ print_info "litellm-enterprise not present — nothing to remove"
274
+ fi
275
+
243
276
  # Step 6.5: Generate Prisma client for LiteLLM database mode.
244
277
  # LiteLLM's PrismaClient does `from prisma import Prisma`, which only works
245
278
  # after `prisma generate` has materialized the Python client against
@@ -259,7 +292,7 @@ echo "Step 7: Validating installation..."
259
292
 
260
293
  # Test critical imports
261
294
  print_info "Testing critical imports..."
262
- $PYTHON_CMD -c "import docling" 2>/dev/null && print_success "docling imported successfully" || print_error "Failed to import docling"
295
+ $PYTHON_CMD -c "import pypdf" 2>/dev/null && print_success "pypdf imported successfully" || print_error "Failed to import pypdf"
263
296
  $PYTHON_CMD -c "import transformers" 2>/dev/null && print_success "transformers imported successfully" || print_error "Failed to import transformers"
264
297
 
265
298
  # Whisper transcription server imports — non-fatal: only needed for
@@ -269,6 +302,12 @@ $PYTHON_CMD -c "import whisperx" 2>/dev/null && print_success "whisperx imported
269
302
  $PYTHON_CMD -c "import pyannote.audio" 2>/dev/null && print_success "pyannote.audio imported successfully" || print_warning "pyannote.audio not importable (diarization will be disabled even with HF_AUTH_TOKEN)"
270
303
  $PYTHON_CMD -c "import fastapi, uvicorn" 2>/dev/null && print_success "fastapi/uvicorn imported successfully" || print_warning "fastapi/uvicorn not importable (transcription server will not start)"
271
304
 
305
+ # litellm must still import after litellm-enterprise was removed in step 6.4.
306
+ # Cheap check on purpose: `import litellm` takes ~2s, while importing
307
+ # litellm.proxy.proxy_server takes ~15s and would eat into the npm postinstall
308
+ # timeout that the CUDA torch download already strains.
309
+ $PYTHON_CMD -c "import litellm" 2>/dev/null && print_success "litellm imported successfully" || print_warning "litellm not importable (LiteLLM proxy will not start; only needed when EXULU_USE_LITELLM=true)"
310
+
272
311
  # Step 8: Display summary
273
312
  echo ""
274
313
  echo -e "${GREEN}========================================${NC}"