@exulu/backend 3.7.4 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-27K2CO47.js → chunk-QMN6MVHQ.js} +1 -1
- package/dist/{chunk-AWMU6QXB.js → chunk-RBEWHG7I.js} +297 -39
- package/dist/cli/start-whisper.js +1 -1
- package/dist/{convert-exulu-tools-to-ai-sdk-tools-XNQ6Q3X6.js → convert-exulu-tools-to-ai-sdk-tools-6RU4IZMI.js} +1 -1
- package/dist/index.cjs +566 -217
- package/dist/index.d.cts +3 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.js +250 -180
- package/dist/{python-setup-JZGHWQCG.js → python-setup-DRJ3QX5F.js} +1 -1
- package/ee/LICENSE.md +2 -2
- package/ee/agentic-retrieval/pipeline/config.test.ts +18 -1
- package/ee/agentic-retrieval/pipeline/config.ts +15 -0
- package/ee/agentic-retrieval/pipeline/index.test.ts +73 -0
- package/ee/agentic-retrieval/pipeline/index.ts +67 -13
- package/ee/agentic-retrieval/pipeline/memory.test.ts +59 -0
- package/ee/agentic-retrieval/pipeline/memory.ts +181 -11
- package/ee/agentic-retrieval/pipeline/pin-rerun.test.ts +17 -0
- package/ee/agentic-retrieval/pipeline/pin-rerun.ts +29 -0
- package/ee/agentic-retrieval/pipeline/routing.test.ts +34 -0
- package/ee/agentic-retrieval/pipeline/routing.ts +96 -5
- package/ee/agentic-retrieval/pipeline/search.ts +9 -6
- package/ee/agentic-retrieval/pipeline/timing.test.ts +24 -0
- package/ee/agentic-retrieval/pipeline/timing.ts +26 -0
- package/ee/agentic-retrieval/pipeline/types.ts +2 -0
- package/ee/python/documents/processing/README.md +2 -3
- package/ee/python/documents/processing/doc_processor.ts +21 -61
- package/ee/python/documents/processing/split_pdf.py +25 -30
- package/ee/python/documents/processing/tests/__init__.py +0 -0
- package/ee/python/documents/processing/tests/test_split_pdf.py +230 -0
- package/ee/python/requirements.txt +12 -2
- package/ee/python/setup.sh +40 -1
- package/ee/python/transcription/pipeline.py +109 -15
- package/ee/python/transcription/tests/test_align_model_licensing.py +184 -0
- package/ee/workers.ts +2 -7
- package/license.md +2 -2
- package/package.json +3 -4
- package/scripts/postinstall.cjs +52 -1
- package/ee/python/documents/processing/document_to_markdown.py +0 -413
|
@@ -126,14 +126,13 @@ IMAGE_RESOLUTION_SCALE = 2.0 # Image resolution multiplier
|
|
|
126
126
|
|
|
127
127
|
This script requires the following Python packages (installed via `npm run python:setup`):
|
|
128
128
|
|
|
129
|
-
- `
|
|
130
|
-
- `docling-hierarchical-pdf` - Hierarchical heading processing
|
|
129
|
+
- `pypdf` - PDF page-range splitting for the `mistral` OCR processor
|
|
131
130
|
- `transformers` - ML-based text processing
|
|
132
131
|
- `PIL` - Image handling
|
|
133
132
|
|
|
134
133
|
### Troubleshooting
|
|
135
134
|
|
|
136
|
-
**Issue: ImportError for
|
|
135
|
+
**Issue: ImportError for pypdf**
|
|
137
136
|
```bash
|
|
138
137
|
npm run python:install
|
|
139
138
|
```
|
|
@@ -12,7 +12,6 @@ import WordExtractor from 'word-extractor';
|
|
|
12
12
|
import { parseOfficeAsync } from "officeparser";
|
|
13
13
|
import { checkLicense } from '@EE/entitlements';
|
|
14
14
|
import { executePythonScript } from '@SRC/utils/python-executor';
|
|
15
|
-
import { setupPythonEnvironment, validatePythonEnvironment } from '@SRC/utils/python-setup';
|
|
16
15
|
import { LiteParse } from '@llamaindex/liteparse';
|
|
17
16
|
import { resolveOcr } from '@SRC/exulu/resolve-ocr';
|
|
18
17
|
import type { ResolveOcrInput } from '@SRC/exulu/resolve-ocr';
|
|
@@ -31,7 +30,7 @@ type DocumentProcessorConfig = {
|
|
|
31
30
|
concurrency: number;
|
|
32
31
|
},
|
|
33
32
|
processor: {
|
|
34
|
-
name: "
|
|
33
|
+
name: "liteparse" | "mistral" | "officeparser"
|
|
35
34
|
/**
|
|
36
35
|
* LiteLLM model_name for the "mistral" OCR processor (declared in
|
|
37
36
|
* config.litellm.yaml). Defaults to "mistral-ocr". OCR is routed through
|
|
@@ -534,7 +533,7 @@ async function validateWithVLM(
|
|
|
534
533
|
verbose: boolean = false,
|
|
535
534
|
concurrency: number = 10
|
|
536
535
|
): Promise<ProcessedDocument> {
|
|
537
|
-
console.log(`[EXULU] Starting VLM validation for
|
|
536
|
+
console.log(`[EXULU] Starting VLM validation for processor output, ${document.length} pages...`);
|
|
538
537
|
console.log(`[EXULU] Concurrency limit: ${concurrency}`);
|
|
539
538
|
|
|
540
539
|
// Create a concurrency limiter
|
|
@@ -683,7 +682,7 @@ async function processDocument(
|
|
|
683
682
|
/*
|
|
684
683
|
tempDir/
|
|
685
684
|
uuid/
|
|
686
|
-
|
|
685
|
+
processed.json
|
|
687
686
|
images/
|
|
688
687
|
*/
|
|
689
688
|
const paths: ProcessingPaths = {
|
|
@@ -751,56 +750,7 @@ async function processPdf(
|
|
|
751
750
|
try {
|
|
752
751
|
let json: ProcessedDocument = [];
|
|
753
752
|
// Call the PDF processor script
|
|
754
|
-
if (config?.processor.name === "
|
|
755
|
-
|
|
756
|
-
// Validate Python environment and setup if needed
|
|
757
|
-
console.log(`[EXULU] Validating Python environment...`);
|
|
758
|
-
const validation = await validatePythonEnvironment(undefined, true);
|
|
759
|
-
|
|
760
|
-
if (!validation.valid) {
|
|
761
|
-
console.log(`[EXULU] Python environment not ready, setting up automatically...`);
|
|
762
|
-
console.log(`[EXULU] Reason: ${validation.message}`);
|
|
763
|
-
|
|
764
|
-
const setupResult = await setupPythonEnvironment({
|
|
765
|
-
verbose: true,
|
|
766
|
-
force: false, // Only setup if not already done
|
|
767
|
-
});
|
|
768
|
-
|
|
769
|
-
if (!setupResult.success) {
|
|
770
|
-
throw new Error(`Failed to setup Python environment: ${setupResult.message}\n\n${setupResult.output || ''}`);
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
console.log(`[EXULU] Python environment setup completed successfully`);
|
|
774
|
-
} else {
|
|
775
|
-
console.log(`[EXULU] Python environment is valid`);
|
|
776
|
-
}
|
|
777
|
-
|
|
778
|
-
console.log(`[EXULU] Processing document with document_to_markdown.py`);
|
|
779
|
-
|
|
780
|
-
const result = await executePythonScript({
|
|
781
|
-
scriptPath: 'ee/python/documents/processing/document_to_markdown.py',
|
|
782
|
-
args: [
|
|
783
|
-
paths.source,
|
|
784
|
-
'-o', paths.json,
|
|
785
|
-
'--images-dir', paths.images
|
|
786
|
-
],
|
|
787
|
-
timeout: 30 * 60 * 1000, // 30 minutes for large documents
|
|
788
|
-
});
|
|
789
|
-
|
|
790
|
-
// Log processing info from stderr
|
|
791
|
-
if (result.stderr) {
|
|
792
|
-
console.log('Processing info:', result.stderr.trim());
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
if (!result.success) {
|
|
796
|
-
throw new Error(`Document processing failed: ${result.stderr}`);
|
|
797
|
-
}
|
|
798
|
-
|
|
799
|
-
// Read the generated JSON file
|
|
800
|
-
const jsonContent = await fs.promises.readFile(paths.json, 'utf-8');
|
|
801
|
-
json = JSON.parse(jsonContent);
|
|
802
|
-
|
|
803
|
-
} else if (config?.processor.name === "officeparser") {
|
|
753
|
+
if (config?.processor.name === "officeparser") {
|
|
804
754
|
const text = await parseOfficeAsync(buffer, {
|
|
805
755
|
outputErrorToConsole: false,
|
|
806
756
|
newlineDelimiter: "\n",
|
|
@@ -934,16 +884,29 @@ async function processPdf(
|
|
|
934
884
|
}));
|
|
935
885
|
|
|
936
886
|
fs.writeFileSync(paths.json, JSON.stringify(json, null, 2));
|
|
887
|
+
} else {
|
|
888
|
+
// Without this the if/else chain fell through leaving `json` empty, and
|
|
889
|
+
// the document was stored as zero pages with no error anywhere — a silent
|
|
890
|
+
// data-loss path. "docling" in particular used to be a valid value here.
|
|
891
|
+
// Every member of the union is handled above, so TypeScript narrows
|
|
892
|
+
// `processor.name` to `never` here. At runtime a caller can still pass
|
|
893
|
+
// anything, because these configs are routinely built from plain JS or
|
|
894
|
+
// from database rows. String() widens it back for the message.
|
|
895
|
+
const configured = String(config?.processor?.name ?? '');
|
|
896
|
+
throw new Error(
|
|
897
|
+
configured === ''
|
|
898
|
+
? '[EXULU] No document processor configured. Set processor.name to one of: mistral, liteparse, officeparser.'
|
|
899
|
+
: `[EXULU] Unknown document processor "${configured}". Supported processors are: mistral, liteparse, officeparser.` +
|
|
900
|
+
(configured === 'docling'
|
|
901
|
+
? ' The "docling" processor was removed: it depended on PyMuPDF, which is AGPL-licensed. Use "mistral" for PDF OCR.'
|
|
902
|
+
: '')
|
|
903
|
+
);
|
|
937
904
|
}
|
|
938
905
|
|
|
939
906
|
console.log(`[EXULU] \n✓ Document processing completed successfully`);
|
|
940
907
|
console.log(`[EXULU] Total pages: ${json.length}`);
|
|
941
908
|
console.log(`[EXULU] Output file: ${paths.json}`);
|
|
942
909
|
|
|
943
|
-
if (config?.vlm?.model) {
|
|
944
|
-
console.error('[EXULU] VLM validation is only supported when docling is enabled, skipping validation.');
|
|
945
|
-
}
|
|
946
|
-
|
|
947
910
|
// Apply VLM validation if enabled
|
|
948
911
|
const vlmModel = config?.vlm?.model ? await resolveVlmModel(config) : undefined;
|
|
949
912
|
if (vlmModel && json.length > 0) {
|
|
@@ -1121,9 +1084,6 @@ export async function documentProcessor({
|
|
|
1121
1084
|
|
|
1122
1085
|
let supportedTypes: string[] = [];
|
|
1123
1086
|
switch (config?.processor.name) {
|
|
1124
|
-
case "docling":
|
|
1125
|
-
supportedTypes = ['pdf', 'docx', 'doc', 'txt', 'md', 'jpg', 'jpeg', 'png', 'gif', 'webp'];
|
|
1126
|
-
break;
|
|
1127
1087
|
case "officeparser":
|
|
1128
1088
|
supportedTypes = ['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'rtf', 'csv', 'md', 'html'];
|
|
1129
1089
|
break;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
"""
|
|
3
|
-
PDF Splitter — splits a PDF into fixed-size page chunks using
|
|
3
|
+
PDF Splitter — splits a PDF into fixed-size page chunks using pypdf.
|
|
4
4
|
|
|
5
5
|
Outputs a JSON array to stdout, each element:
|
|
6
6
|
{ "path": "<absolute-path>", "start_page": <int>, "end_page": <int> }
|
|
@@ -22,26 +22,20 @@ import argparse
|
|
|
22
22
|
|
|
23
23
|
# stdout is this script's result channel and the caller does JSON.parse() on it,
|
|
24
24
|
# so nothing else may write there. Our own prints all pass file=sys.stderr, but
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
# and
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
# keep a private handle for the result, so any library that prints — now or
|
|
32
|
-
# after a future dependency bump — is shunted to the log channel instead of
|
|
33
|
-
# corrupting the payload.
|
|
25
|
+
# a dependency need not: any library is free to emit a deprecation banner or a
|
|
26
|
+
# progress line on stdout, and it would land ahead of the payload and fail the
|
|
27
|
+
# caller with "Unexpected token 'w'". Point sys.stdout at stderr before
|
|
28
|
+
# importing anything and keep a private handle for the result, so a library
|
|
29
|
+
# that prints — now or after a future dependency bump — is shunted to the log
|
|
30
|
+
# channel instead of corrupting the payload.
|
|
34
31
|
_stdout = sys.stdout
|
|
35
32
|
sys.stdout = sys.stderr
|
|
36
33
|
|
|
37
|
-
|
|
38
|
-
import pymupdf as fitz # PyMuPDF >= 1.24.3, where the module was renamed
|
|
39
|
-
except ImportError: # older releases only ship the legacy `fitz` module
|
|
40
|
-
import fitz
|
|
34
|
+
from pypdf import PdfReader, PdfWriter
|
|
41
35
|
|
|
42
36
|
|
|
43
37
|
def _write_chunk(
|
|
44
|
-
|
|
38
|
+
reader: PdfReader,
|
|
45
39
|
output_dir: str,
|
|
46
40
|
chunk_start: int,
|
|
47
41
|
chunk_end: int,
|
|
@@ -56,10 +50,11 @@ def _write_chunk(
|
|
|
56
50
|
"""
|
|
57
51
|
chunk_path = os.path.join(output_dir, f"chunk_{chunk_start}_{chunk_end - 1}.pdf")
|
|
58
52
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
53
|
+
writer = PdfWriter()
|
|
54
|
+
writer.append(reader, pages=(chunk_start, chunk_end))
|
|
55
|
+
with open(chunk_path, "wb") as fh:
|
|
56
|
+
writer.write(fh)
|
|
57
|
+
writer.close()
|
|
63
58
|
|
|
64
59
|
chunk_bytes = os.path.getsize(chunk_path)
|
|
65
60
|
n_pages = chunk_end - chunk_start
|
|
@@ -68,8 +63,8 @@ def _write_chunk(
|
|
|
68
63
|
os.remove(chunk_path)
|
|
69
64
|
mid = chunk_start + n_pages // 2
|
|
70
65
|
return (
|
|
71
|
-
_write_chunk(
|
|
72
|
-
+ _write_chunk(
|
|
66
|
+
_write_chunk(reader, output_dir, chunk_start, mid, max_size_bytes)
|
|
67
|
+
+ _write_chunk(reader, output_dir, mid, chunk_end, max_size_bytes)
|
|
73
68
|
)
|
|
74
69
|
|
|
75
70
|
if max_size_bytes and chunk_bytes > max_size_bytes:
|
|
@@ -88,21 +83,21 @@ def split_pdf(
|
|
|
88
83
|
chunk_size: int,
|
|
89
84
|
max_size_bytes: int | None = None,
|
|
90
85
|
) -> list[dict]:
|
|
91
|
-
|
|
86
|
+
reader = PdfReader(input_path)
|
|
92
87
|
|
|
93
88
|
# Some PDFs are saved with an empty owner/user password by certain writers
|
|
94
89
|
# (e.g. older Adobe Acrobat exports). The OS opens them transparently by
|
|
95
90
|
# trying "" first, but most libraries raise immediately. We replicate that
|
|
96
|
-
# OS-level behaviour here.
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
if not
|
|
91
|
+
# OS-level behaviour here. pypdf's decrypt() returns a PasswordType enum
|
|
92
|
+
# whose NOT_DECRYPTED member is falsy, so a plain truth test is enough.
|
|
93
|
+
if reader.is_encrypted:
|
|
94
|
+
if not reader.decrypt(""):
|
|
100
95
|
raise ValueError(
|
|
101
96
|
"PDF requires a non-empty password and cannot be opened automatically."
|
|
102
97
|
)
|
|
103
98
|
print("[split_pdf] Authenticated with empty password (phantom-password PDF)", file=sys.stderr)
|
|
104
99
|
|
|
105
|
-
total_pages = len(
|
|
100
|
+
total_pages = len(reader.pages)
|
|
106
101
|
file_size = os.path.getsize(input_path)
|
|
107
102
|
print(
|
|
108
103
|
f"[split_pdf] Total pages: {total_pages}, chunk size: {chunk_size}, "
|
|
@@ -115,7 +110,7 @@ def split_pdf(
|
|
|
115
110
|
|
|
116
111
|
if not needs_split:
|
|
117
112
|
print("[split_pdf] No split needed — returning original path", file=sys.stderr)
|
|
118
|
-
|
|
113
|
+
reader.close()
|
|
119
114
|
return [{
|
|
120
115
|
"path": os.path.abspath(input_path),
|
|
121
116
|
"start_page": 0,
|
|
@@ -127,7 +122,7 @@ def split_pdf(
|
|
|
127
122
|
chunks = []
|
|
128
123
|
for start_page in range(0, total_pages, chunk_size):
|
|
129
124
|
end_page = min(start_page + chunk_size, total_pages)
|
|
130
|
-
sub_chunks = _write_chunk(
|
|
125
|
+
sub_chunks = _write_chunk(reader, output_dir, start_page, end_page, max_size_bytes)
|
|
131
126
|
for c in sub_chunks:
|
|
132
127
|
print(
|
|
133
128
|
f"[split_pdf] Chunk {len(chunks) + 1}: pages {c['start_page']}–{c['end_page'] - 1} "
|
|
@@ -136,7 +131,7 @@ def split_pdf(
|
|
|
136
131
|
)
|
|
137
132
|
chunks.extend(sub_chunks)
|
|
138
133
|
|
|
139
|
-
|
|
134
|
+
reader.close()
|
|
140
135
|
return chunks
|
|
141
136
|
|
|
142
137
|
|
|
File without changes
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Tests for split_pdf.py.
|
|
2
|
+
|
|
3
|
+
Builds real PDFs with pypdf rather than using fixtures, so the suite is
|
|
4
|
+
self-contained and the page counts/byte sizes are known exactly.
|
|
5
|
+
|
|
6
|
+
Run from the repo root with the venv active:
|
|
7
|
+
cd ee/python/documents/processing && ../../.venv/bin/python -m pytest tests
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import pytest
|
|
17
|
+
from pypdf import PdfReader, PdfWriter
|
|
18
|
+
from pypdf.generic import DecodedStreamObject, NameObject
|
|
19
|
+
|
|
20
|
+
# Make ee/python/documents/processing importable.
|
|
21
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
22
|
+
|
|
23
|
+
import split_pdf as sp # noqa: E402
|
|
24
|
+
|
|
25
|
+
SCRIPT = Path(__file__).resolve().parent.parent / "split_pdf.py"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _make_pdf(path: Path, pages: int, page_bytes: int = 0) -> Path:
|
|
29
|
+
"""Write a PDF whose pages carry a real content stream.
|
|
30
|
+
|
|
31
|
+
Each page's stream holds a `% exulu-page-N` marker so tests can assert the
|
|
32
|
+
page order survived, plus `page_bytes` of incompressible padding so the
|
|
33
|
+
byte-size bisection tests get a file whose size scales with page count.
|
|
34
|
+
Page content is used rather than custom dictionary keys because a PDF
|
|
35
|
+
library is free to drop unknown keys when it rebuilds pages, and PyMuPDF
|
|
36
|
+
does exactly that.
|
|
37
|
+
"""
|
|
38
|
+
writer = PdfWriter()
|
|
39
|
+
for i in range(pages):
|
|
40
|
+
writer.add_blank_page(width=200, height=200)
|
|
41
|
+
body = f"% exulu-page-{i}\n".encode()
|
|
42
|
+
if page_bytes:
|
|
43
|
+
body += b"% " + os.urandom(page_bytes).hex().encode() + b"\n"
|
|
44
|
+
stream = DecodedStreamObject()
|
|
45
|
+
stream.set_data(body)
|
|
46
|
+
writer.pages[i][NameObject("/Contents")] = writer._add_object(stream)
|
|
47
|
+
with open(path, "wb") as fh:
|
|
48
|
+
writer.write(fh)
|
|
49
|
+
return path
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _page_markers(pdf_path: str) -> list[str]:
|
|
53
|
+
"""Read back the `% exulu-page-N` marker from every page of a chunk."""
|
|
54
|
+
markers = []
|
|
55
|
+
for page in PdfReader(pdf_path).pages:
|
|
56
|
+
data = page["/Contents"].get_data().decode("latin-1")
|
|
57
|
+
markers.append(data.splitlines()[0].removeprefix("% "))
|
|
58
|
+
return markers
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _run_script(*args: str) -> tuple[int, str, str]:
|
|
62
|
+
proc = subprocess.run(
|
|
63
|
+
[sys.executable, str(SCRIPT), *args],
|
|
64
|
+
capture_output=True,
|
|
65
|
+
text=True,
|
|
66
|
+
)
|
|
67
|
+
return proc.returncode, proc.stdout, proc.stderr
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# --- no-split path -----------------------------------------------------------
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_returns_original_when_within_limits(tmp_path):
|
|
74
|
+
src = _make_pdf(tmp_path / "small.pdf", pages=3)
|
|
75
|
+
chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=25)
|
|
76
|
+
|
|
77
|
+
assert len(chunks) == 1
|
|
78
|
+
assert chunks[0]["path"] == str(src.resolve())
|
|
79
|
+
assert chunks[0]["start_page"] == 0
|
|
80
|
+
assert chunks[0]["end_page"] == 3
|
|
81
|
+
# No copy is made, so the output dir is never created.
|
|
82
|
+
assert not (tmp_path / "out").exists()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# --- page-count splitting ----------------------------------------------------
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_splits_on_page_count(tmp_path):
|
|
89
|
+
src = _make_pdf(tmp_path / "big.pdf", pages=10)
|
|
90
|
+
out = tmp_path / "out"
|
|
91
|
+
chunks = sp.split_pdf(str(src), str(out), chunk_size=4)
|
|
92
|
+
|
|
93
|
+
assert [(c["start_page"], c["end_page"]) for c in chunks] == [(0, 4), (4, 8), (8, 10)]
|
|
94
|
+
for c in chunks:
|
|
95
|
+
assert Path(c["path"]).exists()
|
|
96
|
+
assert [len(PdfReader(c["path"]).pages) for c in chunks] == [4, 4, 2]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_chunk_ranges_are_contiguous_and_cover_every_page(tmp_path):
|
|
100
|
+
src = _make_pdf(tmp_path / "doc.pdf", pages=17)
|
|
101
|
+
chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=5)
|
|
102
|
+
|
|
103
|
+
assert chunks[0]["start_page"] == 0
|
|
104
|
+
assert chunks[-1]["end_page"] == 17
|
|
105
|
+
for prev, nxt in zip(chunks, chunks[1:]):
|
|
106
|
+
assert prev["end_page"] == nxt["start_page"]
|
|
107
|
+
assert sum(len(PdfReader(c["path"]).pages) for c in chunks) == 17
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_exact_multiple_of_chunk_size_produces_no_empty_trailing_chunk(tmp_path):
|
|
111
|
+
src = _make_pdf(tmp_path / "doc.pdf", pages=8)
|
|
112
|
+
chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=4)
|
|
113
|
+
|
|
114
|
+
assert [(c["start_page"], c["end_page"]) for c in chunks] == [(0, 4), (4, 8)]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_page_content_is_preserved_in_order(tmp_path):
|
|
118
|
+
"""The Nth page of the source must land at the right offset in its chunk."""
|
|
119
|
+
src = _make_pdf(tmp_path / "marked.pdf", pages=6)
|
|
120
|
+
|
|
121
|
+
chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=2)
|
|
122
|
+
|
|
123
|
+
seen = []
|
|
124
|
+
for c in chunks:
|
|
125
|
+
seen.extend(_page_markers(c["path"]))
|
|
126
|
+
assert seen == [f"exulu-page-{i}" for i in range(6)]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# --- byte-size bisection -----------------------------------------------------
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_oversized_chunk_is_bisected_by_page_count(tmp_path):
|
|
133
|
+
# 4 pages, ~40 KB each. A 4-page chunk exceeds 100 KB and must bisect.
|
|
134
|
+
src = _make_pdf(tmp_path / "heavy.pdf", pages=4, page_bytes=20_000)
|
|
135
|
+
chunks = sp.split_pdf(
|
|
136
|
+
str(src), str(tmp_path / "out"), chunk_size=4, max_size_bytes=100_000
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
assert len(chunks) > 1
|
|
140
|
+
assert [(c["start_page"], c["end_page"]) for c in chunks][0][0] == 0
|
|
141
|
+
assert chunks[-1]["end_page"] == 4
|
|
142
|
+
for prev, nxt in zip(chunks, chunks[1:]):
|
|
143
|
+
assert prev["end_page"] == nxt["start_page"]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_single_page_over_limit_is_kept_with_a_warning(tmp_path, capsys):
|
|
147
|
+
src = _make_pdf(tmp_path / "huge-page.pdf", pages=1, page_bytes=60_000)
|
|
148
|
+
# Force the split path: max_size_bytes below the file size.
|
|
149
|
+
chunks = sp.split_pdf(
|
|
150
|
+
str(src), str(tmp_path / "out"), chunk_size=1, max_size_bytes=1_000
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
assert len(chunks) == 1
|
|
154
|
+
assert Path(chunks[0]["path"]).exists()
|
|
155
|
+
assert "cannot be split further" in capsys.readouterr().err
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_file_over_size_limit_splits_even_when_page_count_fits(tmp_path):
|
|
159
|
+
src = _make_pdf(tmp_path / "wide.pdf", pages=4, page_bytes=20_000)
|
|
160
|
+
chunks = sp.split_pdf(
|
|
161
|
+
str(src), str(tmp_path / "out"), chunk_size=25, max_size_bytes=50_000
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# chunk_size alone would have returned the original untouched.
|
|
165
|
+
assert len(chunks) > 1
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# --- encryption --------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_opens_phantom_password_pdf(tmp_path):
|
|
172
|
+
"""A PDF encrypted with an empty password must open, as the OS does."""
|
|
173
|
+
writer = PdfWriter()
|
|
174
|
+
for _ in range(6):
|
|
175
|
+
writer.add_blank_page(width=200, height=200)
|
|
176
|
+
writer.encrypt("")
|
|
177
|
+
src = tmp_path / "phantom.pdf"
|
|
178
|
+
with open(src, "wb") as fh:
|
|
179
|
+
writer.write(fh)
|
|
180
|
+
|
|
181
|
+
assert PdfReader(src).is_encrypted
|
|
182
|
+
|
|
183
|
+
chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=2)
|
|
184
|
+
assert len(chunks) == 3
|
|
185
|
+
assert sum(len(PdfReader(c["path"]).pages) for c in chunks) == 6
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def test_rejects_pdf_with_a_real_password(tmp_path):
|
|
189
|
+
writer = PdfWriter()
|
|
190
|
+
writer.add_blank_page(width=200, height=200)
|
|
191
|
+
writer.encrypt("hunter2")
|
|
192
|
+
src = tmp_path / "locked.pdf"
|
|
193
|
+
with open(src, "wb") as fh:
|
|
194
|
+
writer.write(fh)
|
|
195
|
+
|
|
196
|
+
with pytest.raises(ValueError, match="non-empty password"):
|
|
197
|
+
sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=1)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# --- CLI contract ------------------------------------------------------------
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def test_stdout_is_pure_json(tmp_path):
|
|
204
|
+
src = _make_pdf(tmp_path / "doc.pdf", pages=6)
|
|
205
|
+
code, out, err = _run_script(str(src), str(tmp_path / "out"), "--chunk-size", "2")
|
|
206
|
+
|
|
207
|
+
assert code == 0
|
|
208
|
+
parsed = json.loads(out) # must not raise: nothing but JSON on stdout
|
|
209
|
+
assert len(parsed) == 3
|
|
210
|
+
# Diagnostics go to stderr, not stdout.
|
|
211
|
+
assert "[split_pdf]" in err
|
|
212
|
+
assert "[split_pdf]" not in out
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def test_cli_max_size_mb_is_converted_to_bytes(tmp_path):
|
|
216
|
+
src = _make_pdf(tmp_path / "heavy.pdf", pages=4, page_bytes=20_000)
|
|
217
|
+
code, out, _ = _run_script(
|
|
218
|
+
str(src), str(tmp_path / "out"), "--chunk-size", "4", "--max-size-mb", "0.05"
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
assert code == 0
|
|
222
|
+
assert len(json.loads(out)) > 1
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def test_exits_nonzero_on_failure(tmp_path):
|
|
226
|
+
code, out, err = _run_script(str(tmp_path / "missing.pdf"), str(tmp_path / "out"))
|
|
227
|
+
|
|
228
|
+
assert code == 1
|
|
229
|
+
assert "[split_pdf] ERROR" in err
|
|
230
|
+
assert out.strip() == ""
|
|
@@ -1,12 +1,15 @@
|
|
|
1
|
-
docling
|
|
2
1
|
# transformers <5: the 5.x line requires huggingface_hub>=1.0, which removed the
|
|
3
2
|
# `use_auth_token` kwarg that pyannote.audio 3.x still passes to hf_hub_download()
|
|
4
3
|
# (→ "unexpected keyword argument 'use_auth_token'", diarization silently
|
|
5
4
|
# disabled). whisperx only needs transformers>=4.48, so the 4.x line is fine.
|
|
6
5
|
transformers>=4.48,<5
|
|
7
6
|
pyinstaller
|
|
8
|
-
docling-hierarchical-pdf
|
|
9
7
|
defusedxml
|
|
8
|
+
# PDF page-range splitting for the "mistral" OCR processor (split_pdf.py).
|
|
9
|
+
# BSD-3-Clause, no dependencies. Replaced PyMuPDF, which is AGPL-3.0 or a paid
|
|
10
|
+
# Artifex commercial licence; see split_pdf.py. Do not reintroduce a PDF library
|
|
11
|
+
# without checking its licence.
|
|
12
|
+
pypdf>=6.1
|
|
10
13
|
# Whisper transcription server. Used by `npx @exulu/backend exulu-start-whisper`.
|
|
11
14
|
#
|
|
12
15
|
# Notes on the pins:
|
|
@@ -30,6 +33,11 @@ requests
|
|
|
30
33
|
# LiteLLM proxy — only used when EXULU_USE_LITELLM=true. Always installed so
|
|
31
34
|
# the dep is ready when the env var is flipped. Pinned to a tested version;
|
|
32
35
|
# upgrade deliberately.
|
|
36
|
+
# NOTE: the [proxy] extra hard-requires litellm-enterprise, which is NOT MIT —
|
|
37
|
+
# it carries the proprietary BerriAI Enterprise License. setup.sh step 6.4
|
|
38
|
+
# uninstalls it right after this file is installed; see the comment there. Keep
|
|
39
|
+
# the extra (we need its other 29 packages) and keep that uninstall step
|
|
40
|
+
# together.
|
|
33
41
|
litellm[proxy]==1.97.0
|
|
34
42
|
# FastAPI ceiling — NOT cosmetic. litellm 1.97.0 declares fastapi>=0.136.3,<1.0
|
|
35
43
|
# but imports the private `get_flat_dependant` from fastapi.dependencies.utils
|
|
@@ -57,3 +65,5 @@ google-cloud-aiplatform>=1.38
|
|
|
57
65
|
# skill scripts (e.g. the built-in docx-manipulation skill) find their libraries without
|
|
58
66
|
# creating a venv per session (which mirrored thousands of files into the tool result).
|
|
59
67
|
python-docx>=1.1
|
|
68
|
+
python-pptx>=1.0
|
|
69
|
+
openpyxl>=3.1
|
package/ee/python/setup.sh
CHANGED
|
@@ -240,6 +240,39 @@ pip install -r "$REQUIREMENTS_FILE"
|
|
|
240
240
|
|
|
241
241
|
print_success "All dependencies installed successfully"
|
|
242
242
|
|
|
243
|
+
# Step 6.4: Remove the proprietary litellm-enterprise package.
|
|
244
|
+
#
|
|
245
|
+
# requirements.txt pins `litellm[proxy]`, and that extra hard-requires
|
|
246
|
+
# litellm-enterprise. We need the other 29 packages the extra brings in
|
|
247
|
+
# (gunicorn, fastapi, apscheduler, boto3, mcp, litellm-proxy-extras, …) but not
|
|
248
|
+
# this one: it is NOT MIT like litellm itself. It carries the BerriAI Enterprise
|
|
249
|
+
# License, which permits production use only with a paid per-seat subscription
|
|
250
|
+
# and forbids redistributing the package. Exulu ships ee/ (including this script
|
|
251
|
+
# and requirements.txt) inside the public npm package, so leaving it installed
|
|
252
|
+
# would put a proprietary dependency on every customer's machine.
|
|
253
|
+
#
|
|
254
|
+
# Removing it is safe. Every one of litellm's 36 `litellm_enterprise` import
|
|
255
|
+
# sites is wrapped in `try/except ImportError`; with the package absent the
|
|
256
|
+
# proxy imports cleanly, `enterprise_proxy_config` falls back to None and the
|
|
257
|
+
# route table is unchanged. Exulu never sets LITELLM_LICENSE, so `premium_user`
|
|
258
|
+
# is False and no enterprise feature was reachable in the first place.
|
|
259
|
+
#
|
|
260
|
+
# If you DO hold a BerriAI Enterprise subscription and want the enterprise
|
|
261
|
+
# callbacks, set EXULU_KEEP_LITELLM_ENTERPRISE=true to skip this step.
|
|
262
|
+
echo ""
|
|
263
|
+
if [ "${EXULU_KEEP_LITELLM_ENTERPRISE:-false}" = "true" ]; then
|
|
264
|
+
print_warning "EXULU_KEEP_LITELLM_ENTERPRISE=true — keeping litellm-enterprise (proprietary; requires a BerriAI subscription for production use)."
|
|
265
|
+
elif pip show litellm-enterprise > /dev/null 2>&1; then
|
|
266
|
+
print_info "Removing litellm-enterprise (proprietary BerriAI package pulled in by litellm[proxy])..."
|
|
267
|
+
if pip uninstall -y litellm-enterprise > /dev/null 2>&1; then
|
|
268
|
+
print_success "litellm-enterprise removed; LiteLLM proxy runs on the MIT-licensed core"
|
|
269
|
+
else
|
|
270
|
+
print_warning "Could not remove litellm-enterprise. It is proprietary (BerriAI Enterprise License) and production use requires a paid subscription — remove it manually with 'pip uninstall -y litellm-enterprise' or set EXULU_KEEP_LITELLM_ENTERPRISE=true if you hold one."
|
|
271
|
+
fi
|
|
272
|
+
else
|
|
273
|
+
print_info "litellm-enterprise not present — nothing to remove"
|
|
274
|
+
fi
|
|
275
|
+
|
|
243
276
|
# Step 6.5: Generate Prisma client for LiteLLM database mode.
|
|
244
277
|
# LiteLLM's PrismaClient does `from prisma import Prisma`, which only works
|
|
245
278
|
# after `prisma generate` has materialized the Python client against
|
|
@@ -259,7 +292,7 @@ echo "Step 7: Validating installation..."
|
|
|
259
292
|
|
|
260
293
|
# Test critical imports
|
|
261
294
|
print_info "Testing critical imports..."
|
|
262
|
-
$PYTHON_CMD -c "import
|
|
295
|
+
$PYTHON_CMD -c "import pypdf" 2>/dev/null && print_success "pypdf imported successfully" || print_error "Failed to import pypdf"
|
|
263
296
|
$PYTHON_CMD -c "import transformers" 2>/dev/null && print_success "transformers imported successfully" || print_error "Failed to import transformers"
|
|
264
297
|
|
|
265
298
|
# Whisper transcription server imports — non-fatal: only needed for
|
|
@@ -269,6 +302,12 @@ $PYTHON_CMD -c "import whisperx" 2>/dev/null && print_success "whisperx imported
|
|
|
269
302
|
$PYTHON_CMD -c "import pyannote.audio" 2>/dev/null && print_success "pyannote.audio imported successfully" || print_warning "pyannote.audio not importable (diarization will be disabled even with HF_AUTH_TOKEN)"
|
|
270
303
|
$PYTHON_CMD -c "import fastapi, uvicorn" 2>/dev/null && print_success "fastapi/uvicorn imported successfully" || print_warning "fastapi/uvicorn not importable (transcription server will not start)"
|
|
271
304
|
|
|
305
|
+
# litellm must still import after litellm-enterprise was removed in step 6.4.
|
|
306
|
+
# Cheap check on purpose: `import litellm` takes ~2s, while importing
|
|
307
|
+
# litellm.proxy.proxy_server takes ~15s and would eat into the npm postinstall
|
|
308
|
+
# timeout that the CUDA torch download already strains.
|
|
309
|
+
$PYTHON_CMD -c "import litellm" 2>/dev/null && print_success "litellm imported successfully" || print_warning "litellm not importable (LiteLLM proxy will not start; only needed when EXULU_USE_LITELLM=true)"
|
|
310
|
+
|
|
272
311
|
# Step 8: Display summary
|
|
273
312
|
echo ""
|
|
274
313
|
echo -e "${GREEN}========================================${NC}"
|