@exulu/backend 3.7.3 → 4.0.0

package/ee/python/documents/processing/tests/test_split_pdf.py

@@ -0,0 +1,230 @@
+"""Tests for split_pdf.py.
+
+Builds real PDFs with pypdf rather than using fixtures, so the suite is
+self-contained and the page counts/byte sizes are known exactly.
+
+Run from the repo root with the venv active:
+    cd ee/python/documents/processing && ../../.venv/bin/python -m pytest tests
+"""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from pypdf import PdfReader, PdfWriter
+from pypdf.generic import DecodedStreamObject, NameObject
+
+# Make ee/python/documents/processing importable.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import split_pdf as sp  # noqa: E402
+
+SCRIPT = Path(__file__).resolve().parent.parent / "split_pdf.py"
+
+
+def _make_pdf(path: Path, pages: int, page_bytes: int = 0) -> Path:
+    """Write a PDF whose pages carry a real content stream.
+
+    Each page's stream holds a `% exulu-page-N` marker so tests can assert the
+    page order survived, plus `page_bytes` of incompressible padding so the
+    byte-size bisection tests get a file whose size scales with page count.
+    Page content is used rather than custom dictionary keys because a PDF
+    library is free to drop unknown keys when it rebuilds pages, and PyMuPDF
+    does exactly that.
+    """
+    writer = PdfWriter()
+    for i in range(pages):
+        writer.add_blank_page(width=200, height=200)
+        body = f"% exulu-page-{i}\n".encode()
+        if page_bytes:
+            body += b"% " + os.urandom(page_bytes).hex().encode() + b"\n"
+        stream = DecodedStreamObject()
+        stream.set_data(body)
+        writer.pages[i][NameObject("/Contents")] = writer._add_object(stream)
+    with open(path, "wb") as fh:
+        writer.write(fh)
+    return path
+
+
+def _page_markers(pdf_path: str) -> list[str]:
+    """Read back the `% exulu-page-N` marker from every page of a chunk."""
+    markers = []
+    for page in PdfReader(pdf_path).pages:
+        data = page["/Contents"].get_data().decode("latin-1")
+        markers.append(data.splitlines()[0].removeprefix("% "))
+    return markers
+
+
+def _run_script(*args: str) -> tuple[int, str, str]:
+    proc = subprocess.run(
+        [sys.executable, str(SCRIPT), *args],
+        capture_output=True,
+        text=True,
+    )
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+# --- no-split path -----------------------------------------------------------
+
+
+def test_returns_original_when_within_limits(tmp_path):
+    src = _make_pdf(tmp_path / "small.pdf", pages=3)
+    chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=25)
+
+    assert len(chunks) == 1
+    assert chunks[0]["path"] == str(src.resolve())
+    assert chunks[0]["start_page"] == 0
+    assert chunks[0]["end_page"] == 3
+    # No copy is made, so the output dir is never created.
+    assert not (tmp_path / "out").exists()
+
+
+# --- page-count splitting ----------------------------------------------------
+
+
+def test_splits_on_page_count(tmp_path):
+    src = _make_pdf(tmp_path / "big.pdf", pages=10)
+    out = tmp_path / "out"
+    chunks = sp.split_pdf(str(src), str(out), chunk_size=4)
+
+    assert [(c["start_page"], c["end_page"]) for c in chunks] == [(0, 4), (4, 8), (8, 10)]
+    for c in chunks:
+        assert Path(c["path"]).exists()
+    assert [len(PdfReader(c["path"]).pages) for c in chunks] == [4, 4, 2]
+
+
+def test_chunk_ranges_are_contiguous_and_cover_every_page(tmp_path):
+    src = _make_pdf(tmp_path / "doc.pdf", pages=17)
+    chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=5)
+
+    assert chunks[0]["start_page"] == 0
+    assert chunks[-1]["end_page"] == 17
+    for prev, nxt in zip(chunks, chunks[1:]):
+        assert prev["end_page"] == nxt["start_page"]
+    assert sum(len(PdfReader(c["path"]).pages) for c in chunks) == 17
+
+
+def test_exact_multiple_of_chunk_size_produces_no_empty_trailing_chunk(tmp_path):
+    src = _make_pdf(tmp_path / "doc.pdf", pages=8)
+    chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=4)
+
+    assert [(c["start_page"], c["end_page"]) for c in chunks] == [(0, 4), (4, 8)]
+
+
+def test_page_content_is_preserved_in_order(tmp_path):
+    """The Nth page of the source must land at the right offset in its chunk."""
+    src = _make_pdf(tmp_path / "marked.pdf", pages=6)
+
+    chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=2)
+
+    seen = []
+    for c in chunks:
+        seen.extend(_page_markers(c["path"]))
+    assert seen == [f"exulu-page-{i}" for i in range(6)]
+
+
+# --- byte-size bisection -----------------------------------------------------
+
+
+def test_oversized_chunk_is_bisected_by_page_count(tmp_path):
+    # 4 pages, ~40 KB each. A 4-page chunk exceeds 100 KB and must bisect.
+    src = _make_pdf(tmp_path / "heavy.pdf", pages=4, page_bytes=20_000)
+    chunks = sp.split_pdf(
+        str(src), str(tmp_path / "out"), chunk_size=4, max_size_bytes=100_000
+    )
+
+    assert len(chunks) > 1
+    assert [(c["start_page"], c["end_page"]) for c in chunks][0][0] == 0
+    assert chunks[-1]["end_page"] == 4
+    for prev, nxt in zip(chunks, chunks[1:]):
+        assert prev["end_page"] == nxt["start_page"]
+
+
+def test_single_page_over_limit_is_kept_with_a_warning(tmp_path, capsys):
+    src = _make_pdf(tmp_path / "huge-page.pdf", pages=1, page_bytes=60_000)
+    # Force the split path: max_size_bytes below the file size.
+    chunks = sp.split_pdf(
+        str(src), str(tmp_path / "out"), chunk_size=1, max_size_bytes=1_000
+    )
+
+    assert len(chunks) == 1
+    assert Path(chunks[0]["path"]).exists()
+    assert "cannot be split further" in capsys.readouterr().err
+
+
+def test_file_over_size_limit_splits_even_when_page_count_fits(tmp_path):
+    src = _make_pdf(tmp_path / "wide.pdf", pages=4, page_bytes=20_000)
+    chunks = sp.split_pdf(
+        str(src), str(tmp_path / "out"), chunk_size=25, max_size_bytes=50_000
+    )
+
+    # chunk_size alone would have returned the original untouched.
+    assert len(chunks) > 1
+
+
+# --- encryption --------------------------------------------------------------
+
+
+def test_opens_phantom_password_pdf(tmp_path):
+    """A PDF encrypted with an empty password must open, as the OS does."""
+    writer = PdfWriter()
+    for _ in range(6):
+        writer.add_blank_page(width=200, height=200)
+    writer.encrypt("")
+    src = tmp_path / "phantom.pdf"
+    with open(src, "wb") as fh:
+        writer.write(fh)
+
+    assert PdfReader(src).is_encrypted
+
+    chunks = sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=2)
+    assert len(chunks) == 3
+    assert sum(len(PdfReader(c["path"]).pages) for c in chunks) == 6
+
+
+def test_rejects_pdf_with_a_real_password(tmp_path):
+    writer = PdfWriter()
+    writer.add_blank_page(width=200, height=200)
+    writer.encrypt("hunter2")
+    src = tmp_path / "locked.pdf"
+    with open(src, "wb") as fh:
+        writer.write(fh)
+
+    with pytest.raises(ValueError, match="non-empty password"):
+        sp.split_pdf(str(src), str(tmp_path / "out"), chunk_size=1)
+
+
+# --- CLI contract ------------------------------------------------------------
+
+
+def test_stdout_is_pure_json(tmp_path):
+    src = _make_pdf(tmp_path / "doc.pdf", pages=6)
+    code, out, err = _run_script(str(src), str(tmp_path / "out"), "--chunk-size", "2")
+
+    assert code == 0
+    parsed = json.loads(out)  # must not raise: nothing but JSON on stdout
+    assert len(parsed) == 3
+    # Diagnostics go to stderr, not stdout.
+    assert "[split_pdf]" in err
+    assert "[split_pdf]" not in out
+
+
+def test_cli_max_size_mb_is_converted_to_bytes(tmp_path):
+    src = _make_pdf(tmp_path / "heavy.pdf", pages=4, page_bytes=20_000)
+    code, out, _ = _run_script(
+        str(src), str(tmp_path / "out"), "--chunk-size", "4", "--max-size-mb", "0.05"
+    )
+
+    assert code == 0
+    assert len(json.loads(out)) > 1
+
+
+def test_exits_nonzero_on_failure(tmp_path):
+    code, out, err = _run_script(str(tmp_path / "missing.pdf"), str(tmp_path / "out"))
+
+    assert code == 1
+    assert "[split_pdf] ERROR" in err
+    assert out.strip() == ""

package/ee/python/requirements.txt

@@ -1,12 +1,15 @@
-docling
 # transformers <5: the 5.x line requires huggingface_hub>=1.0, which removed the
 # `use_auth_token` kwarg that pyannote.audio 3.x still passes to hf_hub_download()
 # (→ "unexpected keyword argument 'use_auth_token'", diarization silently
 # disabled). whisperx only needs transformers>=4.48, so the 4.x line is fine.
 transformers>=4.48,<5
 pyinstaller
-docling-hierarchical-pdf
 defusedxml
+# PDF page-range splitting for the "mistral" OCR processor (split_pdf.py).
+# BSD-3-Clause, no dependencies. Replaced PyMuPDF, which is AGPL-3.0 or a paid
+# Artifex commercial licence; see split_pdf.py. Do not reintroduce a PDF library
+# without checking its licence.
+pypdf>=6.1
 # Whisper transcription server. Used by `npx @exulu/backend exulu-start-whisper`.
 #
 # Notes on the pins:
@@ -30,6 +33,11 @@ requests
 # LiteLLM proxy — only used when EXULU_USE_LITELLM=true. Always installed so
 # the dep is ready when the env var is flipped. Pinned to a tested version;
 # upgrade deliberately.
+# NOTE: the [proxy] extra hard-requires litellm-enterprise, which is NOT MIT —
+# it carries the proprietary BerriAI Enterprise License. setup.sh step 6.4
+# uninstalls it right after this file is installed; see the comment there. Keep
+# the extra (we need its other 29 packages) and keep that uninstall step
+# together.
 litellm[proxy]==1.97.0
 # FastAPI ceiling — NOT cosmetic. litellm 1.97.0 declares fastapi>=0.136.3,<1.0
 # but imports the private `get_flat_dependant` from fastapi.dependencies.utils
@@ -52,3 +60,10 @@ prisma==0.15.0
 # NOT included in litellm[proxy]; without it, requests to Vertex models
 # fail with "No module named 'vertexai'".
 google-cloud-aiplatform>=1.38
+
+# Skill runtime dependencies — the session sandbox puts this venv's bin dir on PATH so
+# skill scripts (e.g. the built-in docx-manipulation skill) find their libraries without
+# creating a venv per session (which mirrored thousands of files into the tool result).
+python-docx>=1.1
+python-pptx>=1.0
+openpyxl>=3.1

package/ee/python/setup.sh

@@ -240,6 +240,39 @@ pip install -r "$REQUIREMENTS_FILE"
 
 print_success "All dependencies installed successfully"
 
+# Step 6.4: Remove the proprietary litellm-enterprise package.
+#
+# requirements.txt pins `litellm[proxy]`, and that extra hard-requires
+# litellm-enterprise. We need the other 29 packages the extra brings in
+# (gunicorn, fastapi, apscheduler, boto3, mcp, litellm-proxy-extras, …) but not
+# this one: it is NOT MIT like litellm itself. It carries the BerriAI Enterprise
+# License, which permits production use only with a paid per-seat subscription
+# and forbids redistributing the package. Exulu ships ee/ (including this script
+# and requirements.txt) inside the public npm package, so leaving it installed
+# would put a proprietary dependency on every customer's machine.
+#
+# Removing it is safe. Every one of litellm's 36 `litellm_enterprise` import
+# sites is wrapped in `try/except ImportError`; with the package absent the
+# proxy imports cleanly, `enterprise_proxy_config` falls back to None and the
+# route table is unchanged. Exulu never sets LITELLM_LICENSE, so `premium_user`
+# is False and no enterprise feature was reachable in the first place.
+#
+# If you DO hold a BerriAI Enterprise subscription and want the enterprise
+# callbacks, set EXULU_KEEP_LITELLM_ENTERPRISE=true to skip this step.
+echo ""
+if [ "${EXULU_KEEP_LITELLM_ENTERPRISE:-false}" = "true" ]; then
+    print_warning "EXULU_KEEP_LITELLM_ENTERPRISE=true — keeping litellm-enterprise (proprietary; requires a BerriAI subscription for production use)."
+elif pip show litellm-enterprise > /dev/null 2>&1; then
+    print_info "Removing litellm-enterprise (proprietary BerriAI package pulled in by litellm[proxy])..."
+    if pip uninstall -y litellm-enterprise > /dev/null 2>&1; then
+        print_success "litellm-enterprise removed; LiteLLM proxy runs on the MIT-licensed core"
+    else
+        print_warning "Could not remove litellm-enterprise. It is proprietary (BerriAI Enterprise License) and production use requires a paid subscription — remove it manually with 'pip uninstall -y litellm-enterprise' or set EXULU_KEEP_LITELLM_ENTERPRISE=true if you hold one."
+    fi
+else
+    print_info "litellm-enterprise not present — nothing to remove"
+fi
+
 # Step 6.5: Generate Prisma client for LiteLLM database mode.
 # LiteLLM's PrismaClient does `from prisma import Prisma`, which only works
 # after `prisma generate` has materialized the Python client against
@@ -259,7 +292,7 @@ echo "Step 7: Validating installation..."
 
 # Test critical imports
 print_info "Testing critical imports..."
-$PYTHON_CMD -c "import docling" 2>/dev/null && print_success "docling imported successfully" || print_error "Failed to import docling"
+$PYTHON_CMD -c "import pypdf" 2>/dev/null && print_success "pypdf imported successfully" || print_error "Failed to import pypdf"
 $PYTHON_CMD -c "import transformers" 2>/dev/null && print_success "transformers imported successfully" || print_error "Failed to import transformers"
 
 # Whisper transcription server imports — non-fatal: only needed for
@@ -269,6 +302,12 @@ $PYTHON_CMD -c "import whisperx" 2>/dev/null && print_success "whisperx imported
 $PYTHON_CMD -c "import pyannote.audio" 2>/dev/null && print_success "pyannote.audio imported successfully" || print_warning "pyannote.audio not importable (diarization will be disabled even with HF_AUTH_TOKEN)"
 $PYTHON_CMD -c "import fastapi, uvicorn" 2>/dev/null && print_success "fastapi/uvicorn imported successfully" || print_warning "fastapi/uvicorn not importable (transcription server will not start)"
 
+# litellm must still import after litellm-enterprise was removed in step 6.4.
+# Cheap check on purpose: `import litellm` takes ~2s, while importing
+# litellm.proxy.proxy_server takes ~15s and would eat into the npm postinstall
+# timeout that the CUDA torch download already strains.
+$PYTHON_CMD -c "import litellm" 2>/dev/null && print_success "litellm imported successfully" || print_warning "litellm not importable (LiteLLM proxy will not start; only needed when EXULU_USE_LITELLM=true)"
+
 # Step 8: Display summary
 echo ""
 echo -e "${GREEN}========================================${NC}"

package/ee/python/transcription/pipeline.py

@@ -35,6 +35,51 @@ class CancelledError(Exception):
     pass
 
 
+# Forced-alignment models whisperx would download that are NOT licence-cleared
+# for commercial use. Keys are the Hugging Face repo ids in whisperx's own
+# DEFAULT_ALIGN_MODELS_HF table; values are the reason, which is logged verbatim.
+#
+# whisperx picks an alignment model from the language Whisper *detected*, so
+# without this guard an ordinary upload in one of these languages silently pulls
+# the model onto the server and uses it. Alignment only refines timestamps to
+# word level: when it is skipped the transcript, its segment-level timings and
+# its speaker labels are all still produced (see _get_align_model).
+#
+# To allow one of these again, supply a licence-cleared replacement through
+# EXULU_ALIGN_MODEL_<LANG> rather than deleting the entry — e.g.
+# EXULU_ALIGN_MODEL_VI=my-org/licensed-vi-aligner.
+RESTRICTED_ALIGN_MODELS: dict[str, str] = {
+    "nguyenvulebinh/wav2vec2-base-vi":
+        "CC-BY-NC-4.0 — non-commercial use only",
+    "classla/wav2vec2-xls-r-parlaspeech-hr":
+        "no licence stated on the model card or in the Hugging Face metadata",
+    "imvladikon/wav2vec2-xls-r-300m-hebrew":
+        "no licence stated on the model card or in the Hugging Face metadata",
+    "theainerd/Wav2Vec2-large-xlsr-hindi":
+        "no licence stated on the model card or in the Hugging Face metadata",
+    # Danish is blocked on the conservative side: the model card states only
+    # that use "needs to adhere to this license from the Danish Parliament",
+    # and those terms have not been reviewed. Remove this entry once they have.
+    "saattrupdan/wav2vec2-xls-r-300m-ftspeech":
+        "licence is 'other' — refers to unreviewed Danish Parliament terms",
+}
+
+
+def _default_align_model_for(language_code: str) -> Optional[str]:
+    """The model id whisperx would resolve for this language, without loading it.
+
+    Mirrors load_align_model's own lookup order. Returns None for a language
+    whisperx has no default for, which it treats as an error anyway.
+    """
+    from whisperx.alignment import DEFAULT_ALIGN_MODELS_HF, DEFAULT_ALIGN_MODELS_TORCH
+
+    # The TORCH table holds torchaudio bundle names, which ship with torchaudio
+    # under BSD-2-Clause rather than being downloaded from Hugging Face.
+    if language_code in DEFAULT_ALIGN_MODELS_TORCH:
+        return DEFAULT_ALIGN_MODELS_TORCH[language_code]
+    return DEFAULT_ALIGN_MODELS_HF.get(language_code)
+
+
 def detect_device(requested: str = "auto") -> str:
     if requested != "auto":
         return requested
@@ -72,7 +117,9 @@ class TranscriptionPipeline:
         self.diarize_model = None
         self.diarization_enabled = False
         self.diarization_disabled_reason: str = "not attempted"
-        self.align_models: dict[str, tuple] = {}
+        self.align_models: dict[str, Optional[tuple]] = {}
+        # language_code -> why alignment was skipped, for observability
+        self.align_skipped_reasons: dict[str, str] = {}
 
     def load(self) -> None:
         # whisperx doesn't ship MPS support; run whisper on CPU when DEVICE=mps
@@ -122,12 +169,49 @@ class TranscriptionPipeline:
             print(f"[pipeline] Failed to load pyannote ({self.diarization_disabled_reason}); diarization disabled", flush=True)
 
     def _get_align_model(self, language_code: str):
-        if language_code not in self.align_models:
-            device = "cpu" if self.device == "mps" else self.device
-            print(f"[pipeline] Loading align model for {language_code}", flush=True)
-            self.align_models[language_code] = whisperx.load_align_model(
-                language_code=language_code, device=device
+        """Load the forced-alignment model for a language, or None if blocked.
+
+        Returns None when the model whisperx would use is not licence-cleared
+        and no replacement is configured. The caller skips alignment in that
+        case; nothing is downloaded, because the check runs before the load.
+        """
+        if language_code in self.align_models:
+            return self.align_models[language_code]
+
+        device = "cpu" if self.device == "mps" else self.device
+
+        # An operator-supplied replacement wins over whisperx's default, so a
+        # deployment that has licensed a model for one of the blocked languages
+        # can use it without patching this file.
+        override = os.getenv(f"EXULU_ALIGN_MODEL_{language_code.upper()}") or None
+        model_name = override or _default_align_model_for(language_code)
+
+        if override:
+            print(
+                f"[pipeline] Align model for {language_code} overridden by "
+                f"EXULU_ALIGN_MODEL_{language_code.upper()}={override}",
+                flush=True,
+            )
+        elif model_name in RESTRICTED_ALIGN_MODELS:
+            reason = RESTRICTED_ALIGN_MODELS[model_name]
+            self.align_skipped_reasons[language_code] = reason
+            print(
+                f"[pipeline] WARNING: word-level alignment skipped for "
+                f"'{language_code}'. Its default model '{model_name}' is not "
+                f"licence-cleared ({reason}) and was neither downloaded nor used. "
+                f"The transcript, segment timings and speaker labels are "
+                f"unaffected; only word-level timing precision is lost. Set "
+                f"EXULU_ALIGN_MODEL_{language_code.upper()} to a licensed model "
+                f"to re-enable alignment for this language.",
+                flush=True,
             )
+            self.align_models[language_code] = None
+            return None
+
+        print(f"[pipeline] Loading align model for {language_code}", flush=True)
+        self.align_models[language_code] = whisperx.load_align_model(
+            language_code=language_code, device=device, model_name=model_name
+        )
         return self.align_models[language_code]
 
     def transcribe(
@@ -170,15 +254,25 @@ class TranscriptionPipeline:
 
         language = transcribe_result["language"]
         align_device = "cpu" if self.device == "mps" else self.device
-        model_a, metadata = self._get_align_model(language)
-        aligned = whisperx.align(
-            transcribe_result["segments"],
-            model_a,
-            metadata,
-            audio,
-            align_device,
-            return_char_alignments=False,
-        )
+        align_bundle = self._get_align_model(language)
+        if align_bundle is None:
+            # Alignment blocked for this language (see RESTRICTED_ALIGN_MODELS).
+            # Whisper's own segments already carry start/end/text, and
+            # assign_word_speakers accepts an unaligned TranscriptionResult —
+            # it assigns speakers per segment and only walks words when a
+            # segment has them. So the transcript degrades to segment-level
+            # timing rather than failing.
+            aligned = transcribe_result
+        else:
+            model_a, metadata = align_bundle
+            aligned = whisperx.align(
+                transcribe_result["segments"],
+                model_a,
+                metadata,
+                audio,
+                align_device,
+                return_char_alignments=False,
+            )
 
         if is_cancelled():
             raise CancelledError()

package/ee/python/transcription/tests/test_align_model_licensing.py

@@ -0,0 +1,184 @@
+"""Tests for the forced-alignment licence guard in pipeline.py.
+
+The guard exists so that an ordinary upload in one of a handful of languages
+cannot pull a non-commercial or unlicensed wav2vec2 model onto the server.
+These tests assert the two things that matter: the blocked models are never
+loaded (so never downloaded), and a blocked language still produces a usable
+transcript.
+
+Run from the repo root with the venv active:
+    cd ee/python/transcription && ../.venv/bin/python -m pytest tests
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+# Make ee/python/transcription importable.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import pipeline as pl  # noqa: E402
+
+
+def _pipeline():
+    p = pl.TranscriptionPipeline("large-v3", "cpu", 4)
+    return p
+
+
+# --- the block list itself -------------------------------------------------
+
+
+def test_restricted_models_match_whisperx_defaults():
+    """Every blocked id must actually be a model whisperx would resolve.
+
+    Guards against a typo or an upstream rename silently disarming the guard.
+    """
+    from whisperx.alignment import DEFAULT_ALIGN_MODELS_HF
+
+    defaults = set(DEFAULT_ALIGN_MODELS_HF.values())
+    for model_id in pl.RESTRICTED_ALIGN_MODELS:
+        assert model_id in defaults, (
+            f"{model_id} is blocked but is no longer a whisperx default — "
+            "the upstream table changed and the guard may be stale"
+        )
+
+
+@pytest.mark.parametrize(
+    "language,expected_model",
+    [
+        ("vi", "nguyenvulebinh/wav2vec2-base-vi"),
+        ("hr", "classla/wav2vec2-xls-r-parlaspeech-hr"),
+        ("he", "imvladikon/wav2vec2-xls-r-300m-hebrew"),
+        ("hi", "theainerd/Wav2Vec2-large-xlsr-hindi"),
+        ("da", "saattrupdan/wav2vec2-xls-r-300m-ftspeech"),
+    ],
+)
+def test_restricted_languages_resolve_to_blocked_models(language, expected_model):
+    assert pl._default_align_model_for(language) == expected_model
+    assert expected_model in pl.RESTRICTED_ALIGN_MODELS
+
+
+# --- the guard -------------------------------------------------------------
+
+
+@pytest.mark.parametrize("language", ["vi", "hr", "he", "hi", "da"])
+def test_blocked_language_never_loads_a_model(language):
+    """The decisive test: load_align_model must not be called at all.
+
+    load_align_model is what reaches Hugging Face, so not calling it means the
+    weights are neither downloaded nor used.
+    """
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model") as load:
+        assert p._get_align_model(language) is None
+        load.assert_not_called()
+
+
+@pytest.mark.parametrize("language", ["vi", "hr", "he", "hi", "da"])
+def test_blocked_language_records_and_logs_a_reason(language, capsys):
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model"):
+        p._get_align_model(language)
+
+    assert language in p.align_skipped_reasons
+    out = capsys.readouterr().out
+    assert "WARNING" in out
+    assert "not licence-cleared" in out
+    assert pl.RESTRICTED_ALIGN_MODELS[pl._default_align_model_for(language)] in out
+
+
+def test_blocked_result_is_cached_so_the_warning_is_not_repeated(capsys):
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model") as load:
+        p._get_align_model("vi")
+        first = capsys.readouterr().out
+        p._get_align_model("vi")
+        second = capsys.readouterr().out
+        load.assert_not_called()
+
+    assert "WARNING" in first
+    assert second.strip() == ""
+
+
+# --- languages that are fine ------------------------------------------------
+
+
+@pytest.mark.parametrize("language", ["en", "de", "nl", "fr", "es"])
+def test_permitted_language_still_loads_normally(language):
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("model", "meta")) as load:
+        assert p._get_align_model(language) == ("model", "meta")
+        load.assert_called_once()
+        assert load.call_args.kwargs["language_code"] == language
+
+    assert p.align_skipped_reasons == {}
+
+
+def test_permitted_language_is_cached():
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("m", "meta")) as load:
+        p._get_align_model("en")
+        p._get_align_model("en")
+        load.assert_called_once()
+
+
+# --- operator override ------------------------------------------------------
+
+
+def test_override_re_enables_a_blocked_language(monkeypatch):
+    monkeypatch.setenv("EXULU_ALIGN_MODEL_VI", "my-org/licensed-vi-aligner")
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("m", "meta")) as load:
+        assert p._get_align_model("vi") == ("m", "meta")
+        assert load.call_args.kwargs["model_name"] == "my-org/licensed-vi-aligner"
+
+    assert p.align_skipped_reasons == {}
+
+
+def test_override_applies_to_a_permitted_language_too(monkeypatch):
+    monkeypatch.setenv("EXULU_ALIGN_MODEL_EN", "my-org/custom-en")
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("m", "meta")) as load:
+        p._get_align_model("en")
+        assert load.call_args.kwargs["model_name"] == "my-org/custom-en"
+
+
+def test_no_override_passes_the_whisperx_default_through(monkeypatch):
+    monkeypatch.delenv("EXULU_ALIGN_MODEL_EN", raising=False)
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("m", "meta")) as load:
+        p._get_align_model("en")
+        assert load.call_args.kwargs["model_name"] == pl._default_align_model_for("en")
+
+
+# --- graceful degradation ---------------------------------------------------
+
+
+def test_unaligned_segments_carry_everything_the_output_needs():
+    """transcribe() reads start/end/text/speaker; Whisper's own segments have
+    the first three, so skipping alignment cannot break the response shape."""
+    unaligned = {
+        "segments": [{"start": 0.0, "end": 1.5, "text": "xin chao"}],
+        "language": "vi",
+    }
+    for seg in unaligned["segments"]:
+        assert "start" in seg and "end" in seg and "text" in seg
+
+
+def test_assign_word_speakers_accepts_unaligned_input():
+    """The diarization step must tolerate segments that have no 'words' key,
+    otherwise a blocked language would crash instead of degrading."""
+    import pandas as pd
+    from whisperx.diarize import assign_word_speakers
+
+    diarize_df = pd.DataFrame(
+        [{"start": 0.0, "end": 2.0, "speaker": "SPEAKER_00", "segment": None, "label": "a"}]
+    )
+    unaligned = {"segments": [{"start": 0.0, "end": 1.5, "text": "xin chao"}]}
+
+    result = assign_word_speakers(diarize_df, unaligned)
+
+    assert result["segments"][0]["speaker"] == "SPEAKER_00"
+    assert result["segments"][0]["text"] == "xin chao"