@exulu/backend 3.7.4 → 4.0.0

package/ee/python/transcription/pipeline.py

@@ -35,6 +35,51 @@ class CancelledError(Exception):
     pass
 
 
+# Forced-alignment models whisperx would download that are NOT licence-cleared
+# for commercial use. Keys are the Hugging Face repo ids in whisperx's own
+# DEFAULT_ALIGN_MODELS_HF table; values are the reason, which is logged verbatim.
+#
+# whisperx picks an alignment model from the language Whisper *detected*, so
+# without this guard an ordinary upload in one of these languages silently pulls
+# the model onto the server and uses it. Alignment only refines timestamps to
+# word level: when it is skipped the transcript, its segment-level timings and
+# its speaker labels are all still produced (see _get_align_model).
+#
+# To allow one of these again, supply a licence-cleared replacement through
+# EXULU_ALIGN_MODEL_<LANG> rather than deleting the entry — e.g.
+# EXULU_ALIGN_MODEL_VI=my-org/licensed-vi-aligner.
+RESTRICTED_ALIGN_MODELS: dict[str, str] = {
+    "nguyenvulebinh/wav2vec2-base-vi":
+        "CC-BY-NC-4.0 — non-commercial use only",
+    "classla/wav2vec2-xls-r-parlaspeech-hr":
+        "no licence stated on the model card or in the Hugging Face metadata",
+    "imvladikon/wav2vec2-xls-r-300m-hebrew":
+        "no licence stated on the model card or in the Hugging Face metadata",
+    "theainerd/Wav2Vec2-large-xlsr-hindi":
+        "no licence stated on the model card or in the Hugging Face metadata",
+    # Danish is blocked on the conservative side: the model card states only
+    # that use "needs to adhere to this license from the Danish Parliament",
+    # and those terms have not been reviewed. Remove this entry once they have.
+    "saattrupdan/wav2vec2-xls-r-300m-ftspeech":
+        "licence is 'other' — refers to unreviewed Danish Parliament terms",
+}
+
+
+def _default_align_model_for(language_code: str) -> Optional[str]:
+    """The model id whisperx would resolve for this language, without loading it.
+
+    Mirrors load_align_model's own lookup order. Returns None for a language
+    whisperx has no default for, which it treats as an error anyway.
+    """
+    from whisperx.alignment import DEFAULT_ALIGN_MODELS_HF, DEFAULT_ALIGN_MODELS_TORCH
+
+    # The TORCH table holds torchaudio bundle names, which ship with torchaudio
+    # under BSD-2-Clause rather than being downloaded from Hugging Face.
+    if language_code in DEFAULT_ALIGN_MODELS_TORCH:
+        return DEFAULT_ALIGN_MODELS_TORCH[language_code]
+    return DEFAULT_ALIGN_MODELS_HF.get(language_code)
+
+
 def detect_device(requested: str = "auto") -> str:
     if requested != "auto":
         return requested
@@ -72,7 +117,9 @@ class TranscriptionPipeline:
         self.diarize_model = None
         self.diarization_enabled = False
         self.diarization_disabled_reason: str = "not attempted"
-        self.align_models: dict[str, tuple] = {}
+        self.align_models: dict[str, Optional[tuple]] = {}
+        # language_code -> why alignment was skipped, for observability
+        self.align_skipped_reasons: dict[str, str] = {}
 
     def load(self) -> None:
         # whisperx doesn't ship MPS support; run whisper on CPU when DEVICE=mps
@@ -122,12 +169,49 @@ class TranscriptionPipeline:
             print(f"[pipeline] Failed to load pyannote ({self.diarization_disabled_reason}); diarization disabled", flush=True)
 
     def _get_align_model(self, language_code: str):
-        if language_code not in self.align_models:
-            device = "cpu" if self.device == "mps" else self.device
-            print(f"[pipeline] Loading align model for {language_code}", flush=True)
-            self.align_models[language_code] = whisperx.load_align_model(
-                language_code=language_code, device=device
+        """Load the forced-alignment model for a language, or None if blocked.
+
+        Returns None when the model whisperx would use is not licence-cleared
+        and no replacement is configured. The caller skips alignment in that
+        case; nothing is downloaded, because the check runs before the load.
+        """
+        if language_code in self.align_models:
+            return self.align_models[language_code]
+
+        device = "cpu" if self.device == "mps" else self.device
+
+        # An operator-supplied replacement wins over whisperx's default, so a
+        # deployment that has licensed a model for one of the blocked languages
+        # can use it without patching this file.
+        override = os.getenv(f"EXULU_ALIGN_MODEL_{language_code.upper()}") or None
+        model_name = override or _default_align_model_for(language_code)
+
+        if override:
+            print(
+                f"[pipeline] Align model for {language_code} overridden by "
+                f"EXULU_ALIGN_MODEL_{language_code.upper()}={override}",
+                flush=True,
+            )
+        elif model_name in RESTRICTED_ALIGN_MODELS:
+            reason = RESTRICTED_ALIGN_MODELS[model_name]
+            self.align_skipped_reasons[language_code] = reason
+            print(
+                f"[pipeline] WARNING: word-level alignment skipped for "
+                f"'{language_code}'. Its default model '{model_name}' is not "
+                f"licence-cleared ({reason}) and was neither downloaded nor used. "
+                f"The transcript, segment timings and speaker labels are "
+                f"unaffected; only word-level timing precision is lost. Set "
+                f"EXULU_ALIGN_MODEL_{language_code.upper()} to a licensed model "
+                f"to re-enable alignment for this language.",
+                flush=True,
             )
+            self.align_models[language_code] = None
+            return None
+
+        print(f"[pipeline] Loading align model for {language_code}", flush=True)
+        self.align_models[language_code] = whisperx.load_align_model(
+            language_code=language_code, device=device, model_name=model_name
+        )
         return self.align_models[language_code]
 
     def transcribe(
@@ -170,15 +254,25 @@ class TranscriptionPipeline:
 
         language = transcribe_result["language"]
         align_device = "cpu" if self.device == "mps" else self.device
-        model_a, metadata = self._get_align_model(language)
-        aligned = whisperx.align(
-            transcribe_result["segments"],
-            model_a,
-            metadata,
-            audio,
-            align_device,
-            return_char_alignments=False,
-        )
+        align_bundle = self._get_align_model(language)
+        if align_bundle is None:
+            # Alignment blocked for this language (see RESTRICTED_ALIGN_MODELS).
+            # Whisper's own segments already carry start/end/text, and
+            # assign_word_speakers accepts an unaligned TranscriptionResult —
+            # it assigns speakers per segment and only walks words when a
+            # segment has them. So the transcript degrades to segment-level
+            # timing rather than failing.
+            aligned = transcribe_result
+        else:
+            model_a, metadata = align_bundle
+            aligned = whisperx.align(
+                transcribe_result["segments"],
+                model_a,
+                metadata,
+                audio,
+                align_device,
+                return_char_alignments=False,
+            )
 
         if is_cancelled():
             raise CancelledError()

package/ee/python/transcription/tests/test_align_model_licensing.py

@@ -0,0 +1,184 @@
+"""Tests for the forced-alignment licence guard in pipeline.py.
+
+The guard exists so that an ordinary upload in one of a handful of languages
+cannot pull a non-commercial or unlicensed wav2vec2 model onto the server.
+These tests assert the two things that matter: the blocked models are never
+loaded (so never downloaded), and a blocked language still produces a usable
+transcript.
+
+Run from the repo root with the venv active:
+    cd ee/python/transcription && ../.venv/bin/python -m pytest tests
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+# Make ee/python/transcription importable.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import pipeline as pl  # noqa: E402
+
+
+def _pipeline():
+    p = pl.TranscriptionPipeline("large-v3", "cpu", 4)
+    return p
+
+
+# --- the block list itself -------------------------------------------------
+
+
+def test_restricted_models_match_whisperx_defaults():
+    """Every blocked id must actually be a model whisperx would resolve.
+
+    Guards against a typo or an upstream rename silently disarming the guard.
+    """
+    from whisperx.alignment import DEFAULT_ALIGN_MODELS_HF
+
+    defaults = set(DEFAULT_ALIGN_MODELS_HF.values())
+    for model_id in pl.RESTRICTED_ALIGN_MODELS:
+        assert model_id in defaults, (
+            f"{model_id} is blocked but is no longer a whisperx default — "
+            "the upstream table changed and the guard may be stale"
+        )
+
+
+@pytest.mark.parametrize(
+    "language,expected_model",
+    [
+        ("vi", "nguyenvulebinh/wav2vec2-base-vi"),
+        ("hr", "classla/wav2vec2-xls-r-parlaspeech-hr"),
+        ("he", "imvladikon/wav2vec2-xls-r-300m-hebrew"),
+        ("hi", "theainerd/Wav2Vec2-large-xlsr-hindi"),
+        ("da", "saattrupdan/wav2vec2-xls-r-300m-ftspeech"),
+    ],
+)
+def test_restricted_languages_resolve_to_blocked_models(language, expected_model):
+    assert pl._default_align_model_for(language) == expected_model
+    assert expected_model in pl.RESTRICTED_ALIGN_MODELS
+
+
+# --- the guard -------------------------------------------------------------
+
+
+@pytest.mark.parametrize("language", ["vi", "hr", "he", "hi", "da"])
+def test_blocked_language_never_loads_a_model(language):
+    """The decisive test: load_align_model must not be called at all.
+
+    load_align_model is what reaches Hugging Face, so not calling it means the
+    weights are neither downloaded nor used.
+    """
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model") as load:
+        assert p._get_align_model(language) is None
+        load.assert_not_called()
+
+
+@pytest.mark.parametrize("language", ["vi", "hr", "he", "hi", "da"])
+def test_blocked_language_records_and_logs_a_reason(language, capsys):
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model"):
+        p._get_align_model(language)
+
+    assert language in p.align_skipped_reasons
+    out = capsys.readouterr().out
+    assert "WARNING" in out
+    assert "not licence-cleared" in out
+    assert pl.RESTRICTED_ALIGN_MODELS[pl._default_align_model_for(language)] in out
+
+
+def test_blocked_result_is_cached_so_the_warning_is_not_repeated(capsys):
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model") as load:
+        p._get_align_model("vi")
+        first = capsys.readouterr().out
+        p._get_align_model("vi")
+        second = capsys.readouterr().out
+        load.assert_not_called()
+
+    assert "WARNING" in first
+    assert second.strip() == ""
+
+
+# --- languages that are fine ------------------------------------------------
+
+
+@pytest.mark.parametrize("language", ["en", "de", "nl", "fr", "es"])
+def test_permitted_language_still_loads_normally(language):
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("model", "meta")) as load:
+        assert p._get_align_model(language) == ("model", "meta")
+        load.assert_called_once()
+        assert load.call_args.kwargs["language_code"] == language
+
+    assert p.align_skipped_reasons == {}
+
+
+def test_permitted_language_is_cached():
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("m", "meta")) as load:
+        p._get_align_model("en")
+        p._get_align_model("en")
+        load.assert_called_once()
+
+
+# --- operator override ------------------------------------------------------
+
+
+def test_override_re_enables_a_blocked_language(monkeypatch):
+    monkeypatch.setenv("EXULU_ALIGN_MODEL_VI", "my-org/licensed-vi-aligner")
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("m", "meta")) as load:
+        assert p._get_align_model("vi") == ("m", "meta")
+        assert load.call_args.kwargs["model_name"] == "my-org/licensed-vi-aligner"
+
+    assert p.align_skipped_reasons == {}
+
+
+def test_override_applies_to_a_permitted_language_too(monkeypatch):
+    monkeypatch.setenv("EXULU_ALIGN_MODEL_EN", "my-org/custom-en")
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("m", "meta")) as load:
+        p._get_align_model("en")
+        assert load.call_args.kwargs["model_name"] == "my-org/custom-en"
+
+
+def test_no_override_passes_the_whisperx_default_through(monkeypatch):
+    monkeypatch.delenv("EXULU_ALIGN_MODEL_EN", raising=False)
+    p = _pipeline()
+    with patch.object(pl.whisperx, "load_align_model", return_value=("m", "meta")) as load:
+        p._get_align_model("en")
+        assert load.call_args.kwargs["model_name"] == pl._default_align_model_for("en")
+
+
+# --- graceful degradation ---------------------------------------------------
+
+
+def test_unaligned_segments_carry_everything_the_output_needs():
+    """transcribe() reads start/end/text/speaker; Whisper's own segments have
+    the first three, so skipping alignment cannot break the response shape."""
+    unaligned = {
+        "segments": [{"start": 0.0, "end": 1.5, "text": "xin chao"}],
+        "language": "vi",
+    }
+    for seg in unaligned["segments"]:
+        assert "start" in seg and "end" in seg and "text" in seg
+
+
+def test_assign_word_speakers_accepts_unaligned_input():
+    """The diarization step must tolerate segments that have no 'words' key,
+    otherwise a blocked language would crash instead of degrading."""
+    import pandas as pd
+    from whisperx.diarize import assign_word_speakers
+
+    diarize_df = pd.DataFrame(
+        [{"start": 0.0, "end": 2.0, "speaker": "SPEAKER_00", "segment": None, "label": "a"}]
+    )
+    unaligned = {"segments": [{"start": 0.0, "end": 1.5, "text": "xin chao"}]}
+
+    result = assign_word_speakers(diarize_df, unaligned)
+
+    assert result["segments"][0]["speaker"] == "SPEAKER_00"
+    assert result["segments"][0]["text"] == "xin chao"

package/ee/workers.ts

@@ -4,6 +4,7 @@ import { guardRedisStartup, logRedisErrors } from "@EE/queues/redis-startup.ts";
 import { Job, Worker, type JobState } from "bullmq";
 import { bullmq } from "@SRC/validators/bullmq.ts";
 import { serializeError } from "@SRC/utils/serialize-error.ts";
+import { finishTurnMetadata } from "@SRC/exulu/turn-metadata.ts";
 import { getEnabledTools } from "@SRC/utils/enabled-tools.ts";
 import { ExuluStorage } from "@SRC/exulu/storage.ts";
 import type { ExuluAgent } from "@EXULU_TYPES/models/agent.ts";
@@ -1702,13 +1703,7 @@ export const processUiMessagesFlow = async ({
               messageMetadata: ({ part }) => {
                 console.log("[EXULU] part", part.type);
                 if (part.type === "finish") {
-                  return {
-                    totalTokens: part.totalUsage.totalTokens,
-                    reasoningTokens: part.totalUsage.reasoningTokens,
-                    inputTokens: part.totalUsage.inputTokens,
-                    outputTokens: part.totalUsage.outputTokens,
-                    cachedInputTokens: part.totalUsage.cachedInputTokens,
-                  };
+                  return finishTurnMetadata({ totalUsage: part.totalUsage, startedAt: startTime });
                 }
                 return undefined;
               },

package/license.md

@@ -1,4 +1,4 @@
-Copyright (c) 2023-2026 Qventu B.v.
+Copyright (c) 2023-2026 Qventu B.V.
 
 Certain portions of this software are licensed as described below:
 
@@ -102,4 +102,4 @@ these terms.
 
 **use** means anything you do with the software requiring one of your licenses.
 
-**trademark** means trademarks, service marks, and similar rights.
+**trademark** means trademarks, service marks, and similar rights.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
-  "author": "Qventu Bv.",
-  "version": "3.7.4",
+  "author": "Qventu B.V.",
+  "version": "4.0.0",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {
@@ -48,7 +48,7 @@
     "start:whisper": "node ./dist/cli/start-whisper.cjs",
     "python:setup": "cd ee/python && ./setup.sh",
     "python:install": "source ee/python/.venv/bin/activate && pip install -r ee/python/requirements.txt",
-    "python:validate": "source ee/python/.venv/bin/activate && python -c 'import docling; import transformers; print(\"✓ Python environment is working correctly\")'",
+    "python:validate": "source ee/python/.venv/bin/activate && python -c 'import pypdf; import transformers; print(\"✓ Python environment is working correctly\")'",
     "python:clean": "rm -rf ee/python/.venv ee/python/__pycache__ ee/python/**/__pycache__ ee/python/**/*.pyc",
     "python:rebuild": "npm run python:clean && npm run python:setup"
   },
@@ -143,7 +143,6 @@
     "jose": "^6.0.10",
     "json-schema-to-zod": "^2.6.1",
     "jsonwebtoken": "^9.0.2",
-    "just-bash": "^2.14.0",
     "knex": "^3.1.0",
     "link": "^2.1.1",
     "mailparser": "^3.9.14",

package/scripts/postinstall.cjs

@@ -8,7 +8,7 @@
 
 const { exec } = require('child_process');
 const { promisify } = require('util');
-const { existsSync } = require('fs');
+const { existsSync, rmSync } = require('fs');
 const { resolve, join } = require('path');
 
 const execAsync = promisify(exec);
@@ -82,6 +82,53 @@ async function setupPythonEnvironment() {
   }
 }
 
+/**
+ * Remove node-liblzma (LGPL-3.0) from the installed tree.
+ *
+ * It arrives as an OPTIONAL dependency of just-bash, which reaches us through
+ * bash-tool. just-bash is the only dependent. It is loaded lazily, by a dynamic
+ * import inside a try/catch, and only when a shell command uses xz compression;
+ * with it absent that path throws its own "xz compression requires
+ * node-liblzma" error and every other command is unaffected.
+ *
+ * Why not `npm install --omit=optional`: that flag is all-or-nothing, and 24 of
+ * the 91 runtime optional packages are the `@img/sharp-*` platform binaries.
+ * Omitting them leaves sharp unable to load, which breaks document processing.
+ * So the removal has to be targeted at this one package.
+ *
+ * Set EXULU_KEEP_NODE_LIBLZMA=true to keep it — for example if you want xz
+ * support and have satisfied yourself about LGPL-3.0 in your deployment.
+ *
+ * Best-effort by design: it walks up from this package looking for the hoisted
+ * copy, and silently does nothing if the layout differs (pnpm, yarn PnP) or the
+ * directory is not writable. Never fails the install.
+ */
+function removeNodeLiblzma() {
+  if (String(process.env.EXULU_KEEP_NODE_LIBLZMA).toLowerCase() === 'true') {
+    console.log(`${colors.yellow}⊘${colors.reset} Keeping node-liblzma (EXULU_KEEP_NODE_LIBLZMA=true) — LGPL-3.0`);
+    return;
+  }
+
+  // Walk up from this package looking for a hoisted node_modules/node-liblzma.
+  let dir = resolve(__dirname, '..');
+  for (let depth = 0; depth < 6; depth++) {
+    const candidate = join(dir, 'node_modules', 'node-liblzma');
+    if (existsSync(candidate)) {
+      try {
+        rmSync(candidate, { recursive: true, force: true });
+        console.log(`${colors.green}✓${colors.reset} Removed node-liblzma (LGPL-3.0; optional, xz compression only)`);
+      } catch (err) {
+        console.log(`${colors.yellow}!${colors.reset} Could not remove node-liblzma (LGPL-3.0): ${err.message}`);
+        console.log('  Remove it manually, or set EXULU_KEEP_NODE_LIBLZMA=true to keep it deliberately.');
+      }
+      return;
+    }
+    const parent = resolve(dir, '..');
+    if (parent === dir) break;
+    dir = parent;
+  }
+}
+
 /**
  * Main postinstall function
  */
@@ -92,6 +139,10 @@ async function main() {
   console.log(`${colors.blue}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${colors.reset}`);
   console.log('');
 
+  // Runs before the Python setup and before any early return below, so it
+  // happens even when SKIP_PYTHON_SETUP=1 or the venv already exists.
+  removeNodeLiblzma();
+
   // Check if we should skip setup
   if (shouldSkipSetup()) {
     console.log(`${colors.yellow}⊘${colors.reset} Skipping Python setup (SKIP_PYTHON_SETUP=1)`);