@exulu/backend 1.59.0 → 1.61.0

package/dist/{catalog-EOKGOHTY.js → catalog-BWE6SLE2.js}

@@ -2,7 +2,7 @@ import {
   __resetLiteLLMCatalogCacheForTesting,
   fetchLiteLLMCatalog,
   findLiteLLMModel
-} from "./chunk-YS27XOXI.js";
+} from "./chunk-ILAHW4UT.js";
 export {
   __resetLiteLLMCatalogCacheForTesting,
   fetchLiteLLMCatalog,

package/dist/chunk-IDHS2BZO.js

@@ -0,0 +1,210 @@
+// src/utils/python-setup.ts
+import { exec } from "child_process";
+import { promisify } from "util";
+import { resolve, join, dirname } from "path";
+import { existsSync, readFileSync } from "fs";
+import { fileURLToPath } from "url";
+var execAsync = promisify(exec);
+function getPackageRoot() {
+  const currentFile = fileURLToPath(import.meta.url);
+  let currentDir = dirname(currentFile);
+  let attempts = 0;
+  const maxAttempts = 10;
+  while (attempts < maxAttempts) {
+    const packageJsonPath = join(currentDir, "package.json");
+    if (existsSync(packageJsonPath)) {
+      try {
+        const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
+        if (packageJson.name === "@exulu/backend") {
+          return currentDir;
+        }
+      } catch {
+      }
+    }
+    const parentDir = resolve(currentDir, "..");
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+    attempts++;
+  }
+  const fallback = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+  return fallback;
+}
+function getSetupScriptPath(packageRoot) {
+  return resolve(packageRoot, "ee/python/setup.sh");
+}
+function getVenvPath(packageRoot) {
+  return resolve(packageRoot, "ee/python/.venv");
+}
+function isPythonEnvironmentSetup(packageRoot) {
+  const root = packageRoot ?? getPackageRoot();
+  const venvPath = getVenvPath(root);
+  const pythonPath = join(venvPath, "bin", "python");
+  return existsSync(venvPath) && existsSync(pythonPath);
+}
+async function setupPythonEnvironment(options = {}) {
+  const {
+    packageRoot = getPackageRoot(),
+    force = false,
+    verbose = false,
+    timeout = 6e5
+    // 10 minutes
+  } = options;
+  if (!force && isPythonEnvironmentSetup(packageRoot)) {
+    if (verbose) {
+      console.log("\u2713 Python environment already set up");
+    }
+    return {
+      success: true,
+      message: "Python environment already exists",
+      alreadyExists: true
+    };
+  }
+  const setupScriptPath = getSetupScriptPath(packageRoot);
+  if (!existsSync(setupScriptPath)) {
+    return {
+      success: false,
+      message: `Setup script not found at: ${setupScriptPath}`,
+      alreadyExists: false
+    };
+  }
+  try {
+    if (verbose) {
+      console.log("Setting up Python environment...");
+    }
+    const { stdout, stderr } = await execAsync(`bash "${setupScriptPath}"`, {
+      cwd: packageRoot,
+      timeout,
+      env: {
+        ...process.env,
+        // Ensure script can write to the directory
+        PYTHONDONTWRITEBYTECODE: "1"
+      },
+      maxBuffer: 10 * 1024 * 1024
+      // 10MB buffer
+    });
+    const output = stdout + stderr;
+    const versionMatch = output.match(/Python (\d+\.\d+\.\d+)/);
+    const pythonVersion = versionMatch ? versionMatch[1] : void 0;
+    if (verbose) {
+      console.log(output);
+    }
+    return {
+      success: true,
+      message: "Python environment set up successfully",
+      alreadyExists: false,
+      pythonVersion,
+      output
+    };
+  } catch (error) {
+    const errorOutput = error.stdout + error.stderr;
+    return {
+      success: false,
+      message: `Setup failed: ${error.message}`,
+      alreadyExists: false,
+      output: errorOutput
+    };
+  }
+}
+function getPythonSetupInstructions() {
+  return `
+Python environment not set up. Please run one of the following commands:
+
+Option 1 (Automatic):
+  import { setupPythonEnvironment } from '@exulu/backend';
+  await setupPythonEnvironment();
+
+Option 2 (Manual - for package consumers):
+  npx @exulu/backend setup-python
+
+Option 3 (Manual - for contributors):
+  npm run python:setup
+
+These commands will automatically create a Python virtual environment (.venv)
+in the @exulu/backend package and install all required dependencies.
+
+Requirements:
+  - Python 3.10 or higher must be installed
+  - pip must be available
+  - venv module must be available (for creating virtual environments)
+
+If Python dependencies are not installed, install them first, then run one of the commands above:
+  - macOS: brew install python@3.12
+  - Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
+  - Alpine Linux: apk add python3 py3-pip python3-dev
+  - Windows: Download from https://www.python.org/downloads/
+
+Note: In Docker containers, ensure you install all three components:
+  Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
+  Alpine: apk add python3 py3-pip python3-dev
+`.trim();
+}
+async function validatePythonEnvironment(packageRoot, checkPackages = true) {
+  const root = packageRoot ?? getPackageRoot();
+  const venvPath = getVenvPath(root);
+  const pythonPath = join(venvPath, "bin", "python");
+  if (!existsSync(venvPath)) {
+    return {
+      valid: false,
+      message: getPythonSetupInstructions()
+    };
+  }
+  if (!existsSync(pythonPath)) {
+    return {
+      valid: false,
+      message: "Python virtual environment is corrupted. Please run:\n  await setupPythonEnvironment({ force: true })"
+    };
+  }
+  try {
+    await execAsync(`"${pythonPath}" --version`, { cwd: root });
+  } catch {
+    return {
+      valid: false,
+      message: "Python executable is not working. Please run:\n  await setupPythonEnvironment({ force: true })"
+    };
+  }
+  if (checkPackages) {
+    const criticalPackages = ["docling", "transformers"];
+    const missingPackages = [];
+    for (const pkg of criticalPackages) {
+      try {
+        await execAsync(`"${pythonPath}" -c "import ${pkg}"`, {
+          cwd: root,
+          timeout: 1e4
+          // 10 second timeout per import check
+        });
+      } catch {
+        missingPackages.push(pkg);
+      }
+    }
+    if (missingPackages.length > 0) {
+      return {
+        valid: false,
+        message: `Python environment exists but required packages are not installed: ${missingPackages.join(", ")}
+
+This usually happens when:
+1. The .venv folder was copied but dependencies were not installed
+2. The package was installed via npm but setup script was not run
+
+Please run:
+  await setupPythonEnvironment({ force: true })
+
+Or manually run the setup script:
+  bash ` + getSetupScriptPath(root)
+      };
+    }
+  }
+  return {
+    valid: true,
+    message: "Python environment is valid"
+  };
+}
+
+export {
+  getPackageRoot,
+  isPythonEnvironmentSetup,
+  setupPythonEnvironment,
+  getPythonSetupInstructions,
+  validatePythonEnvironment
+};

package/dist/{chunk-YS27XOXI.js → chunk-ILAHW4UT.js}

@@ -40,7 +40,11 @@ var fetchLiteLLMCatalog = async () => {
       supports_vision: !!m.model_info?.supports_vision,
       supports_function_calling: !!m.model_info?.supports_function_calling,
       supports_pdf_input: !!m.model_info?.supports_pdf_input,
-      supports_audio_input: !!m.model_info?.supports_audio_input
+      supports_audio_input: !!m.model_info?.supports_audio_input,
+      sizes: Array.isArray(m.model_info?.sizes) ? m.model_info.sizes : null,
+      qualities: Array.isArray(m.model_info?.qualities) ? m.model_info.qualities : null,
+      supports_edit: !!m.model_info?.supports_edit,
+      max_n: typeof m.model_info?.max_n === "number" ? m.model_info.max_n : null
     }));
     _cache = { expiresAt: Date.now() + CACHE_TTL_MS, items };
     return items.filter((m) => m.type !== "speech_to_text" && m.type !== "text_to_speech");

package/dist/{chunk-U36VJDZ7.js → chunk-MPV7HBV6.js}

@@ -262,7 +262,8 @@ var spawnLiteLLM = (cfg) => {
   log(
     `Spawning LiteLLM: ${cfg.litellmBin} --config ${cfg.configPath} --port ${cfg.port} --host ${cfg.host}`
   );
-  const { DEBUG: _debug, ...envWithoutDebug } = process.env;
+  const { DEBUG: _debug, ...rest } = process.env;
+  const childEnv = { ...rest, DEBUG: "false" };
   const child = spawn(
     cfg.litellmBin,
     [
@@ -275,7 +276,7 @@ var spawnLiteLLM = (cfg) => {
     ],
     {
       stdio: ["ignore", "pipe", "pipe"],
-      env: envWithoutDebug
+      env: childEnv
     }
   );
   child.stdout?.on("data", (chunk) => {
@@ -743,7 +744,7 @@ var ExuluTool = class {
       });
       providerapikey = resolved.apiKey;
     }
-    const { convertExuluToolsToAiSdkTools: convertExuluToolsToAiSdkTools2 } = await import("./convert-exulu-tools-to-ai-sdk-tools-ZEECMX43.js");
+    const { convertExuluToolsToAiSdkTools: convertExuluToolsToAiSdkTools2 } = await import("./convert-exulu-tools-to-ai-sdk-tools-CULC37U6.js");
     const tools = await convertExuluToolsToAiSdkTools2(
       [this],
       [],
@@ -3167,6 +3168,10 @@ var usersSchema = {
       name: "anthropic_token",
       type: "text"
     },
+    {
+      name: "personal_system_prompt",
+      type: "longText"
+    },
     {
       name: "role",
       type: "uuid"
@@ -3175,6 +3180,7 @@ var usersSchema = {
 };
 var platformConfigurationsSchema = {
   type: "platform_configurations",
+  RBAC: true,
   name: {
     plural: "platform_configurations",
     singular: "platform_configuration"
@@ -3294,6 +3300,60 @@ var promptFavoritesSchema = {
     }
   ]
 };
+var transcriptionJobsSchema = {
+  type: "transcription_jobs",
+  name: {
+    plural: "transcription_jobs",
+    singular: "transcription_job"
+  },
+  RBAC: true,
+  fields: [
+    { name: "audio", type: "file" },
+    { name: "title", type: "text" },
+    { name: "status", type: "text", index: true },
+    { name: "whisper_job_id", type: "text" },
+    { name: "raw_segments", type: "json" },
+    { name: "speakers", type: "json" },
+    { name: "language", type: "text" },
+    { name: "duration_seconds", type: "number" },
+    { name: "project_id", type: "uuid", required: false },
+    { name: "target_rights_mode", type: "text", default: "private" },
+    { name: "target_rbac_users", type: "json" },
+    { name: "target_rbac_roles", type: "json" },
+    { name: "saved_item_id", type: "uuid", required: false },
+    { name: "error", type: "text" }
+  ]
+};
+var imageGenerationsSchema = {
+  type: "image_generations",
+  name: {
+    plural: "image_generations",
+    singular: "image_generation"
+  },
+  // Access is gated by the parent agent_sessions RBAC — rows have no
+  // independent visibility, so no row-level RBAC fields are needed here.
+  RBAC: false,
+  fields: [
+    { name: "session_id", type: "uuid", required: true, index: true },
+    { name: "tool_call_id", type: "text", required: true, index: true },
+    { name: "user_id", type: "number", required: true, index: true },
+    { name: "operation", type: "text", required: true },
+    // 'generate' | 'edit'
+    { name: "model", type: "text", required: true },
+    { name: "prompt", type: "longText", required: true },
+    { name: "applied_style_id", type: "uuid", required: false },
+    { name: "applied_style_markdown", type: "longText", required: false },
+    { name: "size", type: "text", required: false },
+    { name: "quality", type: "text", required: false },
+    { name: "n", type: "number", default: 1 },
+    { name: "reference_image_keys", type: "json", required: false },
+    { name: "mask_image_key", type: "text", required: false },
+    { name: "image_keys", type: "json", required: true },
+    { name: "revised_prompts", type: "json", required: false },
+    { name: "selected", type: "boolean", default: false },
+    { name: "error", type: "text", required: false }
+  ]
+};
 var contextPresetsSchema = {
   type: "context_presets",
   name: {
@@ -3384,7 +3444,9 @@ var coreSchemas = {
       promptLibrarySchema: () => addCoreFields(promptLibrarySchema),
       embedderSettingsSchema: () => addCoreFields(embedderSettingsSchema),
       promptFavoritesSchema: () => addCoreFields(promptFavoritesSchema),
-      contextPresetsSchema: () => addCoreFields(contextPresetsSchema)
+      contextPresetsSchema: () => addCoreFields(contextPresetsSchema),
+      transcriptionJobsSchema: () => addCoreFields(transcriptionJobsSchema),
+      imageGenerationsSchema: () => addCoreFields(imageGenerationsSchema)
     };
     if (license["agent-feedback"]) {
       schemas.feedbackSchema = () => addCoreFields(feedbackSchema);

package/dist/cli/start-whisper.cjs

@@ -0,0 +1,240 @@
+#!/usr/bin/env node
+"use strict";
+
+// node_modules/tsup/assets/cjs_shims.js
+var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.src || new URL("main.js", document.baseURI).href;
+var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
+
+// src/cli/start-whisper.ts
+var import_config = require("dotenv/config");
+
+// src/utils/python-setup.ts
+var import_child_process = require("child_process");
+var import_util = require("util");
+var import_path = require("path");
+var import_fs = require("fs");
+var import_url = require("url");
+var execAsync = (0, import_util.promisify)(import_child_process.exec);
+function getPackageRoot() {
+  const currentFile = (0, import_url.fileURLToPath)(importMetaUrl);
+  let currentDir = (0, import_path.dirname)(currentFile);
+  let attempts = 0;
+  const maxAttempts = 10;
+  while (attempts < maxAttempts) {
+    const packageJsonPath = (0, import_path.join)(currentDir, "package.json");
+    if ((0, import_fs.existsSync)(packageJsonPath)) {
+      try {
+        const packageJson = JSON.parse((0, import_fs.readFileSync)(packageJsonPath, "utf-8"));
+        if (packageJson.name === "@exulu/backend") {
+          return currentDir;
+        }
+      } catch {
+      }
+    }
+    const parentDir = (0, import_path.resolve)(currentDir, "..");
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+    attempts++;
+  }
+  const fallback = (0, import_path.resolve)((0, import_path.dirname)((0, import_url.fileURLToPath)(importMetaUrl)), "../..");
+  return fallback;
+}
+
+// src/exulu/transcription/supervisor.ts
+var import_node_child_process = require("child_process");
+var import_node_fs = require("fs");
+var import_node_path = require("path");
+var MAX_CRASHES = 5;
+var INITIAL_BACKOFF_MS = 1e3;
+var MAX_BACKOFF_MS = 3e4;
+var READY_TIMEOUT_MS = 30 * 6e4;
+var READY_POLL_INTERVAL_MS = 500;
+var SHUTDOWN_GRACE_MS = 5e3;
+var internal = {
+  child: void 0,
+  state: "idle",
+  crashCount: 0,
+  backoffMs: INITIAL_BACKOFF_MS,
+  readyPromise: void 0,
+  shutdownRequested: false
+};
+var resolveConfig = (packageRoot) => {
+  const host = process.env.WHISPER_HOST ?? "127.0.0.1";
+  const port = process.env.WHISPER_PORT ?? "9876";
+  const venvBin = (0, import_node_path.resolve)(packageRoot, "ee/python/.venv/bin");
+  const venvPython = (0, import_node_path.resolve)(venvBin, "python");
+  const cwd = (0, import_node_path.resolve)(packageRoot, "ee/python/transcription");
+  return { host, port, venvBin, venvPython, cwd };
+};
+var log = (line) => {
+  const text = line.startsWith("[EXULU-WHISPER]") ? line : `[EXULU-WHISPER] ${line}`;
+  console.log(text);
+};
+var pollHealth = async (host, port) => {
+  const url = `http://${host}:${port}/healthz`;
+  const deadline = Date.now() + READY_TIMEOUT_MS;
+  while (Date.now() < deadline) {
+    try {
+      const res = await fetch(url, { method: "GET" });
+      if (res.ok) return;
+    } catch {
+    }
+    await new Promise((r) => setTimeout(r, READY_POLL_INTERVAL_MS));
+  }
+  throw new Error(
+    `Whisper server did not become ready at ${url} within ${READY_TIMEOUT_MS}ms`
+  );
+};
+var spawnWhisper = (cfg) => {
+  log(
+    `Spawning: ${cfg.venvPython} -m uvicorn server:app --host ${cfg.host} --port ${cfg.port}`
+  );
+  const child = (0, import_node_child_process.spawn)(
+    cfg.venvPython,
+    [
+      "-m",
+      "uvicorn",
+      "server:app",
+      "--host",
+      cfg.host,
+      "--port",
+      cfg.port,
+      "--log-level",
+      "info"
+    ],
+    {
+      cwd: cfg.cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+      env: process.env
+    }
+  );
+  child.stdout?.on("data", (chunk) => {
+    chunk.toString().split("\n").filter((l) => l.length > 0).forEach((l) => log(l));
+  });
+  child.stderr?.on("data", (chunk) => {
+    chunk.toString().split("\n").filter((l) => l.length > 0).forEach((l) => log(`stderr: ${l}`));
+  });
+  return child;
+};
+var supervise = async (cfg) => {
+  while (!internal.shutdownRequested && internal.crashCount < MAX_CRASHES) {
+    internal.state = internal.crashCount === 0 ? "starting" : "respawning";
+    internal.child = spawnWhisper(cfg);
+    const exitPromise = new Promise((resolveFn) => {
+      internal.child.on("exit", (code2) => resolveFn(code2));
+    });
+    try {
+      await Promise.race([
+        pollHealth(cfg.host, cfg.port).then(() => "ready"),
+        exitPromise.then((code2) => ({ exited: code2 }))
+      ]);
+    } catch (err) {
+      log(`Readiness probe failed: ${err.message}`);
+      try {
+        internal.child?.kill("SIGTERM");
+      } catch {
+      }
+    }
+    if (!internal.child?.killed && internal.child?.exitCode === null) {
+      internal.state = "ready";
+      internal.crashCount = 0;
+      internal.backoffMs = INITIAL_BACKOFF_MS;
+      log("Whisper server is ready.");
+    }
+    const code = await exitPromise;
+    internal.state = "respawning";
+    internal.child = void 0;
+    if (internal.shutdownRequested) {
+      log("Child exited during shutdown; supervisor stopping.");
+      internal.state = "stopped";
+      return;
+    }
+    internal.crashCount += 1;
+    log(
+      `Whisper server exited (code=${code}). Crash ${internal.crashCount}/${MAX_CRASHES}. Respawning in ${internal.backoffMs}ms.`
+    );
+    if (internal.crashCount >= MAX_CRASHES) {
+      log(
+        "Whisper server keeps crashing \u2014 fix the install (try `npx @exulu/backend setup-python --force`) and re-run `npx @exulu/backend exulu-start-whisper`. Giving up."
+      );
+      internal.state = "given_up";
+      return;
+    }
+    await new Promise((r) => setTimeout(r, internal.backoffMs));
+    internal.backoffMs = Math.min(internal.backoffMs * 2, MAX_BACKOFF_MS);
+  }
+};
+var startWhisperSupervisor = async (options) => {
+  if (internal.readyPromise) {
+    return internal.readyPromise;
+  }
+  const cfg = resolveConfig(options.packageRoot);
+  if (!(0, import_node_fs.existsSync)(cfg.venvPython)) {
+    throw new Error(
+      `Whisper supervisor: Python venv not found at ${cfg.venvPython}. Run \`npx @exulu/backend setup-python\` first.`
+    );
+  }
+  if (!(0, import_node_fs.existsSync)(cfg.cwd)) {
+    throw new Error(
+      `Whisper supervisor: transcription scripts not found at ${cfg.cwd}. The @exulu/backend package may be corrupt; reinstall it.`
+    );
+  }
+  internal.readyPromise = (async () => {
+    supervise(cfg);
+    const deadline = Date.now() + READY_TIMEOUT_MS + 5e3;
+    while (Date.now() < deadline) {
+      if (internal.state === "ready") return;
+      if (internal.state === "given_up") {
+        throw new Error("Whisper supervisor gave up before becoming ready.");
+      }
+      await new Promise((r) => setTimeout(r, READY_POLL_INTERVAL_MS));
+    }
+    throw new Error("Timed out waiting for whisper supervisor readiness.");
+  })();
+  registerShutdownHandlers();
+  return internal.readyPromise;
+};
+var stopWhisper = (signal = "SIGTERM") => {
+  internal.shutdownRequested = true;
+  const child = internal.child;
+  if (!child) return;
+  try {
+    child.kill(signal);
+  } catch {
+  }
+  setTimeout(() => {
+    try {
+      if (!child.killed && child.exitCode === null) {
+        child.kill("SIGKILL");
+      }
+    } catch {
+    }
+  }, SHUTDOWN_GRACE_MS).unref();
+};
+var shutdownHandlersRegistered = false;
+var registerShutdownHandlers = () => {
+  if (shutdownHandlersRegistered) return;
+  shutdownHandlersRegistered = true;
+  process.on("SIGINT", () => stopWhisper("SIGTERM"));
+  process.on("SIGTERM", () => stopWhisper("SIGTERM"));
+  process.on("exit", () => stopWhisper("SIGTERM"));
+};
+
+// src/cli/start-whisper.ts
+var main = async () => {
+  const packageRoot = getPackageRoot();
+  try {
+    await startWhisperSupervisor({ packageRoot });
+  } catch (err) {
+    console.error(`[EXULU-WHISPER] ${err.message}`);
+    process.exit(1);
+  }
+  await new Promise(() => {
+  });
+};
+main().catch((err) => {
+  console.error(`[EXULU-WHISPER] Unexpected error: ${err.stack ?? err}`);
+  process.exit(1);
+});

package/dist/cli/start-whisper.d.cts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli/start-whisper.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node