@exulu/backend 1.59.0 → 1.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -429,7 +429,8 @@ var init_supervisor = __esm({
429
429
  log(
430
430
  `Spawning LiteLLM: ${cfg.litellmBin} --config ${cfg.configPath} --port ${cfg.port} --host ${cfg.host}`
431
431
  );
432
- const { DEBUG: _debug, ...envWithoutDebug } = process.env;
432
+ const { DEBUG: _debug, ...rest } = process.env;
433
+ const childEnv = { ...rest, DEBUG: "false" };
433
434
  const child = (0, import_node_child_process.spawn)(
434
435
  cfg.litellmBin,
435
436
  [
@@ -442,7 +443,7 @@ var init_supervisor = __esm({
442
443
  ],
443
444
  {
444
445
  stdio: ["ignore", "pipe", "pipe"],
445
- env: envWithoutDebug
446
+ env: childEnv
446
447
  }
447
448
  );
448
449
  child.stdout?.on("data", (chunk) => {
@@ -605,15 +606,15 @@ var init_check_record_access = __esm({
605
606
  "use strict";
606
607
  init_cjs_shims();
607
608
  checkRecordAccessCache = /* @__PURE__ */ new Map();
608
- checkRecordAccess = async (record, request, user) => {
609
+ checkRecordAccess = async (record, request2, user) => {
609
610
  const setRecordAccessCache = (hasAccess2) => {
610
- checkRecordAccessCache.set(`${record.id}-${request}-${user?.id}`, {
611
+ checkRecordAccessCache.set(`${record.id}-${request2}-${user?.id}`, {
611
612
  hasAccess: hasAccess2,
612
613
  expiresAt: new Date(Date.now() + 1e3 * 60 * 1)
613
614
  // 1 minute
614
615
  });
615
616
  };
616
- const cachedAccess = checkRecordAccessCache.get(`${record.id}-${request}-${user?.id}`);
617
+ const cachedAccess = checkRecordAccessCache.get(`${record.id}-${request2}-${user?.id}`);
617
618
  if (cachedAccess && cachedAccess.expiresAt > /* @__PURE__ */ new Date()) {
618
619
  return cachedAccess.hasAccess;
619
620
  }
@@ -625,7 +626,7 @@ var init_check_record_access = __esm({
625
626
  const isAdmin = user ? user.super_admin : false;
626
627
  const isApi = user ? user.type === "api" : false;
627
628
  const isAdminApi = isApi && (!user.scope_mode || user.scope_mode === "admin");
628
- const isAgentsScopedApi = isApi && user.scope_mode === "agents" && request === "read" && Array.isArray(user.agent_ids) && user.agent_ids.includes(String(record.id));
629
+ const isAgentsScopedApi = isApi && user.scope_mode === "agents" && request2 === "read" && Array.isArray(user.agent_ids) && user.agent_ids.includes(String(record.id));
629
630
  let hasAccess = "none";
630
631
  if (isPublic || isCreator || isAdmin || isAdminApi || isAgentsScopedApi) {
631
632
  setRecordAccessCache(true);
@@ -637,7 +638,7 @@ var init_check_record_access = __esm({
637
638
  return false;
638
639
  }
639
640
  hasAccess = record.RBAC?.users?.find((x) => x.id === user.id)?.rights || "none";
640
- if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
641
+ if (!hasAccess || hasAccess === "none" || hasAccess !== request2) {
641
642
  console.error(
642
643
  `[EXULU] Your current user ${user.id} does not have access to this record, current access type is: ${hasAccess}.`
643
644
  );
@@ -654,7 +655,7 @@ var init_check_record_access = __esm({
654
655
  return false;
655
656
  }
656
657
  hasAccess = record.RBAC?.roles?.find((x) => x.id === user.role?.id)?.rights || "none";
657
- if (!hasAccess || hasAccess === "none" || hasAccess !== request) {
658
+ if (!hasAccess || hasAccess === "none" || hasAccess !== request2) {
658
659
  console.error(
659
660
  `[EXULU] Your current role ${user.role?.name} does not have access to this record, current access type is: ${hasAccess}.`
660
661
  );
@@ -1519,7 +1520,7 @@ var init_uppy = __esm({
1519
1520
  if (error.name === "SignatureDoesNotMatch" || error.name === "InvalidAccessKeyId" || error.name === "AccessDenied") {
1520
1521
  if (attempt < maxRetries) {
1521
1522
  const backoffMs = Math.pow(2, attempt) * 1e3;
1522
- await new Promise((resolve6) => setTimeout(resolve6, backoffMs));
1523
+ await new Promise((resolve7) => setTimeout(resolve7, backoffMs));
1523
1524
  s3Client = void 0;
1524
1525
  getS3Client(config);
1525
1526
  continue;
@@ -3290,7 +3291,7 @@ async function withRetry(generateFn, maxRetries = 3) {
3290
3291
  if (attempt === maxRetries) {
3291
3292
  throw error;
3292
3293
  }
3293
- await new Promise((resolve6) => setTimeout(resolve6, Math.pow(2, attempt) * 1e3));
3294
+ await new Promise((resolve7) => setTimeout(resolve7, Math.pow(2, attempt) * 1e3));
3294
3295
  }
3295
3296
  }
3296
3297
  throw lastError;
@@ -4086,7 +4087,7 @@ var init_schemas = __esm({
4086
4087
  });
4087
4088
 
4088
4089
  // src/postgres/core-schema.ts
4089
- var agentMessagesSchema, agentSessionsSchema, skillsSchema, variablesSchema, projectsSchema, agentsSchema, modelsSchema, usersSchema, platformConfigurationsSchema, embedderSettingsSchema, promptLibrarySchema, promptFavoritesSchema, contextPresetsSchema, addCoreFields, coreSchemas;
4090
+ var agentMessagesSchema, agentSessionsSchema, skillsSchema, variablesSchema, projectsSchema, agentsSchema, modelsSchema, usersSchema, platformConfigurationsSchema, embedderSettingsSchema, promptLibrarySchema, promptFavoritesSchema, transcriptionJobsSchema, imageGenerationsSchema, contextPresetsSchema, addCoreFields, coreSchemas;
4090
4091
  var init_core_schema = __esm({
4091
4092
  "src/postgres/core-schema.ts"() {
4092
4093
  "use strict";
@@ -4499,6 +4500,10 @@ var init_core_schema = __esm({
4499
4500
  name: "anthropic_token",
4500
4501
  type: "text"
4501
4502
  },
4503
+ {
4504
+ name: "personal_system_prompt",
4505
+ type: "longText"
4506
+ },
4502
4507
  {
4503
4508
  name: "role",
4504
4509
  type: "uuid"
@@ -4507,6 +4512,7 @@ var init_core_schema = __esm({
4507
4512
  };
4508
4513
  platformConfigurationsSchema = {
4509
4514
  type: "platform_configurations",
4515
+ RBAC: true,
4510
4516
  name: {
4511
4517
  plural: "platform_configurations",
4512
4518
  singular: "platform_configuration"
@@ -4626,6 +4632,60 @@ var init_core_schema = __esm({
4626
4632
  }
4627
4633
  ]
4628
4634
  };
4635
+ transcriptionJobsSchema = {
4636
+ type: "transcription_jobs",
4637
+ name: {
4638
+ plural: "transcription_jobs",
4639
+ singular: "transcription_job"
4640
+ },
4641
+ RBAC: true,
4642
+ fields: [
4643
+ { name: "audio", type: "file" },
4644
+ { name: "title", type: "text" },
4645
+ { name: "status", type: "text", index: true },
4646
+ { name: "whisper_job_id", type: "text" },
4647
+ { name: "raw_segments", type: "json" },
4648
+ { name: "speakers", type: "json" },
4649
+ { name: "language", type: "text" },
4650
+ { name: "duration_seconds", type: "number" },
4651
+ { name: "project_id", type: "uuid", required: false },
4652
+ { name: "target_rights_mode", type: "text", default: "private" },
4653
+ { name: "target_rbac_users", type: "json" },
4654
+ { name: "target_rbac_roles", type: "json" },
4655
+ { name: "saved_item_id", type: "uuid", required: false },
4656
+ { name: "error", type: "text" }
4657
+ ]
4658
+ };
4659
+ imageGenerationsSchema = {
4660
+ type: "image_generations",
4661
+ name: {
4662
+ plural: "image_generations",
4663
+ singular: "image_generation"
4664
+ },
4665
+ // Access is gated by the parent agent_sessions RBAC — rows have no
4666
+ // independent visibility, so no row-level RBAC fields are needed here.
4667
+ RBAC: false,
4668
+ fields: [
4669
+ { name: "session_id", type: "uuid", required: true, index: true },
4670
+ { name: "tool_call_id", type: "text", required: true, index: true },
4671
+ { name: "user_id", type: "number", required: true, index: true },
4672
+ { name: "operation", type: "text", required: true },
4673
+ // 'generate' | 'edit'
4674
+ { name: "model", type: "text", required: true },
4675
+ { name: "prompt", type: "longText", required: true },
4676
+ { name: "applied_style_id", type: "uuid", required: false },
4677
+ { name: "applied_style_markdown", type: "longText", required: false },
4678
+ { name: "size", type: "text", required: false },
4679
+ { name: "quality", type: "text", required: false },
4680
+ { name: "n", type: "number", default: 1 },
4681
+ { name: "reference_image_keys", type: "json", required: false },
4682
+ { name: "mask_image_key", type: "text", required: false },
4683
+ { name: "image_keys", type: "json", required: true },
4684
+ { name: "revised_prompts", type: "json", required: false },
4685
+ { name: "selected", type: "boolean", default: false },
4686
+ { name: "error", type: "text", required: false }
4687
+ ]
4688
+ };
4629
4689
  contextPresetsSchema = {
4630
4690
  type: "context_presets",
4631
4691
  name: {
@@ -4716,7 +4776,9 @@ var init_core_schema = __esm({
4716
4776
  promptLibrarySchema: () => addCoreFields(promptLibrarySchema),
4717
4777
  embedderSettingsSchema: () => addCoreFields(embedderSettingsSchema),
4718
4778
  promptFavoritesSchema: () => addCoreFields(promptFavoritesSchema),
4719
- contextPresetsSchema: () => addCoreFields(contextPresetsSchema)
4779
+ contextPresetsSchema: () => addCoreFields(contextPresetsSchema),
4780
+ transcriptionJobsSchema: () => addCoreFields(transcriptionJobsSchema),
4781
+ imageGenerationsSchema: () => addCoreFields(imageGenerationsSchema)
4720
4782
  };
4721
4783
  if (license["agent-feedback"]) {
4722
4784
  schemas.feedbackSchema = () => addCoreFields(feedbackSchema);
@@ -7618,7 +7680,11 @@ var init_catalog = __esm({
7618
7680
  supports_vision: !!m.model_info?.supports_vision,
7619
7681
  supports_function_calling: !!m.model_info?.supports_function_calling,
7620
7682
  supports_pdf_input: !!m.model_info?.supports_pdf_input,
7621
- supports_audio_input: !!m.model_info?.supports_audio_input
7683
+ supports_audio_input: !!m.model_info?.supports_audio_input,
7684
+ sizes: Array.isArray(m.model_info?.sizes) ? m.model_info.sizes : null,
7685
+ qualities: Array.isArray(m.model_info?.qualities) ? m.model_info.qualities : null,
7686
+ supports_edit: !!m.model_info?.supports_edit,
7687
+ max_n: typeof m.model_info?.max_n === "number" ? m.model_info.max_n : null
7622
7688
  }));
7623
7689
  _cache = { expiresAt: Date.now() + CACHE_TTL_MS2, items };
7624
7690
  return items.filter((m) => m.type !== "speech_to_text" && m.type !== "text_to_speech");
@@ -10021,7 +10087,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
10021
10087
  );
10022
10088
  if (attempt < retries) {
10023
10089
  const backoffMs = 500 * Math.pow(2, attempt - 1);
10024
- await new Promise((resolve6) => setTimeout(resolve6, backoffMs));
10090
+ await new Promise((resolve7) => setTimeout(resolve7, backoffMs));
10025
10091
  }
10026
10092
  }
10027
10093
  }
@@ -10225,7 +10291,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
10225
10291
  } = await validateWorkflowPayload(data, providers);
10226
10292
  const retries2 = 3;
10227
10293
  let attempts = 0;
10228
- const promise = new Promise(async (resolve6, reject) => {
10294
+ const promise = new Promise(async (resolve7, reject) => {
10229
10295
  while (attempts < retries2) {
10230
10296
  try {
10231
10297
  const messages2 = await processUiMessagesFlow({
@@ -10240,7 +10306,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
10240
10306
  config,
10241
10307
  variables: data.inputs
10242
10308
  });
10243
- resolve6(messages2);
10309
+ resolve7(messages2);
10244
10310
  break;
10245
10311
  } catch (error) {
10246
10312
  console.error(
@@ -10251,7 +10317,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
10251
10317
  if (attempts >= retries2) {
10252
10318
  reject(new Error(error instanceof Error ? error.message : String(error)));
10253
10319
  }
10254
- await new Promise((resolve7) => setTimeout((resolve8) => resolve8(true), 2e3));
10320
+ await new Promise((resolve8) => setTimeout((resolve9) => resolve9(true), 2e3));
10255
10321
  }
10256
10322
  }
10257
10323
  });
@@ -10301,7 +10367,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
10301
10367
  } = await validateEvalPayload(data, providers);
10302
10368
  const retries2 = 3;
10303
10369
  let attempts = 0;
10304
- const promise = new Promise(async (resolve6, reject) => {
10370
+ const promise = new Promise(async (resolve7, reject) => {
10305
10371
  while (attempts < retries2) {
10306
10372
  try {
10307
10373
  const messages2 = await processUiMessagesFlow({
@@ -10315,7 +10381,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
10315
10381
  tools,
10316
10382
  config
10317
10383
  });
10318
- resolve6(messages2);
10384
+ resolve7(messages2);
10319
10385
  break;
10320
10386
  } catch (error) {
10321
10387
  console.error(
@@ -10326,7 +10392,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
10326
10392
  if (attempts >= retries2) {
10327
10393
  reject(new Error(error instanceof Error ? error.message : String(error)));
10328
10394
  }
10329
- await new Promise((resolve7) => setTimeout((resolve8) => resolve8(true), 2e3));
10395
+ await new Promise((resolve8) => setTimeout((resolve9) => resolve9(true), 2e3));
10330
10396
  }
10331
10397
  }
10332
10398
  });
@@ -10801,7 +10867,7 @@ var pollJobResult = async ({
10801
10867
  attempts++;
10802
10868
  const job = await import_bullmq3.Job.fromId(queue.queue, jobId);
10803
10869
  if (!job) {
10804
- await new Promise((resolve6) => setTimeout((resolve7) => resolve7(true), 2e3));
10870
+ await new Promise((resolve7) => setTimeout((resolve8) => resolve8(true), 2e3));
10805
10871
  continue;
10806
10872
  }
10807
10873
  const elapsedTime = Date.now() - startTime;
@@ -10831,7 +10897,7 @@ var pollJobResult = async ({
10831
10897
  console.log(`[EXULU] eval function ${job.id} result: ${result}`);
10832
10898
  break;
10833
10899
  }
10834
- await new Promise((resolve6) => setTimeout(() => resolve6(true), 2e3));
10900
+ await new Promise((resolve7) => setTimeout(() => resolve7(true), 2e3));
10835
10901
  }
10836
10902
  return result;
10837
10903
  };
@@ -10931,7 +10997,7 @@ var processUiMessagesFlow = async ({
10931
10997
  label: agent.name,
10932
10998
  trigger: "agent"
10933
10999
  };
10934
- messageHistory = await new Promise(async (resolve6, reject) => {
11000
+ messageHistory = await new Promise(async (resolve7, reject) => {
10935
11001
  const startTime = Date.now();
10936
11002
  try {
10937
11003
  const result = await provider.generateStream({
@@ -11009,7 +11075,7 @@ var processUiMessagesFlow = async ({
11009
11075
  })
11010
11076
  ] : []
11011
11077
  ]);
11012
- resolve6({
11078
+ resolve7({
11013
11079
  messages,
11014
11080
  metadata: {
11015
11081
  tokens: {
@@ -11065,6 +11131,373 @@ function getAverage(arr) {
11065
11131
  // src/graphql/schemas/index.ts
11066
11132
  init_entitlements();
11067
11133
  var import_fs = require("fs");
11134
+
11135
+ // src/exulu/transcription/service.ts
11136
+ init_cjs_shims();
11137
+ init_singleton();
11138
+ init_client();
11139
+ init_uppy();
11140
+
11141
+ // src/exulu/transcription/client.ts
11142
+ init_cjs_shims();
11143
+ var TranscriptionServerUnavailable = class extends Error {
11144
+ constructor(message) {
11145
+ super(message);
11146
+ this.name = "TranscriptionServerUnavailable";
11147
+ }
11148
+ };
11149
+ var getBaseUrl = () => {
11150
+ const url = process.env.TRANSCRIPTION_SERVER;
11151
+ if (!url) {
11152
+ throw new TranscriptionServerUnavailable(
11153
+ "TRANSCRIPTION_SERVER env var is not set. Start a whisper server with `npx @exulu/backend exulu-start-whisper` and point TRANSCRIPTION_SERVER at it."
11154
+ );
11155
+ }
11156
+ return url.replace(/\/$/, "");
11157
+ };
11158
+ var request = async (path3, init = {}) => {
11159
+ const url = `${getBaseUrl()}${path3}`;
11160
+ let res;
11161
+ try {
11162
+ res = await fetch(url, init);
11163
+ } catch (err) {
11164
+ throw new TranscriptionServerUnavailable(
11165
+ `Unable to reach whisper server at ${url}: ${err.message}`
11166
+ );
11167
+ }
11168
+ if (res.status === 404) {
11169
+ const err = new Error(`whisper server returned 404 for ${path3}`);
11170
+ err.code = "JOB_NOT_FOUND";
11171
+ throw err;
11172
+ }
11173
+ if (!res.ok) {
11174
+ throw new Error(
11175
+ `whisper server returned ${res.status} for ${path3}: ${await res.text()}`
11176
+ );
11177
+ }
11178
+ return await res.json();
11179
+ };
11180
+ var transcriptionClient = {
11181
+ submitJob: (opts) => request("/jobs", {
11182
+ method: "POST",
11183
+ headers: { "content-type": "application/json" },
11184
+ body: JSON.stringify(opts)
11185
+ }),
11186
+ getJob: (jobId) => request(`/jobs/${jobId}`),
11187
+ cancelJob: (jobId) => request(`/jobs/${jobId}`, {
11188
+ method: "DELETE"
11189
+ }),
11190
+ health: () => request("/healthz"),
11191
+ isConfigured: () => Boolean(process.env.TRANSCRIPTION_SERVER)
11192
+ };
11193
+
11194
+ // src/exulu/transcription/transcript-text.ts
11195
+ init_cjs_shims();
11196
+ var renderTranscript = (segments, speakers) => {
11197
+ if (!segments || segments.length === 0) return "";
11198
+ const blocks = [];
11199
+ for (const seg of segments) {
11200
+ const text = (seg.text ?? "").trim();
11201
+ if (!text) continue;
11202
+ const label = speakers[seg.speaker] ?? seg.speaker ?? "unknown";
11203
+ const last = blocks[blocks.length - 1];
11204
+ if (last && last.speaker === label) {
11205
+ last.text = `${last.text} ${text}`.trim();
11206
+ } else {
11207
+ blocks.push({ speaker: label, text });
11208
+ }
11209
+ }
11210
+ return blocks.map((b) => `${b.speaker}: ${b.text}`).join("\n");
11211
+ };
11212
+
11213
+ // src/exulu/transcription/service.ts
11214
+ var TABLE = "transcription_jobs";
11215
+ var log2 = (msg) => console.log(`[EXULU-TRANSCRIPTION] ${msg}`);
11216
+ var parseJsonField = (v) => {
11217
+ if (v == null) return null;
11218
+ if (typeof v === "string") {
11219
+ try {
11220
+ return JSON.parse(v);
11221
+ } catch {
11222
+ return null;
11223
+ }
11224
+ }
11225
+ return v;
11226
+ };
11227
+ var presignAudio = async (s3Key) => {
11228
+ const app = exuluApp.get();
11229
+ const config = app._config ?? app.config;
11230
+ const configuredBucket = config?.fileUploads?.s3Bucket;
11231
+ if (!configuredBucket) {
11232
+ throw new Error("File uploads are not configured (s3Bucket missing).");
11233
+ }
11234
+ const firstSlash = s3Key.indexOf("/");
11235
+ const bucket = firstSlash > 0 ? s3Key.slice(0, firstSlash) : configuredBucket;
11236
+ const objectKey = firstSlash > 0 ? s3Key.slice(firstSlash + 1) : s3Key;
11237
+ return getPresignedUrl(bucket, objectKey, config);
11238
+ };
11239
+ var transcriptionService = {
11240
+ /**
11241
+ * Create a transcription job row and dispatch it to the whisper server.
11242
+ * Throws TranscriptionServerUnavailable if the feature is off.
11243
+ */
11244
+ async startJob(input) {
11245
+ if (!transcriptionClient.isConfigured()) {
11246
+ throw new TranscriptionServerUnavailable(
11247
+ "TRANSCRIPTION_SERVER is not set. Start a whisper server with `npx @exulu/backend exulu-start-whisper` and point TRANSCRIPTION_SERVER at it."
11248
+ );
11249
+ }
11250
+ const { db: db2 } = await postgresClient();
11251
+ const now = /* @__PURE__ */ new Date();
11252
+ const [inserted] = await db2(TABLE).insert({
11253
+ audio_s3key: input.s3Key,
11254
+ title: input.title ?? input.filename,
11255
+ status: "queued",
11256
+ project_id: input.project_id ?? null,
11257
+ target_rights_mode: input.target_rights_mode ?? "private",
11258
+ target_rbac_users: input.target_rbac_users ? JSON.stringify(input.target_rbac_users) : null,
11259
+ target_rbac_roles: input.target_rbac_roles ? JSON.stringify(input.target_rbac_roles) : null,
11260
+ rights_mode: "private",
11261
+ created_by: input.userId,
11262
+ createdAt: now,
11263
+ updatedAt: now
11264
+ }).returning("*");
11265
+ const row = this._rowFromDb(inserted);
11266
+ try {
11267
+ const audioUrl = await presignAudio(input.s3Key);
11268
+ const submitted = await transcriptionClient.submitJob({
11269
+ audio_url: audioUrl,
11270
+ language: input.language ?? void 0,
11271
+ num_speakers: input.num_speakers ?? void 0,
11272
+ hotwords: input.hotwords
11273
+ });
11274
+ const [updated] = await db2(TABLE).where({ id: row.id }).update({
11275
+ whisper_job_id: submitted.job_id,
11276
+ status: "transcribing",
11277
+ updatedAt: /* @__PURE__ */ new Date()
11278
+ }).returning("*");
11279
+ return this._rowFromDb(updated);
11280
+ } catch (err) {
11281
+ const [failed] = await db2(TABLE).where({ id: row.id }).update({
11282
+ status: "failed",
11283
+ error: err.message,
11284
+ updatedAt: /* @__PURE__ */ new Date()
11285
+ }).returning("*");
11286
+ log2(`Failed to dispatch job ${row.id}: ${err.message}`);
11287
+ return this._rowFromDb(failed);
11288
+ }
11289
+ },
11290
+ /**
11291
+ * Reconcile every transcribing row against the whisper server. Called from
11292
+ * the polling loop on a fixed interval. Caps how many rows we touch per
11293
+ * tick so a backlog can't starve the event loop.
11294
+ */
11295
+ async pollOnce(maxPerTick = 50) {
11296
+ if (!transcriptionClient.isConfigured()) return;
11297
+ const { db: db2 } = await postgresClient();
11298
+ const rows = await db2(TABLE).where({ status: "transcribing" }).whereNotNull("whisper_job_id").limit(maxPerTick);
11299
+ for (const dbRow of rows) {
11300
+ const row = this._rowFromDb(dbRow);
11301
+ if (!row.whisper_job_id) continue;
11302
+ try {
11303
+ const job = await transcriptionClient.getJob(row.whisper_job_id);
11304
+ await this._applyJobUpdate(row, job);
11305
+ } catch (err) {
11306
+ const code = err.code;
11307
+ if (code === "JOB_NOT_FOUND") {
11308
+ await db2(TABLE).where({ id: row.id }).update({
11309
+ status: "failed",
11310
+ error: "lost on server restart",
11311
+ updatedAt: /* @__PURE__ */ new Date()
11312
+ });
11313
+ } else if (err instanceof TranscriptionServerUnavailable) {
11314
+ log2(`Whisper server unreachable while polling ${row.id}; will retry`);
11315
+ } else {
11316
+ log2(`Error polling job ${row.id}: ${err.message}`);
11317
+ }
11318
+ }
11319
+ }
11320
+ },
11321
+ async _applyJobUpdate(row, job) {
11322
+ const { db: db2 } = await postgresClient();
11323
+ if ((job.status === "queued" || job.status === "running") && job.duration_seconds != null && row.duration_seconds !== job.duration_seconds) {
11324
+ await db2(TABLE).where({ id: row.id }).update({
11325
+ duration_seconds: job.duration_seconds,
11326
+ updatedAt: /* @__PURE__ */ new Date()
11327
+ });
11328
+ }
11329
+ if (job.status === "running" || job.status === "queued") return;
11330
+ if (job.status === "completed") {
11331
+ await db2(TABLE).where({ id: row.id }).update({
11332
+ status: "awaiting_review",
11333
+ raw_segments: JSON.stringify(job.segments ?? []),
11334
+ language: job.language ?? null,
11335
+ duration_seconds: job.duration_seconds ?? null,
11336
+ updatedAt: /* @__PURE__ */ new Date()
11337
+ });
11338
+ return;
11339
+ }
11340
+ if (job.status === "failed") {
11341
+ await db2(TABLE).where({ id: row.id }).update({
11342
+ status: "failed",
11343
+ error: job.error ?? "transcription failed",
11344
+ updatedAt: /* @__PURE__ */ new Date()
11345
+ });
11346
+ return;
11347
+ }
11348
+ if (job.status === "cancelled") {
11349
+ await db2(TABLE).where({ id: row.id }).update({
11350
+ status: "cancelled",
11351
+ updatedAt: /* @__PURE__ */ new Date()
11352
+ });
11353
+ }
11354
+ },
11355
+ async cancelJob(id) {
11356
+ const { db: db2 } = await postgresClient();
11357
+ const dbRow = await db2(TABLE).where({ id }).first();
11358
+ if (!dbRow) throw new Error(`transcription_job ${id} not found`);
11359
+ const row = this._rowFromDb(dbRow);
11360
+ if (row.whisper_job_id && transcriptionClient.isConfigured()) {
11361
+ try {
11362
+ await transcriptionClient.cancelJob(row.whisper_job_id);
11363
+ } catch (err) {
11364
+ const code = err.code;
11365
+ if (code !== "JOB_NOT_FOUND") {
11366
+ log2(`Best-effort cancel of whisper job failed: ${err.message}`);
11367
+ }
11368
+ }
11369
+ }
11370
+ const [updated] = await db2(TABLE).where({ id }).update({ status: "cancelled", updatedAt: /* @__PURE__ */ new Date() }).returning("*");
11371
+ return this._rowFromDb(updated);
11372
+ },
11373
+ /**
11374
+ * User clicked Save in the review panel.
11375
+ *
11376
+ * - From 'awaiting_review': render the speaker-labeled transcript, create a
11377
+ * new ExuluContext item, apply RBAC + optional project linkage, mark the
11378
+ * job saved.
11379
+ * - From 'saved': re-render the transcript with the (possibly updated)
11380
+ * speaker map and upsert the existing context item by id. Used by the
11381
+ * Completed-section re-edit flow.
11382
+ */
11383
+ async finalize(id, input) {
11384
+ const { db: db2 } = await postgresClient();
11385
+ const dbRow = await db2(TABLE).where({ id }).first();
11386
+ if (!dbRow) throw new Error(`transcription_job ${id} not found`);
11387
+ const row = this._rowFromDb(dbRow);
11388
+ if (row.status !== "awaiting_review" && row.status !== "saved") {
11389
+ throw new Error(
11390
+ `transcription_job ${id} is in status '${row.status}'; can only finalize from 'awaiting_review' or 'saved'`
11391
+ );
11392
+ }
11393
+ if (!row.raw_segments) {
11394
+ throw new Error(`transcription_job ${id} has no raw_segments to finalize`);
11395
+ }
11396
+ const app = exuluApp.get();
11397
+ const context = app.context("transcriptions");
11398
+ if (!context) {
11399
+ throw new Error("Built-in transcriptions context not registered");
11400
+ }
11401
+ const config = app._config ?? app.config;
11402
+ const transcriptText = renderTranscript(row.raw_segments, input.speakers);
11403
+ const rightsMode = input.target_rights_mode ?? row.target_rights_mode ?? "private";
11404
+ const isReSave = row.status === "saved" && !!row.saved_item_id;
11405
+ const itemInput = {
11406
+ // Carrying the id on re-save makes context.createItem upsert in place.
11407
+ ...isReSave && row.saved_item_id ? { id: row.saved_item_id } : {},
11408
+ name: input.title ?? row.title ?? "Transcript",
11409
+ transcript_text: transcriptText,
11410
+ audio_s3key: row.audio_s3key,
11411
+ language: row.language ?? void 0,
11412
+ duration_seconds: row.duration_seconds ?? void 0,
11413
+ speakers: input.speakers,
11414
+ raw_segments: row.raw_segments,
11415
+ rights_mode: rightsMode,
11416
+ created_by: row.created_by
11417
+ };
11418
+ let item;
11419
+ try {
11420
+ const result = await context.createItem(
11421
+ itemInput,
11422
+ config,
11423
+ row.created_by,
11424
+ void 0,
11425
+ isReSave
11426
+ // upsert when re-saving
11427
+ );
11428
+ item = result.item;
11429
+ } catch (err) {
11430
+ await db2(TABLE).where({ id }).update({
11431
+ speakers: JSON.stringify(input.speakers),
11432
+ error: `Failed to save: ${err.message}`,
11433
+ updatedAt: /* @__PURE__ */ new Date()
11434
+ });
11435
+ throw err;
11436
+ }
11437
+ const itemId = item.id ?? row.saved_item_id ?? "";
11438
+ const users = input.target_rbac_users ?? row.target_rbac_users ?? [];
11439
+ const roles = input.target_rbac_roles ?? row.target_rbac_roles ?? [];
11440
+ if ((users.length || roles.length) && rightsMode !== "private") {
11441
+ try {
11442
+ await handleRBACUpdate(
11443
+ db2,
11444
+ "transcriptions",
11445
+ itemId,
11446
+ { users, roles },
11447
+ []
11448
+ );
11449
+ } catch (err) {
11450
+ log2(`RBAC update failed for item ${itemId}: ${err.message}`);
11451
+ }
11452
+ }
11453
+ const projectId = input.project_id ?? row.project_id ?? null;
11454
+ let projectWarning = null;
11455
+ if (projectId && !isReSave) {
11456
+ try {
11457
+ const project = await db2("projects").where({ id: projectId }).first();
11458
+ if (!project) {
11459
+ projectWarning = `project ${projectId} not found`;
11460
+ } else {
11461
+ const existing = parseJsonField(project.project_items) ?? [];
11462
+ const globalId = `transcriptions/${itemId}`;
11463
+ if (!existing.includes(globalId)) {
11464
+ existing.push(globalId);
11465
+ await db2("projects").where({ id: projectId }).update({
11466
+ project_items: JSON.stringify(existing),
11467
+ updatedAt: /* @__PURE__ */ new Date()
11468
+ });
11469
+ }
11470
+ }
11471
+ } catch (err) {
11472
+ projectWarning = err.message;
11473
+ }
11474
+ }
11475
+ const [updated] = await db2(TABLE).where({ id }).update({
11476
+ status: "saved",
11477
+ saved_item_id: itemId,
11478
+ title: input.title ?? row.title ?? null,
11479
+ speakers: JSON.stringify(input.speakers),
11480
+ error: projectWarning ? `Saved, but could not attach to project: ${projectWarning}` : null,
11481
+ updatedAt: /* @__PURE__ */ new Date()
11482
+ }).returning("*");
11483
+ return { item, row: this._rowFromDb(updated) };
11484
+ },
11485
+ _rowFromDb(dbRow) {
11486
+ return {
11487
+ ...dbRow,
11488
+ raw_segments: parseJsonField(dbRow.raw_segments),
11489
+ speakers: parseJsonField(dbRow.speakers),
11490
+ target_rbac_users: parseJsonField(
11491
+ dbRow.target_rbac_users
11492
+ ),
11493
+ target_rbac_roles: parseJsonField(
11494
+ dbRow.target_rbac_roles
11495
+ )
11496
+ };
11497
+ }
11498
+ };
11499
+
11500
+ // src/graphql/schemas/index.ts
11068
11501
  function createExuluContextsTypeDefs(table) {
11069
11502
  const enumDefs = table.fields.filter((field) => field.type === "enum" && field.enumValues).map((field) => {
11070
11503
  if (!field.enumValues) {
@@ -11482,6 +11915,39 @@ type PageInfo {
11482
11915
  mutationDefs += `
11483
11916
  deleteJob(queue: QueueEnum!, id: String!): JobActionReturnPayload
11484
11917
  `;
11918
+ mutationDefs += `
11919
+ transcriptionJobStart(input: TranscriptionJobStartInput!): transcription_job
11920
+ transcriptionJobFinalize(id: ID!, input: TranscriptionJobFinalizeInput!): TranscriptionJobFinalizeResult
11921
+ transcriptionJobCancel(id: ID!): transcription_job
11922
+ `;
11923
+ modelDefs += `
11924
+ input TranscriptionJobStartInput {
11925
+ audio_s3key: String!
11926
+ filename: String!
11927
+ title: String
11928
+ language: String
11929
+ num_speakers: Int
11930
+ hotwords: [String!]
11931
+ project_id: ID
11932
+ target_rights_mode: String
11933
+ target_rbac_users: [RBACUserInput!]
11934
+ target_rbac_roles: [RBACRoleInput!]
11935
+ }
11936
+
11937
+ input TranscriptionJobFinalizeInput {
11938
+ title: String
11939
+ speakers: JSON!
11940
+ project_id: ID
11941
+ target_rights_mode: String
11942
+ target_rbac_users: [RBACUserInput!]
11943
+ target_rbac_roles: [RBACRoleInput!]
11944
+ }
11945
+
11946
+ type TranscriptionJobFinalizeResult {
11947
+ job: transcription_job!
11948
+ item_id: ID!
11949
+ }
11950
+ `;
11485
11951
  typeDefs += `
11486
11952
  tools(search: String, category: String, limit: Int, page: Int): ToolPaginationResult
11487
11953
  toolCategories: [String!]!
@@ -11836,7 +12302,7 @@ type LiteLLMModel {
11836
12302
  } = await validateWorkflowPayload(jobData, providers);
11837
12303
  const retries = 3;
11838
12304
  let attempts = 0;
11839
- const promise = new Promise(async (resolve6, reject) => {
12305
+ const promise = new Promise(async (resolve7, reject) => {
11840
12306
  while (attempts < retries) {
11841
12307
  try {
11842
12308
  const messages2 = await processUiMessagesFlow({
@@ -11851,7 +12317,7 @@ type LiteLLMModel {
11851
12317
  config,
11852
12318
  variables: args.variables
11853
12319
  });
11854
- resolve6(messages2);
12320
+ resolve7(messages2);
11855
12321
  break;
11856
12322
  } catch (error) {
11857
12323
  console.error(
@@ -11865,7 +12331,7 @@ type LiteLLMModel {
11865
12331
  if (attempts >= retries) {
11866
12332
  reject(error instanceof Error ? error : new Error(String(error)));
11867
12333
  }
11868
- await new Promise((resolve7) => setTimeout((resolve8) => resolve8(true), 2e3));
12334
+ await new Promise((resolve8) => setTimeout((resolve9) => resolve9(true), 2e3));
11869
12335
  }
11870
12336
  }
11871
12337
  });
@@ -12047,6 +12513,54 @@ type LiteLLMModel {
12047
12513
  await config2.queue.remove(args.id);
12048
12514
  return { success: true };
12049
12515
  };
12516
+ const assertOwnsTranscriptionJob = async (id, context) => {
12517
+ const { db: db2, user } = context;
12518
+ if (!user) throw new Error("Authentication required");
12519
+ if (user.super_admin === true) return;
12520
+ const row = await db2.from("transcription_jobs").select(["created_by", "rights_mode"]).where({ id }).first();
12521
+ if (!row) throw new Error(`transcription_job ${id} not found`);
12522
+ if (row.rights_mode === "public") return;
12523
+ if (row.created_by === user.id) return;
12524
+ throw new Error("Not authorized to act on this transcription job");
12525
+ };
12526
+ resolvers.Mutation["transcriptionJobStart"] = async (_, args, context) => {
12527
+ const { user } = context;
12528
+ if (!user) throw new Error("Authentication required");
12529
+ if (!transcriptionClient.isConfigured()) {
12530
+ throw new Error(
12531
+ "TRANSCRIPTION_DISABLED: TRANSCRIPTION_SERVER not set on this server. Ask the operator to start a whisper server with `npx @exulu/backend exulu-start-whisper`."
12532
+ );
12533
+ }
12534
+ return transcriptionService.startJob({
12535
+ userId: user.id,
12536
+ s3Key: args.input.audio_s3key,
12537
+ filename: args.input.filename,
12538
+ title: args.input.title,
12539
+ language: args.input.language ?? void 0,
12540
+ num_speakers: args.input.num_speakers ?? void 0,
12541
+ hotwords: args.input.hotwords ?? void 0,
12542
+ project_id: args.input.project_id ?? null,
12543
+ target_rights_mode: args.input.target_rights_mode ?? null,
12544
+ target_rbac_users: args.input.target_rbac_users ?? void 0,
12545
+ target_rbac_roles: args.input.target_rbac_roles ?? void 0
12546
+ });
12547
+ };
12548
+ resolvers.Mutation["transcriptionJobFinalize"] = async (_, args, context) => {
12549
+ await assertOwnsTranscriptionJob(args.id, context);
12550
+ const { item, row } = await transcriptionService.finalize(args.id, {
12551
+ title: args.input.title,
12552
+ speakers: args.input.speakers,
12553
+ project_id: args.input.project_id ?? null,
12554
+ target_rights_mode: args.input.target_rights_mode ?? null,
12555
+ target_rbac_users: args.input.target_rbac_users ?? void 0,
12556
+ target_rbac_roles: args.input.target_rbac_roles ?? void 0
12557
+ });
12558
+ return { job: row, item_id: item.id };
12559
+ };
12560
+ resolvers.Mutation["transcriptionJobCancel"] = async (_, args, context) => {
12561
+ await assertOwnsTranscriptionJob(args.id, context);
12562
+ return transcriptionService.cancelJob(args.id);
12563
+ };
12050
12564
  resolvers.Query["evals"] = async (_, args, context, info) => {
12051
12565
  const requestedFields = getRequestedFields(info);
12052
12566
  return {
@@ -12118,10 +12632,10 @@ type LiteLLMModel {
12118
12632
  contexts.map(async (context2) => {
12119
12633
  let processor = null;
12120
12634
  if (context2.processor) {
12121
- processor = await new Promise(async (resolve6, reject) => {
12635
+ processor = await new Promise(async (resolve7, reject) => {
12122
12636
  const config2 = context2.processor?.config;
12123
12637
  const queue = await config2?.queue;
12124
- resolve6({
12638
+ resolve7({
12125
12639
  name: context2.processor.name,
12126
12640
  description: context2.processor.description,
12127
12641
  queue: queue?.queue?.name || void 0,
@@ -12202,10 +12716,10 @@ type LiteLLMModel {
12202
12716
  }
12203
12717
  let processor = null;
12204
12718
  if (data.processor) {
12205
- processor = await new Promise(async (resolve6, reject) => {
12719
+ processor = await new Promise(async (resolve7, reject) => {
12206
12720
  const config2 = data.processor?.config;
12207
12721
  const queue = await config2?.queue;
12208
- resolve6({
12722
+ resolve7({
12209
12723
  name: data.processor.name,
12210
12724
  description: data.processor.description,
12211
12725
  queue: queue?.queue?.name || void 0,
@@ -12899,7 +13413,7 @@ var import_utils5 = require("@apollo/utils.keyvaluecache");
12899
13413
  var import_body_parser = __toESM(require("body-parser"), 1);
12900
13414
  var import_crypto_js8 = __toESM(require("crypto-js"), 1);
12901
13415
  var import_openai = __toESM(require("openai"), 1);
12902
- var import_fs3 = __toESM(require("fs"), 1);
13416
+ var import_fs4 = __toESM(require("fs"), 1);
12903
13417
  var import_node_crypto5 = require("crypto");
12904
13418
  var import_api2 = require("@opentelemetry/api");
12905
13419
  init_check_record_access();
@@ -13255,6 +13769,9 @@ var ExuluProvider = class {
13255
13769
  If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume
13256
13770
  they are talking with the current date in mind as a reference.`;
13257
13771
  let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
13772
+ if (user?.personal_system_prompt?.trim()) {
13773
+ system += "\n\nUser preferences:\n" + user.personal_system_prompt.trim();
13774
+ }
13258
13775
  system += "\n\n" + genericContext;
13259
13776
  if (memoryContext) {
13260
13777
  system += "\n\n" + memoryContext;
@@ -13354,7 +13871,10 @@ When a tool execution is not approved by the user, do not retry it unless explic
13354
13871
  agent,
13355
13872
  memoryItems
13356
13873
  ),
13357
- stopWhen: [(0, import_ai9.stepCountIs)(maxStepCount || 5)]
13874
+ // Stop after the image_generation tool fires — the widget IS the
13875
+ // assistant's response, no follow-up text turn is wanted (same
13876
+ // reasoning as question_ask: the UI artifact is the message).
13877
+ stopWhen: [(0, import_ai9.stepCountIs)(maxStepCount || 5), (0, import_ai9.hasToolCall)("image_generation")]
13358
13878
  // make configurable
13359
13879
  });
13360
13880
  console.log("[EXULU] Output: " + JSON.stringify(output, null, 2));
@@ -13435,7 +13955,7 @@ When a tool execution is not approved by the user, do not retry it unless explic
13435
13955
  agent,
13436
13956
  memoryItems
13437
13957
  ),
13438
- stopWhen: [(0, import_ai9.stepCountIs)(maxStepCount || 5)]
13958
+ stopWhen: [(0, import_ai9.stepCountIs)(maxStepCount || 5), (0, import_ai9.hasToolCall)("image_generation")]
13439
13959
  });
13440
13960
  if (statistics) {
13441
13961
  await Promise.all([
@@ -13660,6 +14180,9 @@ ${extractedText}
13660
14180
  messages = await this.processFilePartsInMessages(messages);
13661
14181
  const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
13662
14182
  let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
14183
+ if (user?.personal_system_prompt?.trim()) {
14184
+ system += "\n\nUser preferences:\n" + user.personal_system_prompt.trim();
14185
+ }
13663
14186
  system += "\n\n" + genericContext;
13664
14187
  const includesContextSearchTool = currentTools?.some(
13665
14188
  (tool7) => tool7.name.toLowerCase().includes("context_search") || tool7.id.includes("context_search") || tool7.type === "context"
@@ -13816,7 +14339,7 @@ When a tool execution is not approved by the user, do not retry it unless explic
13816
14339
  },
13817
14340
  // provide more loops for skills because they are more complex to execute
13818
14341
  // todo allow configuring this per skill
13819
- stopWhen: [(0, import_ai9.stepCountIs)(maxStepCount || currentSkills?.length ? 10 : 5)]
14342
+ stopWhen: [(0, import_ai9.stepCountIs)(maxStepCount || currentSkills?.length ? 10 : 5), (0, import_ai9.hasToolCall)("image_generation")]
13820
14343
  });
13821
14344
  return {
13822
14345
  stream: result,
@@ -14026,80 +14549,547 @@ async function synthesizeSpeech(args) {
14026
14549
  return Buffer.from(arrayBuf);
14027
14550
  }
14028
14551
 
14029
- // src/exulu/routes.ts
14030
- init_tags();
14031
- var import_multer = __toESM(require("multer"), 1);
14032
-
14033
- // src/utils/check-provider-rate-limit.ts
14552
+ // src/exulu/image-generation.ts
14034
14553
  init_cjs_shims();
14035
- var checkProviderRateLimit = async (provider) => {
14036
- if (provider.rateLimit) {
14037
- console.log("[EXULU] rate limiting provider.", provider.rateLimit);
14038
- const limit = await providerRateLimiter(
14039
- provider.rateLimit.name || provider.id,
14040
- provider.rateLimit.rate_limit.time,
14041
- provider.rateLimit.rate_limit.limit,
14042
- 1
14043
- );
14044
- if (!limit.status) {
14045
- throw new Error("Rate limit exceeded.");
14046
- }
14554
+ var ImageGenerationError = class extends Error {
14555
+ constructor(upstreamStatus, message) {
14556
+ super(message);
14557
+ this.upstreamStatus = upstreamStatus;
14558
+ this.name = "ImageGenerationError";
14047
14559
  }
14048
14560
  };
14049
- var providerRateLimiter = async (key2, windowSeconds, limit, points) => {
14050
- try {
14051
- const { client: client2 } = await redisClient();
14052
- if (!client2) {
14053
- console.warn("[EXULU] Rate limiting disabled - Redis not available");
14054
- return {
14055
- status: true,
14056
- retryAfter: null
14057
- };
14058
- }
14059
- const redisKey = `exulu/${key2}`;
14060
- const current = await client2.incrBy(redisKey, points);
14061
- if (current === points) {
14062
- await client2.expire(redisKey, windowSeconds);
14063
- }
14064
- if (current > limit) {
14065
- const ttl = await client2.ttl(redisKey);
14066
- return {
14067
- status: false,
14068
- retryAfter: ttl
14069
- };
14561
+ var resolveProxyConfig = () => {
14562
+ const host = process.env.LITELLM_HOST ?? "127.0.0.1";
14563
+ const port = process.env.LITELLM_PORT ?? "4000";
14564
+ const masterKey = process.env.LITELLM_MASTER_KEY;
14565
+ if (!masterKey) throw new Error("LITELLM_MASTER_KEY is not set");
14566
+ return { host, port, masterKey };
14567
+ };
14568
+ var normalizeDataEntries = async (data) => {
14569
+ const out = [];
14570
+ for (const entry of data) {
14571
+ let buffer;
14572
+ let contentType = "image/png";
14573
+ let extension = "png";
14574
+ if (entry.b64_json) {
14575
+ buffer = Buffer.from(entry.b64_json, "base64");
14576
+ } else if (entry.url) {
14577
+ const upstream = await fetch(entry.url);
14578
+ if (!upstream.ok) {
14579
+ throw new ImageGenerationError(
14580
+ upstream.status,
14581
+ `Failed to download generated image from ${entry.url}: ${upstream.status} ${upstream.statusText}`
14582
+ );
14583
+ }
14584
+ const ct = upstream.headers.get("content-type");
14585
+ if (ct && ct.startsWith("image/")) {
14586
+ contentType = ct;
14587
+ const inferred = ct.split("/")[1]?.split(";")[0]?.trim();
14588
+ if (inferred) extension = inferred === "jpeg" ? "jpg" : inferred;
14589
+ }
14590
+ buffer = Buffer.from(await upstream.arrayBuffer());
14591
+ } else {
14592
+ throw new ImageGenerationError(
14593
+ 0,
14594
+ "LiteLLM image response entry contained neither b64_json nor url."
14595
+ );
14070
14596
  }
14071
- return {
14072
- status: true,
14073
- retryAfter: null
14074
- };
14075
- } catch (error) {
14076
- console.error("[EXULU] Rate limiting error:", error);
14077
- return {
14078
- status: true,
14079
- retryAfter: null
14080
- };
14597
+ out.push({ buffer, contentType, extension, revisedPrompt: entry.revised_prompt });
14081
14598
  }
14599
+ return out;
14082
14600
  };
14083
-
14084
- // src/utils/check-api-key-scope.ts
14085
- init_cjs_shims();
14086
- function checkApiKeyScope(user, agentId) {
14087
- if (!user || user.type !== "api") return { allowed: true };
14088
- if (!user.scope_mode || user.scope_mode === "admin") return { allowed: true };
14089
- if (user.scope_mode === "agents") {
14090
- const ids = Array.isArray(user.agent_ids) ? user.agent_ids : [];
14091
- if (!ids.includes(agentId)) {
14092
- return {
14093
- allowed: false,
14094
- reason: `API key is not scoped to agent ${agentId}.`,
14095
- code: 403
14096
- };
14097
- }
14098
- return { allowed: true };
14601
+ async function generateImage(args) {
14602
+ if (!args.model) throw new Error("model is required");
14603
+ if (!args.prompt) throw new Error("prompt is required");
14604
+ const cfg = resolveProxyConfig();
14605
+ const body = {
14606
+ model: args.model,
14607
+ prompt: args.prompt
14608
+ };
14609
+ if (args.size) body.size = args.size;
14610
+ if (args.quality) body.quality = args.quality;
14611
+ if (args.n) body.n = args.n;
14612
+ const res = await fetch(`http://${cfg.host}:${cfg.port}/v1/images/generations`, {
14613
+ method: "POST",
14614
+ headers: {
14615
+ Authorization: `Bearer ${cfg.masterKey}`,
14616
+ "Content-Type": "application/json"
14617
+ },
14618
+ body: JSON.stringify(body),
14619
+ signal: args.signal
14620
+ });
14621
+ if (!res.ok) {
14622
+ const text = await res.text().catch(() => "");
14623
+ throw new ImageGenerationError(
14624
+ res.status,
14625
+ `LiteLLM image generation failed (status ${res.status}): ${text}`.trim()
14626
+ );
14099
14627
  }
14100
- return { allowed: false, reason: "Unknown scope_mode.", code: 401 };
14101
- }
14102
-
14628
+ const json = await res.json();
14629
+ if (!json?.data || json.data.length === 0) {
14630
+ throw new ImageGenerationError(
14631
+ res.status,
14632
+ "LiteLLM returned no image data in the response."
14633
+ );
14634
+ }
14635
+ return normalizeDataEntries(json.data);
14636
+ }
14637
+ async function editImage(args) {
14638
+ if (!args.model) throw new Error("model is required");
14639
+ if (!args.prompt) throw new Error("prompt is required");
14640
+ if (!args.references || args.references.length === 0) {
14641
+ throw new Error("at least one reference image is required");
14642
+ }
14643
+ const cfg = resolveProxyConfig();
14644
+ const form = new FormData();
14645
+ form.append("model", args.model);
14646
+ form.append("prompt", args.prompt);
14647
+ if (args.n != null) form.append("n", String(args.n));
14648
+ if (args.size) form.append("size", args.size);
14649
+ if (args.quality) form.append("quality", args.quality);
14650
+ form.append("response_format", "b64_json");
14651
+ for (const ref of args.references) {
14652
+ form.append(
14653
+ "image",
14654
+ new Blob([ref.buffer], { type: ref.mimetype ?? "image/png" }),
14655
+ ref.filename
14656
+ );
14657
+ }
14658
+ if (args.mask) {
14659
+ form.append(
14660
+ "mask",
14661
+ new Blob([args.mask.buffer], { type: args.mask.mimetype ?? "image/png" }),
14662
+ args.mask.filename
14663
+ );
14664
+ }
14665
+ const res = await fetch(`http://${cfg.host}:${cfg.port}/v1/images/edits`, {
14666
+ method: "POST",
14667
+ headers: { Authorization: `Bearer ${cfg.masterKey}` },
14668
+ body: form,
14669
+ signal: args.signal
14670
+ });
14671
+ if (!res.ok) {
14672
+ const text = await res.text().catch(() => "");
14673
+ throw new ImageGenerationError(
14674
+ res.status,
14675
+ `LiteLLM image edit failed (status ${res.status}): ${text}`.trim()
14676
+ );
14677
+ }
14678
+ const json = await res.json();
14679
+ if (!json?.data || json.data.length === 0) {
14680
+ throw new ImageGenerationError(
14681
+ res.status,
14682
+ "LiteLLM returned no image data in the edit response."
14683
+ );
14684
+ }
14685
+ return normalizeDataEntries(json.data);
14686
+ }
14687
+
14688
+ // src/exulu/litellm/parse-image-models.ts
14689
+ init_cjs_shims();
14690
+ var import_node_fs5 = require("fs");
14691
+ var stripComment = (line) => {
14692
+ const idx = line.indexOf("#");
14693
+ return idx >= 0 ? line.slice(0, idx) : line;
14694
+ };
14695
+ var parseInlineArray = (raw) => {
14696
+ const m = raw.trim().match(/^\[(.*)\]$/);
14697
+ if (!m) return void 0;
14698
+ const inner = m[1] ?? "";
14699
+ if (!inner.trim()) return [];
14700
+ return inner.split(",").map((s) => s.trim().replace(/^["']|["']$/g, "")).filter((s) => s.length > 0);
14701
+ };
14702
+ var parseBool = (raw) => {
14703
+ const v = raw.trim().toLowerCase();
14704
+ if (v === "true" || v === "yes") return true;
14705
+ if (v === "false" || v === "no") return false;
14706
+ return void 0;
14707
+ };
14708
+ var parseInt10 = (raw) => {
14709
+ const n = Number(raw.trim());
14710
+ return Number.isInteger(n) ? n : void 0;
14711
+ };
14712
+ var parseImageGenerationModels = (configPath) => {
14713
+ if (!(0, import_node_fs5.existsSync)(configPath)) return [];
14714
+ const text = (0, import_node_fs5.readFileSync)(configPath, "utf8");
14715
+ const lines = text.split("\n");
14716
+ const entries = [];
14717
+ let current;
14718
+ for (const rawLine of lines) {
14719
+ const noComment = stripComment(rawLine);
14720
+ if (!noComment.trim()) continue;
14721
+ const indent = (rawLine.match(/^\s*/)?.[0] ?? "").length;
14722
+ const modelNameMatch = noComment.match(
14723
+ /^\s*-\s*model_name\s*:\s*["']?([^"'\s#]+)["']?\s*$/
14724
+ );
14725
+ if (modelNameMatch) {
14726
+ if (current) entries.push(current);
14727
+ current = { model_name: modelNameMatch[1], indent };
14728
+ continue;
14729
+ }
14730
+ if (!current) continue;
14731
+ if (indent <= current.indent && !/^\s*-\s/.test(rawLine)) {
14732
+ entries.push(current);
14733
+ current = void 0;
14734
+ continue;
14735
+ }
14736
+ const kvMatch = noComment.match(/^\s*(\w+)\s*:\s*(.+?)\s*$/);
14737
+ if (!kvMatch) continue;
14738
+ const key2 = kvMatch[1] ?? "";
14739
+ const rawValue = kvMatch[2] ?? "";
14740
+ switch (key2) {
14741
+ case "type": {
14742
+ current.type = rawValue.replace(/^["']|["']$/g, "").trim();
14743
+ break;
14744
+ }
14745
+ case "sizes": {
14746
+ current.sizes = parseInlineArray(rawValue);
14747
+ break;
14748
+ }
14749
+ case "qualities": {
14750
+ current.qualities = parseInlineArray(rawValue);
14751
+ break;
14752
+ }
14753
+ case "supports_edit": {
14754
+ current.supports_edit = parseBool(rawValue);
14755
+ break;
14756
+ }
14757
+ case "max_n": {
14758
+ current.max_n = parseInt10(rawValue);
14759
+ break;
14760
+ }
14761
+ }
14762
+ }
14763
+ if (current) entries.push(current);
14764
+ const imageEntries = entries.filter((e) => e.type === "image_generation");
14765
+ const errors = [];
14766
+ const validated = [];
14767
+ for (const e of imageEntries) {
14768
+ const modelErrs = [];
14769
+ if (!Array.isArray(e.sizes) || e.sizes.length === 0) {
14770
+ modelErrs.push(
14771
+ 'model_info.sizes must be a non-empty inline YAML array of strings, e.g. `sizes: ["1024x1024", "1024x1536"]`'
14772
+ );
14773
+ }
14774
+ if (!Array.isArray(e.qualities) || e.qualities.length === 0) {
14775
+ modelErrs.push(
14776
+ 'model_info.qualities must be a non-empty inline YAML array of strings, e.g. `qualities: ["auto", "high"]`'
14777
+ );
14778
+ }
14779
+ if (typeof e.supports_edit !== "boolean") {
14780
+ modelErrs.push(
14781
+ "model_info.supports_edit must be a boolean (true/false)"
14782
+ );
14783
+ }
14784
+ if (typeof e.max_n !== "number" || !Number.isInteger(e.max_n) || e.max_n < 1) {
14785
+ modelErrs.push("model_info.max_n must be an integer \u2265 1");
14786
+ }
14787
+ if (modelErrs.length > 0) {
14788
+ errors.push(
14789
+ ` - "${e.model_name}":
14790
+ - ${modelErrs.join("\n - ")}`
14791
+ );
14792
+ continue;
14793
+ }
14794
+ validated.push({
14795
+ model_name: e.model_name,
14796
+ sizes: e.sizes,
14797
+ qualities: e.qualities,
14798
+ supports_edit: e.supports_edit,
14799
+ max_n: e.max_n
14800
+ });
14801
+ }
14802
+ if (errors.length > 0) {
14803
+ throw new Error(
14804
+ `[EXULU] config.litellm.yaml has image-generation models with missing or invalid model_info keys. Fix and restart Exulu:
14805
+ ${errors.join("\n")}
14806
+ See docs/superpowers/specs/2026-05-31-in-chat-image-generation-design.md for the required schema.`
14807
+ );
14808
+ }
14809
+ return validated;
14810
+ };
14811
+
14812
+ // src/exulu/routes.ts
14813
+ var import_node_path5 = require("path");
14814
+
14815
+ // src/utils/python-setup.ts
14816
+ init_cjs_shims();
14817
+ var import_child_process = require("child_process");
14818
+ var import_util = require("util");
14819
+ var import_path = require("path");
14820
+ var import_fs3 = require("fs");
14821
+ var import_url = require("url");
14822
+ var execAsync4 = (0, import_util.promisify)(import_child_process.exec);
14823
+ function getPackageRoot() {
14824
+ const currentFile = (0, import_url.fileURLToPath)(importMetaUrl);
14825
+ let currentDir = (0, import_path.dirname)(currentFile);
14826
+ let attempts = 0;
14827
+ const maxAttempts = 10;
14828
+ while (attempts < maxAttempts) {
14829
+ const packageJsonPath = (0, import_path.join)(currentDir, "package.json");
14830
+ if ((0, import_fs3.existsSync)(packageJsonPath)) {
14831
+ try {
14832
+ const packageJson = JSON.parse((0, import_fs3.readFileSync)(packageJsonPath, "utf-8"));
14833
+ if (packageJson.name === "@exulu/backend") {
14834
+ return currentDir;
14835
+ }
14836
+ } catch {
14837
+ }
14838
+ }
14839
+ const parentDir = (0, import_path.resolve)(currentDir, "..");
14840
+ if (parentDir === currentDir) {
14841
+ break;
14842
+ }
14843
+ currentDir = parentDir;
14844
+ attempts++;
14845
+ }
14846
+ const fallback = (0, import_path.resolve)((0, import_path.dirname)((0, import_url.fileURLToPath)(importMetaUrl)), "../..");
14847
+ return fallback;
14848
+ }
14849
+ function getSetupScriptPath(packageRoot) {
14850
+ return (0, import_path.resolve)(packageRoot, "ee/python/setup.sh");
14851
+ }
14852
+ function getVenvPath(packageRoot) {
14853
+ return (0, import_path.resolve)(packageRoot, "ee/python/.venv");
14854
+ }
14855
+ function isPythonEnvironmentSetup(packageRoot) {
14856
+ const root = packageRoot ?? getPackageRoot();
14857
+ const venvPath = getVenvPath(root);
14858
+ const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
14859
+ return (0, import_fs3.existsSync)(venvPath) && (0, import_fs3.existsSync)(pythonPath);
14860
+ }
14861
+ async function setupPythonEnvironment(options = {}) {
14862
+ const {
14863
+ packageRoot = getPackageRoot(),
14864
+ force = false,
14865
+ verbose = false,
14866
+ timeout = 6e5
14867
+ // 10 minutes
14868
+ } = options;
14869
+ if (!force && isPythonEnvironmentSetup(packageRoot)) {
14870
+ if (verbose) {
14871
+ console.log("\u2713 Python environment already set up");
14872
+ }
14873
+ return {
14874
+ success: true,
14875
+ message: "Python environment already exists",
14876
+ alreadyExists: true
14877
+ };
14878
+ }
14879
+ const setupScriptPath = getSetupScriptPath(packageRoot);
14880
+ if (!(0, import_fs3.existsSync)(setupScriptPath)) {
14881
+ return {
14882
+ success: false,
14883
+ message: `Setup script not found at: ${setupScriptPath}`,
14884
+ alreadyExists: false
14885
+ };
14886
+ }
14887
+ try {
14888
+ if (verbose) {
14889
+ console.log("Setting up Python environment...");
14890
+ }
14891
+ const { stdout, stderr } = await execAsync4(`bash "${setupScriptPath}"`, {
14892
+ cwd: packageRoot,
14893
+ timeout,
14894
+ env: {
14895
+ ...process.env,
14896
+ // Ensure script can write to the directory
14897
+ PYTHONDONTWRITEBYTECODE: "1"
14898
+ },
14899
+ maxBuffer: 10 * 1024 * 1024
14900
+ // 10MB buffer
14901
+ });
14902
+ const output = stdout + stderr;
14903
+ const versionMatch = output.match(/Python (\d+\.\d+\.\d+)/);
14904
+ const pythonVersion = versionMatch ? versionMatch[1] : void 0;
14905
+ if (verbose) {
14906
+ console.log(output);
14907
+ }
14908
+ return {
14909
+ success: true,
14910
+ message: "Python environment set up successfully",
14911
+ alreadyExists: false,
14912
+ pythonVersion,
14913
+ output
14914
+ };
14915
+ } catch (error) {
14916
+ const errorOutput = error.stdout + error.stderr;
14917
+ return {
14918
+ success: false,
14919
+ message: `Setup failed: ${error.message}`,
14920
+ alreadyExists: false,
14921
+ output: errorOutput
14922
+ };
14923
+ }
14924
+ }
14925
+ function getPythonSetupInstructions() {
14926
+ return `
14927
+ Python environment not set up. Please run one of the following commands:
14928
+
14929
+ Option 1 (Automatic):
14930
+ import { setupPythonEnvironment } from '@exulu/backend';
14931
+ await setupPythonEnvironment();
14932
+
14933
+ Option 2 (Manual - for package consumers):
14934
+ npx @exulu/backend setup-python
14935
+
14936
+ Option 3 (Manual - for contributors):
14937
+ npm run python:setup
14938
+
14939
+ These commands will automatically create a Python virtual environment (.venv)
14940
+ in the @exulu/backend package and install all required dependencies.
14941
+
14942
+ Requirements:
14943
+ - Python 3.10 or higher must be installed
14944
+ - pip must be available
14945
+ - venv module must be available (for creating virtual environments)
14946
+
14947
+ If Python dependencies are not installed, install them first, then run one of the commands above:
14948
+ - macOS: brew install python@3.12
14949
+ - Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
14950
+ - Alpine Linux: apk add python3 py3-pip python3-dev
14951
+ - Windows: Download from https://www.python.org/downloads/
14952
+
14953
+ Note: In Docker containers, ensure you install all three components:
14954
+ Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
14955
+ Alpine: apk add python3 py3-pip python3-dev
14956
+ `.trim();
14957
+ }
14958
+ async function validatePythonEnvironment(packageRoot, checkPackages = true) {
14959
+ const root = packageRoot ?? getPackageRoot();
14960
+ const venvPath = getVenvPath(root);
14961
+ const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
14962
+ if (!(0, import_fs3.existsSync)(venvPath)) {
14963
+ return {
14964
+ valid: false,
14965
+ message: getPythonSetupInstructions()
14966
+ };
14967
+ }
14968
+ if (!(0, import_fs3.existsSync)(pythonPath)) {
14969
+ return {
14970
+ valid: false,
14971
+ message: "Python virtual environment is corrupted. Please run:\n await setupPythonEnvironment({ force: true })"
14972
+ };
14973
+ }
14974
+ try {
14975
+ await execAsync4(`"${pythonPath}" --version`, { cwd: root });
14976
+ } catch {
14977
+ return {
14978
+ valid: false,
14979
+ message: "Python executable is not working. Please run:\n await setupPythonEnvironment({ force: true })"
14980
+ };
14981
+ }
14982
+ if (checkPackages) {
14983
+ const criticalPackages = ["docling", "transformers"];
14984
+ const missingPackages = [];
14985
+ for (const pkg of criticalPackages) {
14986
+ try {
14987
+ await execAsync4(`"${pythonPath}" -c "import ${pkg}"`, {
14988
+ cwd: root,
14989
+ timeout: 1e4
14990
+ // 10 second timeout per import check
14991
+ });
14992
+ } catch {
14993
+ missingPackages.push(pkg);
14994
+ }
14995
+ }
14996
+ if (missingPackages.length > 0) {
14997
+ return {
14998
+ valid: false,
14999
+ message: `Python environment exists but required packages are not installed: ${missingPackages.join(", ")}
15000
+
15001
+ This usually happens when:
15002
+ 1. The .venv folder was copied but dependencies were not installed
15003
+ 2. The package was installed via npm but setup script was not run
15004
+
15005
+ Please run:
15006
+ await setupPythonEnvironment({ force: true })
15007
+
15008
+ Or manually run the setup script:
15009
+ bash ` + getSetupScriptPath(root)
15010
+ };
15011
+ }
15012
+ }
15013
+ return {
15014
+ valid: true,
15015
+ message: "Python environment is valid"
15016
+ };
15017
+ }
15018
+
15019
+ // src/exulu/routes.ts
15020
+ init_tags();
15021
+ var import_multer = __toESM(require("multer"), 1);
15022
+
15023
+ // src/utils/check-provider-rate-limit.ts
15024
+ init_cjs_shims();
15025
+ var checkProviderRateLimit = async (provider) => {
15026
+ if (provider.rateLimit) {
15027
+ console.log("[EXULU] rate limiting provider.", provider.rateLimit);
15028
+ const limit = await providerRateLimiter(
15029
+ provider.rateLimit.name || provider.id,
15030
+ provider.rateLimit.rate_limit.time,
15031
+ provider.rateLimit.rate_limit.limit,
15032
+ 1
15033
+ );
15034
+ if (!limit.status) {
15035
+ throw new Error("Rate limit exceeded.");
15036
+ }
15037
+ }
15038
+ };
15039
+ var providerRateLimiter = async (key2, windowSeconds, limit, points) => {
15040
+ try {
15041
+ const { client: client2 } = await redisClient();
15042
+ if (!client2) {
15043
+ console.warn("[EXULU] Rate limiting disabled - Redis not available");
15044
+ return {
15045
+ status: true,
15046
+ retryAfter: null
15047
+ };
15048
+ }
15049
+ const redisKey = `exulu/${key2}`;
15050
+ const current = await client2.incrBy(redisKey, points);
15051
+ if (current === points) {
15052
+ await client2.expire(redisKey, windowSeconds);
15053
+ }
15054
+ if (current > limit) {
15055
+ const ttl = await client2.ttl(redisKey);
15056
+ return {
15057
+ status: false,
15058
+ retryAfter: ttl
15059
+ };
15060
+ }
15061
+ return {
15062
+ status: true,
15063
+ retryAfter: null
15064
+ };
15065
+ } catch (error) {
15066
+ console.error("[EXULU] Rate limiting error:", error);
15067
+ return {
15068
+ status: true,
15069
+ retryAfter: null
15070
+ };
15071
+ }
15072
+ };
15073
+
15074
+ // src/utils/check-api-key-scope.ts
15075
+ init_cjs_shims();
15076
+ function checkApiKeyScope(user, agentId) {
15077
+ if (!user || user.type !== "api") return { allowed: true };
15078
+ if (!user.scope_mode || user.scope_mode === "admin") return { allowed: true };
15079
+ if (user.scope_mode === "agents") {
15080
+ const ids = Array.isArray(user.agent_ids) ? user.agent_ids : [];
15081
+ if (!ids.includes(agentId)) {
15082
+ return {
15083
+ allowed: false,
15084
+ reason: `API key is not scoped to agent ${agentId}.`,
15085
+ code: 403
15086
+ };
15087
+ }
15088
+ return { allowed: true };
15089
+ }
15090
+ return { allowed: false, reason: "Unknown scope_mode.", code: 401 };
15091
+ }
15092
+
14103
15093
  // src/utils/check-agent-rate-limit.ts
14104
15094
  init_cjs_shims();
14105
15095
  function resolveCallerId(req, userId) {
@@ -14702,7 +15692,7 @@ var REQUEST_SIZE_LIMIT = "50mb";
14702
15692
  var getExuluVersionNumber = async () => {
14703
15693
  try {
14704
15694
  const path3 = process.cwd();
14705
- const packageJson = import_fs3.default.readFileSync(path3 + "/package.json", "utf8");
15695
+ const packageJson = import_fs4.default.readFileSync(path3 + "/package.json", "utf8");
14706
15696
  const packageData = JSON.parse(packageJson);
14707
15697
  const exuluVersion = packageData.dependencies["@exulu/backend"];
14708
15698
  console.log(`[EXULU] Installed exulu-backend version: ${exuluVersion}`);
@@ -14736,7 +15726,8 @@ var {
14736
15726
  contextPresetsSchema: contextPresetsSchema2,
14737
15727
  embedderSettingsSchema: embedderSettingsSchema2,
14738
15728
  promptFavoritesSchema: promptFavoritesSchema2,
14739
- statisticsSchema: statisticsSchema2
15729
+ statisticsSchema: statisticsSchema2,
15730
+ transcriptionJobsSchema: transcriptionJobsSchema2
14740
15731
  } = coreSchemas.get();
14741
15732
  var createExpressRoutes = async (app, providers, tools, contexts, config, evals, tracer, queues2, rerankers) => {
14742
15733
  let corsOptions = {
@@ -14794,7 +15785,8 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
14794
15785
  variablesSchema2(),
14795
15786
  workflowTemplatesSchema2(),
14796
15787
  statisticsSchema2(),
14797
- rbacSchema2()
15788
+ rbacSchema2(),
15789
+ transcriptionJobsSchema2()
14798
15790
  ],
14799
15791
  contexts ?? [],
14800
15792
  providers,
@@ -15542,81 +16534,562 @@ ${customInstructions}` : agent.instructions;
15542
16534
  res.status(503).json({ detail: "Transcription service is not ready. Try again shortly." });
15543
16535
  return;
15544
16536
  }
15545
- const language = typeof req.body?.language === "string" && /^[a-z]{2}$/.test(req.body.language) ? req.body.language : void 0;
15546
- try {
15547
- const { text } = await transcribeAudio({ file, language });
15548
- res.status(200).json({ text });
15549
- } catch (err) {
15550
- if (err instanceof TranscriptionError) {
15551
- const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
15552
- res.status(code).json({ detail: err.message });
15553
- return;
15554
- }
15555
- console.error("[EXULU] /transcribe failed", err);
15556
- res.status(500).json({
15557
- detail: err instanceof Error ? err.message : "Transcription failed."
15558
- });
15559
- }
16537
+ const language = typeof req.body?.language === "string" && /^[a-z]{2}$/.test(req.body.language) ? req.body.language : void 0;
16538
+ try {
16539
+ const { text } = await transcribeAudio({ file, language });
16540
+ res.status(200).json({ text });
16541
+ } catch (err) {
16542
+ if (err instanceof TranscriptionError) {
16543
+ const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
16544
+ res.status(code).json({ detail: err.message });
16545
+ return;
16546
+ }
16547
+ console.error("[EXULU] /transcribe failed", err);
16548
+ res.status(500).json({
16549
+ detail: err instanceof Error ? err.message : "Transcription failed."
16550
+ });
16551
+ }
16552
+ }
16553
+ );
16554
+ const MAX_TTS_INPUT_CHARS = 4e3;
16555
+ app.post(
16556
+ "/speech",
16557
+ import_body_parser.default.json({ limit: "64kb" }),
16558
+ async (req, res) => {
16559
+ if (!isLiteLLMEnabled() || !process.env.TTS_MODEL || !process.env.TTS_VOICE) {
16560
+ res.status(503).json({
16561
+ detail: "Text-to-speech is not enabled on this deployment. Set EXULU_USE_LITELLM=true, TTS_MODEL, and TTS_VOICE in the environment."
16562
+ });
16563
+ return;
16564
+ }
16565
+ const authenticationResult = await requestValidators.authenticate(req);
16566
+ if (!authenticationResult.user?.id) {
16567
+ res.status(authenticationResult.code || 401).json({ detail: authenticationResult.message });
16568
+ return;
16569
+ }
16570
+ const text = typeof req.body?.text === "string" ? req.body.text.trim() : "";
16571
+ if (!text) {
16572
+ res.status(400).json({ detail: "Missing 'text' in request body." });
16573
+ return;
16574
+ }
16575
+ if (text.length > MAX_TTS_INPUT_CHARS) {
16576
+ res.status(400).json({
16577
+ detail: `Text too long (${text.length} chars). Max ${MAX_TTS_INPUT_CHARS}.`
16578
+ });
16579
+ return;
16580
+ }
16581
+ try {
16582
+ await Promise.race([
16583
+ waitForLiteLLMReady(),
16584
+ new Promise(
16585
+ (_, reject) => setTimeout(() => reject(new Error("LiteLLM not ready")), 5e3)
16586
+ )
16587
+ ]);
16588
+ } catch {
16589
+ res.status(503).json({ detail: "Speech service is not ready. Try again shortly." });
16590
+ return;
16591
+ }
16592
+ try {
16593
+ const audio = await synthesizeSpeech({ text });
16594
+ res.status(200);
16595
+ res.setHeader("Content-Type", "audio/mpeg");
16596
+ res.setHeader("Content-Length", String(audio.length));
16597
+ res.setHeader("Cache-Control", "no-store");
16598
+ res.send(audio);
16599
+ } catch (err) {
16600
+ if (err instanceof SpeechError) {
16601
+ const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
16602
+ res.status(code).json({ detail: err.message });
16603
+ return;
16604
+ }
16605
+ console.error("[EXULU] /speech failed", err);
16606
+ res.status(500).json({
16607
+ detail: err instanceof Error ? err.message : "Speech generation failed."
16608
+ });
16609
+ }
16610
+ }
16611
+ );
16612
+ const imageModelsByName = (() => {
16613
+ if (!isLiteLLMEnabled() || !config?.fileUploads) return /* @__PURE__ */ new Map();
16614
+ try {
16615
+ const configPath = process.env.LITELLM_CONFIG_PATH ?? (0, import_node_path5.resolve)(getPackageRoot(), "./config.litellm.yaml");
16616
+ const models2 = parseImageGenerationModels(configPath);
16617
+ return new Map(models2.map((m) => [m.model_name, m]));
16618
+ } catch (err) {
16619
+ console.error(
16620
+ "[EXULU] Skipping /images/* routes due to config.litellm.yaml error:",
16621
+ err.message
16622
+ );
16623
+ return /* @__PURE__ */ new Map();
16624
+ }
16625
+ })();
16626
+ const imageRoutesEnabled = isLiteLLMEnabled() && !!config?.fileUploads?.s3region && !!config?.fileUploads?.s3key && !!config?.fileUploads?.s3secret && !!config?.fileUploads?.s3Bucket && imageModelsByName.size > 0;
16627
+ const respond503ImagesNotEnabled = (res) => {
16628
+ res.status(503).json({
16629
+ detail: "Image generation is not enabled on this deployment. Requires EXULU_USE_LITELLM=true, S3 fileUploads configuration, and at least one model in config.litellm.yaml with model_info.type=image_generation."
16630
+ });
16631
+ };
16632
+ const loadAuthedSession = async (req, res, sessionId, rights) => {
16633
+ const authResult = await requestValidators.authenticate(req);
16634
+ if (!authResult.user?.id) {
16635
+ res.status(authResult.code || 401).json({ detail: authResult.message });
16636
+ return null;
16637
+ }
16638
+ const { db: db2 } = await postgresClient();
16639
+ const session = await db2.from("agent_sessions").where({ id: sessionId }).first();
16640
+ if (!session) {
16641
+ res.status(404).json({ detail: `Session ${sessionId} not found.` });
16642
+ return null;
16643
+ }
16644
+ const sessionRbac = await RBACResolver(
16645
+ db2,
16646
+ "agent_sessions",
16647
+ session.id,
16648
+ session.rights_mode || "private"
16649
+ );
16650
+ const allowed = await checkRecordAccess(
16651
+ { ...session, RBAC: sessionRbac },
16652
+ rights,
16653
+ authResult.user
16654
+ );
16655
+ if (!allowed) {
16656
+ res.status(403).json({ detail: `You don't have ${rights} access to this session.` });
16657
+ return null;
16658
+ }
16659
+ return { user: authResult.user, session, db: db2 };
16660
+ };
16661
+ const loadStyle = async (db2, styleId, user, res) => {
16662
+ if (!styleId) return { markdown: null, id: null };
16663
+ const row = await db2.from("platform_configurations").where({ id: styleId }).first();
16664
+ if (!row) {
16665
+ res.status(404).json({ detail: `Style ${styleId} not found.` });
16666
+ return "error";
16667
+ }
16668
+ const rbac = await RBACResolver(
16669
+ db2,
16670
+ "platform_configurations",
16671
+ row.id,
16672
+ row.rights_mode || "private"
16673
+ );
16674
+ const allowed = await checkRecordAccess(
16675
+ { ...row, RBAC: rbac },
16676
+ "read",
16677
+ user
16678
+ );
16679
+ if (!allowed) {
16680
+ res.status(403).json({ detail: "You don't have access to that style." });
16681
+ return "error";
16682
+ }
16683
+ const value = typeof row.config_value === "string" ? (() => {
16684
+ try {
16685
+ return JSON.parse(row.config_value);
16686
+ } catch {
16687
+ return null;
16688
+ }
16689
+ })() : row.config_value;
16690
+ return { markdown: value?.markdown ?? null, id: row.id };
16691
+ };
16692
+ const validateGenerationParams = (body, res) => {
16693
+ const { model: modelName, prompt, n, size, quality } = body || {};
16694
+ const model = typeof modelName === "string" ? imageModelsByName.get(modelName) : void 0;
16695
+ if (!model) {
16696
+ res.status(400).json({
16697
+ detail: `Unknown image-generation model "${modelName}". Available: ${[...imageModelsByName.keys()].join(", ")}.`
16698
+ });
16699
+ return null;
16700
+ }
16701
+ if (typeof prompt !== "string" || prompt.trim().length === 0) {
16702
+ res.status(400).json({ detail: "prompt must be a non-empty string." });
16703
+ return null;
16704
+ }
16705
+ const requestedN = typeof n === "number" ? n : 1;
16706
+ if (!Number.isInteger(requestedN) || requestedN < 1 || requestedN > model.max_n) {
16707
+ res.status(400).json({
16708
+ detail: `n must be an integer between 1 and ${model.max_n} for model ${model.model_name}.`
16709
+ });
16710
+ return null;
16711
+ }
16712
+ if (size && !model.sizes.includes(size)) {
16713
+ res.status(400).json({
16714
+ detail: `size "${size}" is not supported by ${model.model_name}. Allowed: ${model.sizes.join(", ")}.`
16715
+ });
16716
+ return null;
16717
+ }
16718
+ if (quality && !model.qualities.includes(quality)) {
16719
+ res.status(400).json({
16720
+ detail: `quality "${quality}" is not supported by ${model.model_name}. Allowed: ${model.qualities.join(", ")}.`
16721
+ });
16722
+ return null;
16723
+ }
16724
+ return { model, prompt, n: requestedN, size, quality };
16725
+ };
16726
+ const uploadGeneratedImages = async (images, sessionId, toolCallId, userId) => {
16727
+ if (!config?.fileUploads) {
16728
+ throw new Error("File uploads not configured.");
16729
+ }
16730
+ const keys = [];
16731
+ const revisedPrompts = [];
16732
+ for (const img of images) {
16733
+ const filename = `${(0, import_node_crypto5.randomUUID)()}.${img.extension}`;
16734
+ const key2 = `sessions/${sessionId}/images/${toolCallId}/${filename}`;
16735
+ const fullKey = await uploadFile(
16736
+ img.buffer,
16737
+ key2,
16738
+ config,
16739
+ { contentType: img.contentType },
16740
+ userId
16741
+ );
16742
+ keys.push(fullKey);
16743
+ revisedPrompts.push(img.revisedPrompt ?? null);
16744
+ }
16745
+ const presignedUrls = await Promise.all(
16746
+ keys.map((fullKey) => {
16747
+ const slash = fullKey.indexOf("/");
16748
+ const bucket = slash > 0 ? fullKey.slice(0, slash) : config.fileUploads.s3Bucket;
16749
+ const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
16750
+ return getPresignedUrl(bucket, objectKey, config);
16751
+ })
16752
+ );
16753
+ return { keys, revisedPrompts, presignedUrls };
16754
+ };
16755
+ app.post("/images/generate", async (req, res) => {
16756
+ if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
16757
+ const { sessionId, toolCallId, styleId } = req.body || {};
16758
+ if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
16759
+ res.status(400).json({ detail: "sessionId and toolCallId are required." });
16760
+ return;
16761
+ }
16762
+ const authed = await loadAuthedSession(req, res, sessionId, "write");
16763
+ if (!authed) return;
16764
+ const params = validateGenerationParams(req.body, res);
16765
+ if (!params) return;
16766
+ const style = await loadStyle(authed.db, styleId, authed.user, res);
16767
+ if (style === "error") return;
16768
+ const finalPrompt = style.markdown ? `${params.prompt}
16769
+
16770
+ ${style.markdown}` : params.prompt;
16771
+ try {
16772
+ await Promise.race([
16773
+ waitForLiteLLMReady(),
16774
+ new Promise(
16775
+ (_, reject) => setTimeout(() => reject(new Error("LiteLLM not ready")), 5e3)
16776
+ )
16777
+ ]);
16778
+ } catch {
16779
+ res.status(503).json({ detail: "Image service is not ready. Try again shortly." });
16780
+ return;
16781
+ }
16782
+ const abortController = new AbortController();
16783
+ req.on("close", () => abortController.abort());
16784
+ try {
16785
+ const images = await generateImage({
16786
+ model: params.model.model_name,
16787
+ prompt: finalPrompt,
16788
+ n: params.n,
16789
+ size: params.size,
16790
+ quality: params.quality,
16791
+ signal: abortController.signal
16792
+ });
16793
+ const { keys, revisedPrompts, presignedUrls } = await uploadGeneratedImages(
16794
+ images,
16795
+ sessionId,
16796
+ toolCallId,
16797
+ authed.user.id
16798
+ );
16799
+ const [row] = await authed.db("image_generations").insert({
16800
+ session_id: sessionId,
16801
+ tool_call_id: toolCallId,
16802
+ user_id: authed.user.id,
16803
+ operation: "generate",
16804
+ model: params.model.model_name,
16805
+ prompt: params.prompt,
16806
+ applied_style_id: style.id,
16807
+ applied_style_markdown: style.markdown,
16808
+ size: params.size,
16809
+ quality: params.quality,
16810
+ n: params.n,
16811
+ image_keys: JSON.stringify(keys),
16812
+ revised_prompts: JSON.stringify(revisedPrompts),
16813
+ selected: false
16814
+ }).returning("*");
16815
+ res.status(200).json({
16816
+ generationId: row.id,
16817
+ images: keys.map((key2, i) => ({
16818
+ key: key2,
16819
+ presignedUrl: presignedUrls[i],
16820
+ revisedPrompt: revisedPrompts[i]
16821
+ }))
16822
+ });
16823
+ } catch (err) {
16824
+ if (abortController.signal.aborted) return;
16825
+ if (err instanceof ImageGenerationError) {
16826
+ const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
16827
+ res.status(code).json({ detail: err.message });
16828
+ return;
16829
+ }
16830
+ console.error("[EXULU] /images/generate failed", err);
16831
+ res.status(500).json({
16832
+ detail: err instanceof Error ? err.message : "Image generation failed."
16833
+ });
15560
16834
  }
15561
- );
15562
- const MAX_TTS_INPUT_CHARS = 4e3;
15563
- app.post(
15564
- "/speech",
15565
- import_body_parser.default.json({ limit: "64kb" }),
15566
- async (req, res) => {
15567
- if (!isLiteLLMEnabled() || !process.env.TTS_MODEL || !process.env.TTS_VOICE) {
15568
- res.status(503).json({
15569
- detail: "Text-to-speech is not enabled on this deployment. Set EXULU_USE_LITELLM=true, TTS_MODEL, and TTS_VOICE in the environment."
15570
- });
16835
+ });
16836
+ app.post("/images/edit", async (req, res) => {
16837
+ if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
16838
+ const { sessionId, toolCallId, styleId, referenceImageKeys, maskKey } = req.body || {};
16839
+ if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
16840
+ res.status(400).json({ detail: "sessionId and toolCallId are required." });
16841
+ return;
16842
+ }
16843
+ if (!Array.isArray(referenceImageKeys) || referenceImageKeys.length === 0) {
16844
+ res.status(400).json({ detail: "referenceImageKeys must be a non-empty array." });
16845
+ return;
16846
+ }
16847
+ const authed = await loadAuthedSession(req, res, sessionId, "write");
16848
+ if (!authed) return;
16849
+ const params = validateGenerationParams(req.body, res);
16850
+ if (!params) return;
16851
+ if (!params.model.supports_edit) {
16852
+ res.status(400).json({
16853
+ detail: `Model ${params.model.model_name} does not support image editing.`
16854
+ });
16855
+ return;
16856
+ }
16857
+ const userPrefix = `user_${authed.user.id}/`;
16858
+ const sessionPrefix = `sessions/${sessionId}/`;
16859
+ const ownsKey = (k) => k.includes(userPrefix) || k.includes(sessionPrefix);
16860
+ if (!referenceImageKeys.every((k) => typeof k === "string" && ownsKey(k))) {
16861
+ res.status(403).json({ detail: "One or more reference image keys are not accessible." });
16862
+ return;
16863
+ }
16864
+ if (maskKey && (typeof maskKey !== "string" || !ownsKey(maskKey))) {
16865
+ res.status(403).json({ detail: "Mask image is not accessible." });
16866
+ return;
16867
+ }
16868
+ const style = await loadStyle(authed.db, styleId, authed.user, res);
16869
+ if (style === "error") return;
16870
+ const finalPrompt = style.markdown ? `${params.prompt}
16871
+
16872
+ ${style.markdown}` : params.prompt;
16873
+ try {
16874
+ await Promise.race([
16875
+ waitForLiteLLMReady(),
16876
+ new Promise(
16877
+ (_, reject) => setTimeout(() => reject(new Error("LiteLLM not ready")), 5e3)
16878
+ )
16879
+ ]);
16880
+ } catch {
16881
+ res.status(503).json({ detail: "Image service is not ready. Try again shortly." });
16882
+ return;
16883
+ }
16884
+ const fetchRef = async (fullKey) => {
16885
+ const slash = fullKey.indexOf("/");
16886
+ const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
16887
+ const buf = await getS3ObjectBytes(objectKey, config);
16888
+ const filename = fullKey.split("/").pop() ?? "image.png";
16889
+ const ext = filename.split(".").pop()?.toLowerCase() ?? "png";
16890
+ const mimetype = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : `image/${ext}`;
16891
+ return { buffer: buf, filename, mimetype };
16892
+ };
16893
+ const abortController = new AbortController();
16894
+ req.on("close", () => abortController.abort());
16895
+ try {
16896
+ const references = await Promise.all(referenceImageKeys.map(fetchRef));
16897
+ const mask = maskKey ? await fetchRef(maskKey) : void 0;
16898
+ const images = await editImage({
16899
+ model: params.model.model_name,
16900
+ prompt: finalPrompt,
16901
+ references,
16902
+ mask,
16903
+ n: params.n,
16904
+ size: params.size,
16905
+ quality: params.quality,
16906
+ signal: abortController.signal
16907
+ });
16908
+ const { keys, revisedPrompts, presignedUrls } = await uploadGeneratedImages(
16909
+ images,
16910
+ sessionId,
16911
+ toolCallId,
16912
+ authed.user.id
16913
+ );
16914
+ const [row] = await authed.db("image_generations").insert({
16915
+ session_id: sessionId,
16916
+ tool_call_id: toolCallId,
16917
+ user_id: authed.user.id,
16918
+ operation: "edit",
16919
+ model: params.model.model_name,
16920
+ prompt: params.prompt,
16921
+ applied_style_id: style.id,
16922
+ applied_style_markdown: style.markdown,
16923
+ size: params.size,
16924
+ quality: params.quality,
16925
+ n: params.n,
16926
+ reference_image_keys: JSON.stringify(referenceImageKeys),
16927
+ mask_image_key: maskKey,
16928
+ image_keys: JSON.stringify(keys),
16929
+ revised_prompts: JSON.stringify(revisedPrompts),
16930
+ selected: false
16931
+ }).returning("*");
16932
+ res.status(200).json({
16933
+ generationId: row.id,
16934
+ images: keys.map((key2, i) => ({
16935
+ key: key2,
16936
+ presignedUrl: presignedUrls[i],
16937
+ revisedPrompt: revisedPrompts[i]
16938
+ }))
16939
+ });
16940
+ } catch (err) {
16941
+ if (abortController.signal.aborted) return;
16942
+ if (err instanceof ImageGenerationError) {
16943
+ const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
16944
+ res.status(code).json({ detail: err.message });
15571
16945
  return;
15572
16946
  }
15573
- const authenticationResult = await requestValidators.authenticate(req);
15574
- if (!authenticationResult.user?.id) {
15575
- res.status(authenticationResult.code || 401).json({ detail: authenticationResult.message });
16947
+ console.error("[EXULU] /images/edit failed", err);
16948
+ res.status(500).json({
16949
+ detail: err instanceof Error ? err.message : "Image edit failed."
16950
+ });
16951
+ }
16952
+ });
16953
+ app.post("/images/select", async (req, res) => {
16954
+ if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
16955
+ const { sessionId, toolCallId, selections } = req.body || {};
16956
+ if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
16957
+ res.status(400).json({ detail: "sessionId and toolCallId are required." });
16958
+ return;
16959
+ }
16960
+ if (!Array.isArray(selections) || selections.length === 0) {
16961
+ res.status(400).json({ detail: "selections must be a non-empty array." });
16962
+ return;
16963
+ }
16964
+ const authed = await loadAuthedSession(req, res, sessionId, "write");
16965
+ if (!authed) return;
16966
+ const rows = await authed.db("image_generations").where({ session_id: sessionId, tool_call_id: toolCallId }).select("*");
16967
+ const rowsById = new Map(rows.map((r) => [r.id, r]));
16968
+ const selectedDetails = [];
16969
+ const rowsToMarkSelected = /* @__PURE__ */ new Set();
16970
+ for (const sel of selections) {
16971
+ if (typeof sel?.generationId !== "string" || typeof sel?.imageKey !== "string") {
16972
+ res.status(400).json({ detail: "Each selection needs generationId + imageKey." });
15576
16973
  return;
15577
16974
  }
15578
- const text = typeof req.body?.text === "string" ? req.body.text.trim() : "";
15579
- if (!text) {
15580
- res.status(400).json({ detail: "Missing 'text' in request body." });
16975
+ const row = rowsById.get(sel.generationId);
16976
+ if (!row) {
16977
+ res.status(404).json({ detail: `Generation ${sel.generationId} not found in this tool call.` });
15581
16978
  return;
15582
16979
  }
15583
- if (text.length > MAX_TTS_INPUT_CHARS) {
15584
- res.status(400).json({
15585
- detail: `Text too long (${text.length} chars). Max ${MAX_TTS_INPUT_CHARS}.`
15586
- });
16980
+ const keys = Array.isArray(row.image_keys) ? row.image_keys : (() => {
16981
+ try {
16982
+ return JSON.parse(row.image_keys);
16983
+ } catch {
16984
+ return [];
16985
+ }
16986
+ })();
16987
+ if (!keys.includes(sel.imageKey)) {
16988
+ res.status(400).json({ detail: `imageKey not part of generation ${sel.generationId}.` });
15587
16989
  return;
15588
16990
  }
16991
+ const slash = sel.imageKey.indexOf("/");
16992
+ const bucket = slash > 0 ? sel.imageKey.slice(0, slash) : config.fileUploads.s3Bucket;
16993
+ const objectKey = slash > 0 ? sel.imageKey.slice(slash + 1) : sel.imageKey;
16994
+ const presignedUrl = await getPresignedUrl(bucket, objectKey, config);
16995
+ let styleName = null;
16996
+ if (row.applied_style_id) {
16997
+ const styleRow = await authed.db("platform_configurations").where({ id: row.applied_style_id }).first();
16998
+ const parsed = styleRow?.config_value && typeof styleRow.config_value === "string" ? (() => {
16999
+ try {
17000
+ return JSON.parse(styleRow.config_value);
17001
+ } catch {
17002
+ return null;
17003
+ }
17004
+ })() : styleRow?.config_value;
17005
+ styleName = parsed?.name ?? null;
17006
+ }
17007
+ selectedDetails.push({
17008
+ key: sel.imageKey,
17009
+ presignedUrl,
17010
+ prompt: row.prompt,
17011
+ model: row.model,
17012
+ styleName
17013
+ });
17014
+ rowsToMarkSelected.add(row.id);
17015
+ }
17016
+ await authed.db("image_generations").whereIn("id", [...rowsToMarkSelected]).update({ selected: true });
17017
+ const lines = selectedDetails.map(
17018
+ (d) => `- ${d.presignedUrl} (prompt: "${d.prompt}", model: ${d.model}${d.styleName ? `, style: ${d.styleName}` : ""})`
17019
+ );
17020
+ const messageText = "The user generated and selected the following image(s) in this chat:\n" + lines.join("\n");
17021
+ const messageId = (0, import_node_crypto5.randomUUID)();
17022
+ const uiMessage = {
17023
+ id: messageId,
17024
+ role: "system",
17025
+ parts: [{ type: "text", text: messageText }]
17026
+ };
17027
+ await authed.db("agent_messages").insert({
17028
+ content: JSON.stringify(uiMessage),
17029
+ message_id: messageId,
17030
+ session: sessionId,
17031
+ user: authed.user.id
17032
+ });
17033
+ res.status(200).json({ ok: true, systemMessage: uiMessage, selectedImages: selectedDetails });
17034
+ });
17035
+ app.get("/images/history", async (req, res) => {
17036
+ if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
17037
+ const sessionId = typeof req.query.sessionId === "string" ? req.query.sessionId : "";
17038
+ const toolCallId = typeof req.query.toolCallId === "string" ? req.query.toolCallId : "";
17039
+ if (!sessionId || !toolCallId) {
17040
+ res.status(400).json({ detail: "sessionId and toolCallId query params are required." });
17041
+ return;
17042
+ }
17043
+ const authed = await loadAuthedSession(req, res, sessionId, "read");
17044
+ if (!authed) return;
17045
+ const rows = await authed.db("image_generations").where({ session_id: sessionId, tool_call_id: toolCallId }).orderBy("createdAt", "asc").select("*");
17046
+ const parseList = (v) => {
17047
+ if (!v) return [];
17048
+ if (Array.isArray(v)) return v;
15589
17049
  try {
15590
- await Promise.race([
15591
- waitForLiteLLMReady(),
15592
- new Promise(
15593
- (_, reject) => setTimeout(() => reject(new Error("LiteLLM not ready")), 5e3)
15594
- )
15595
- ]);
17050
+ return JSON.parse(v);
15596
17051
  } catch {
15597
- res.status(503).json({ detail: "Speech service is not ready. Try again shortly." });
15598
- return;
15599
- }
15600
- try {
15601
- const audio = await synthesizeSpeech({ text });
15602
- res.status(200);
15603
- res.setHeader("Content-Type", "audio/mpeg");
15604
- res.setHeader("Content-Length", String(audio.length));
15605
- res.setHeader("Cache-Control", "no-store");
15606
- res.send(audio);
15607
- } catch (err) {
15608
- if (err instanceof SpeechError) {
15609
- const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
15610
- res.status(code).json({ detail: err.message });
15611
- return;
15612
- }
15613
- console.error("[EXULU] /speech failed", err);
15614
- res.status(500).json({
15615
- detail: err instanceof Error ? err.message : "Speech generation failed."
15616
- });
17052
+ return [];
15617
17053
  }
15618
- }
15619
- );
17054
+ };
17055
+ const sign = async (fullKey) => {
17056
+ const slash = fullKey.indexOf("/");
17057
+ const bucket = slash > 0 ? fullKey.slice(0, slash) : config.fileUploads.s3Bucket;
17058
+ const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
17059
+ return getPresignedUrl(bucket, objectKey, config);
17060
+ };
17061
+ const history = await Promise.all(rows.map(async (r) => {
17062
+ const keys = parseList(r.image_keys);
17063
+ const refs = parseList(r.reference_image_keys);
17064
+ const revisedPrompts = parseList(r.revised_prompts);
17065
+ const [imageUrls, referenceUrls] = await Promise.all([
17066
+ Promise.all(keys.map(sign)),
17067
+ Promise.all(refs.map(sign))
17068
+ ]);
17069
+ return {
17070
+ generationId: r.id,
17071
+ createdAt: r.createdAt,
17072
+ operation: r.operation,
17073
+ model: r.model,
17074
+ prompt: r.prompt,
17075
+ appliedStyleId: r.applied_style_id ?? null,
17076
+ appliedStyleMarkdown: r.applied_style_markdown ?? null,
17077
+ size: r.size ?? null,
17078
+ quality: r.quality ?? null,
17079
+ n: r.n ?? 1,
17080
+ selected: !!r.selected,
17081
+ error: r.error ?? null,
17082
+ maskImageKey: r.mask_image_key ?? null,
17083
+ images: keys.map((key2, i) => ({
17084
+ key: key2,
17085
+ presignedUrl: imageUrls[i],
17086
+ revisedPrompt: revisedPrompts[i] ?? null
17087
+ })),
17088
+ references: refs.map((key2, i) => ({ key: key2, presignedUrl: referenceUrls[i] }))
17089
+ };
17090
+ }));
17091
+ res.status(200).json({ history });
17092
+ });
15620
17093
  app.use("/litellm/:project", async (req, res) => {
15621
17094
  if (!isLiteLLMEnabled()) {
15622
17095
  res.status(503).json({
@@ -18680,7 +20153,7 @@ var internetSearchTool = new ExuluTool({
18680
20153
  } catch (error) {
18681
20154
  if (error instanceof import_perplexity_ai.default.RateLimitError && attempt < maxRetries - 1) {
18682
20155
  const delay = Math.pow(2, attempt) * 1e3 + Math.random() * 1e3;
18683
- await new Promise((resolve6) => setTimeout(resolve6, delay));
20156
+ await new Promise((resolve7) => setTimeout(resolve7, delay));
18684
20157
  continue;
18685
20158
  }
18686
20159
  throw error;
@@ -18774,21 +20247,341 @@ var emailTool = new ExuluTool({
18774
20247
  tls: {
18775
20248
  rejectUnauthorized: false
18776
20249
  }
18777
- };
18778
- if (toolVariablesConfig.allowed_recipient_domains) {
18779
- const allowedRecipientDomains = toolVariablesConfig.allowed_recipient_domains.split(",");
18780
- if (!allowedRecipientDomains.some((domain) => recipient.endsWith(`@${domain}`))) {
18781
- return {
18782
- result: "Recipient domain not allowed to send emails to."
18783
- };
20250
+ };
20251
+ if (toolVariablesConfig.allowed_recipient_domains) {
20252
+ const allowedRecipientDomains = toolVariablesConfig.allowed_recipient_domains.split(",");
20253
+ if (!allowedRecipientDomains.some((domain) => recipient.endsWith(`@${domain}`))) {
20254
+ return {
20255
+ result: "Recipient domain not allowed to send emails to."
20256
+ };
20257
+ }
20258
+ }
20259
+ await sendEmail(recipient, subject, html, text, EMAIL_CONFIG);
20260
+ return {
20261
+ result: "Email sent successfully to " + recipient + " with subject " + subject + "."
20262
+ };
20263
+ }
20264
+ });
20265
+
20266
+ // src/templates/tools/transcribe.ts
20267
+ init_cjs_shims();
20268
+ init_tool();
20269
+ init_supervisor();
20270
+ init_uppy();
20271
+ var import_node_crypto8 = require("crypto");
20272
+ var import_promises3 = require("fs/promises");
20273
+ var import_node_path6 = require("path");
20274
+ var import_zod20 = require("zod");
20275
+ var SANDBOX_ROOT = "/tmp/exulu-sessions";
20276
+ var parseSandboxPath = (input) => {
20277
+ const stripped = input.startsWith("file://") ? input.slice("file://".length) : input;
20278
+ if (!stripped.startsWith(`${SANDBOX_ROOT}/`)) return null;
20279
+ const tail = stripped.slice(SANDBOX_ROOT.length + 1);
20280
+ const slash = tail.indexOf("/");
20281
+ if (slash < 1) return null;
20282
+ const sessionId = tail.slice(0, slash);
20283
+ const relPath = tail.slice(slash + 1);
20284
+ if (!relPath) return null;
20285
+ return { sessionId, relPath };
20286
+ };
20287
+ var audioMimetypeFromExtension = (filename) => {
20288
+ const ext = filename.split(".").pop()?.toLowerCase();
20289
+ switch (ext) {
20290
+ case "mp3":
20291
+ return "audio/mpeg";
20292
+ case "m4a":
20293
+ case "mp4":
20294
+ return "audio/mp4";
20295
+ case "wav":
20296
+ return "audio/wav";
20297
+ case "ogg":
20298
+ case "oga":
20299
+ return "audio/ogg";
20300
+ case "flac":
20301
+ return "audio/flac";
20302
+ case "webm":
20303
+ return "audio/webm";
20304
+ case "aac":
20305
+ return "audio/aac";
20306
+ case "mpga":
20307
+ case "mpeg":
20308
+ return "audio/mpeg";
20309
+ default:
20310
+ throw new Error(
20311
+ `Unable to infer an audio mimetype from filename "${filename}". Supported extensions: mp3, m4a, mp4, wav, ogg, flac, webm, aac, mpga.`
20312
+ );
20313
+ }
20314
+ };
20315
+ var transcribeTool = new ExuluTool({
20316
+ id: "transcribe_audio",
20317
+ name: "Transcribe Audio",
20318
+ description: "Transcribe an audio file (mp3, wav, m4a, etc.) from a URL to text using the configured speech-to-text model. The transcript is stored as a .txt file on S3 and the URL is returned; use this for clips that may be too long to inline in the conversation.",
20319
+ inputSchema: import_zod20.z.object({
20320
+ audio_url: import_zod20.z.string().describe(
20321
+ "Location of the audio file to transcribe. Accepts a publicly fetchable URL (https URL or presigned S3 URL), or a sandbox path such as '/tmp/exulu-sessions/<sessionId>/<file>' or 'file:///tmp/exulu-sessions/<sessionId>/<file>' \u2014 sandbox paths are resolved to their persisted S3 copy."
20322
+ ),
20323
+ language: import_zod20.z.string().optional().describe(
20324
+ "ISO-639-1 language code of the audio, e.g. 'en' or 'de'. Omit to let the model auto-detect."
20325
+ )
20326
+ }),
20327
+ type: "function",
20328
+ config: [{
20329
+ name: "default_language",
20330
+ description: "ISO-639-1 language code of the audio, e.g. 'en' or 'de'. Omit to let the model auto-detect.",
20331
+ type: "string",
20332
+ default: void 0
20333
+ }],
20334
+ execute: async ({ audio_url, language, user, exuluConfig, sessionID }) => {
20335
+ if (!language && exuluConfig?.default_language) {
20336
+ language = exuluConfig?.default_language;
20337
+ } else {
20338
+ language = "en";
20339
+ }
20340
+ language = exuluConfig?.default_language;
20341
+ console.log("[EXULU] Exulu config", exuluConfig);
20342
+ if (!isLiteLLMEnabled()) {
20343
+ console.error("[EXULU] Speech-to-text is not enabled on this deployment (EXULU_USE_LITELLM is not 'true').");
20344
+ throw new Error(
20345
+ "Speech-to-text is not enabled on this deployment (EXULU_USE_LITELLM is not 'true')."
20346
+ );
20347
+ }
20348
+ if (!process.env.TRANSCRIPTION_MODEL) {
20349
+ console.error("[EXULU] TRANSCRIPTION_MODEL env var is not set.");
20350
+ throw new Error("TRANSCRIPTION_MODEL env var is not set.");
20351
+ }
20352
+ if (!exuluConfig?.fileUploads) {
20353
+ console.error("[EXULU] File uploads are not configured; the transcribe tool requires S3 to store transcripts.");
20354
+ throw new Error(
20355
+ "File uploads are not configured; the transcribe tool requires S3 to store transcripts."
20356
+ );
20357
+ }
20358
+ const sandboxPath = parseSandboxPath(audio_url);
20359
+ let buffer;
20360
+ let mimetype;
20361
+ let originalname;
20362
+ if (sandboxPath) {
20363
+ if (!user?.id) {
20364
+ throw new Error(
20365
+ "Sandbox audio paths require an authenticated user; got no user on the tool call."
20366
+ );
20367
+ }
20368
+ if (sessionID && sandboxPath.sessionId !== sessionID) {
20369
+ throw new Error(
20370
+ `Refusing to transcribe an audio file from a different session's sandbox (path session=${sandboxPath.sessionId}, current session=${sessionID}).`
20371
+ );
20372
+ }
20373
+ const rawKey = `user_${user.id}/sessions/${sandboxPath.sessionId}/${sandboxPath.relPath}`;
20374
+ console.log("[EXULU] Transcribing audio from sandbox path", {
20375
+ rawKey
20376
+ });
20377
+ const matches = await listS3ObjectsByPrefix(rawKey, exuluConfig);
20378
+ const found = matches.find((m) => m.key.endsWith(rawKey));
20379
+ if (!found) {
20380
+ console.error("[EXULU] Sandbox audio file not found in S3 storage at", {
20381
+ rawKey,
20382
+ matches
20383
+ });
20384
+ throw new Error(
20385
+ `Sandbox audio file not found in S3 storage at "${rawKey}". The file may not have been persisted yet \u2014 try again after the sandbox flushes it.`
20386
+ );
20387
+ }
20388
+ buffer = await getS3ObjectBytes(found.key, exuluConfig);
20389
+ originalname = decodeURIComponent(
20390
+ sandboxPath.relPath.split("/").pop() || "audio"
20391
+ );
20392
+ mimetype = audioMimetypeFromExtension(originalname);
20393
+ } else {
20394
+ console.log("[EXULU] Fetching audio from URL", {
20395
+ audio_url
20396
+ });
20397
+ const upstream = await fetch(audio_url);
20398
+ if (!upstream.ok) {
20399
+ console.error("[EXULU] Failed to fetch audio from", {
20400
+ audio_url,
20401
+ upstream
20402
+ });
20403
+ throw new Error(
20404
+ `Failed to fetch audio from ${audio_url}: ${upstream.status} ${upstream.statusText}`
20405
+ );
20406
+ }
20407
+ mimetype = upstream.headers.get("content-type") || "audio/mpeg";
20408
+ if (!mimetype.startsWith("audio/")) {
20409
+ throw new Error(
20410
+ `URL did not return an audio file (content-type: ${mimetype}).`
20411
+ );
20412
+ }
20413
+ buffer = Buffer.from(await upstream.arrayBuffer());
20414
+ originalname = "audio";
20415
+ try {
20416
+ const pathname = new URL(audio_url).pathname;
20417
+ const last = pathname.split("/").pop();
20418
+ if (last) originalname = decodeURIComponent(last);
20419
+ } catch {
20420
+ }
20421
+ }
20422
+ const { text } = await transcribeAudio({
20423
+ file: { buffer, originalname, mimetype },
20424
+ language
20425
+ });
20426
+ const transcriptBuffer = Buffer.from(text, "utf-8");
20427
+ const transcriptFilename = `${(0, import_node_crypto8.randomUUID)()}.txt`;
20428
+ const transcriptKey = sessionID ? `sessions/${sessionID}/transcripts/${transcriptFilename}` : `transcripts/${transcriptFilename}`;
20429
+ console.log("[EXULU] Uploading transcript to S3", {
20430
+ transcriptFilename,
20431
+ transcriptKey
20432
+ });
20433
+ const url = await uploadFile(
20434
+ transcriptBuffer,
20435
+ transcriptKey,
20436
+ exuluConfig,
20437
+ { contentType: "text/plain" },
20438
+ user?.id
20439
+ );
20440
+ console.log("[EXULU] Uploaded transcript to S3", {
20441
+ url
20442
+ });
20443
+ let sandboxLocalPath;
20444
+ if (sessionID) {
20445
+ sandboxLocalPath = (0, import_node_path6.join)(
20446
+ SANDBOX_ROOT,
20447
+ sessionID,
20448
+ "transcripts",
20449
+ transcriptFilename
20450
+ );
20451
+ console.log("[EXULU] Mirroring transcript into session sandbox", {
20452
+ sandboxLocalPath
20453
+ });
20454
+ try {
20455
+ await (0, import_promises3.mkdir)((0, import_node_path6.dirname)(sandboxLocalPath), { recursive: true });
20456
+ await (0, import_promises3.writeFile)(sandboxLocalPath, transcriptBuffer);
20457
+ } catch (err) {
20458
+ console.error(
20459
+ `[EXULU] Failed to write transcript to sandbox dir ${sandboxLocalPath}; S3 copy is unaffected.`,
20460
+ err
20461
+ );
20462
+ sandboxLocalPath = void 0;
20463
+ }
20464
+ }
20465
+ console.log("[EXULU] Transcribed audio successfully", {
20466
+ text,
20467
+ url,
20468
+ sandboxLocalPath,
20469
+ length: text.length
20470
+ });
20471
+ return {
20472
+ result: sandboxLocalPath ? `Transcript stored at: ${url} (also available in the session sandbox at ${sandboxLocalPath}, ${text.length} characters).` : `${url}`
20473
+ };
20474
+ }
20475
+ });
20476
+
20477
+ // src/templates/tools/image-generation.ts
20478
+ init_cjs_shims();
20479
+ init_tool();
20480
+ init_supervisor();
20481
+ init_client();
20482
+ init_check_record_access();
20483
+ var import_zod21 = require("zod");
20484
+ var _cachedImageModels;
20485
+ var setCachedImageModels = (models2) => {
20486
+ _cachedImageModels = models2;
20487
+ };
20488
+ var buildDefaults = (models2) => {
20489
+ const m = models2[0];
20490
+ if (!m) {
20491
+ return { model: "", size: "1024x1024", quality: "auto", n: 1 };
20492
+ }
20493
+ return {
20494
+ model: m.model_name,
20495
+ size: m.sizes.includes("1024x1024") ? "1024x1024" : m.sizes[0],
20496
+ quality: m.qualities.includes("auto") ? "auto" : m.qualities[0],
20497
+ n: 1
20498
+ };
20499
+ };
20500
+ var loadAvailableStyles = async (user) => {
20501
+ if (!user?.id) return [];
20502
+ const { db: db2 } = await postgresClient();
20503
+ const rows = await db2.from("platform_configurations").where("config_key", "like", "image_generation_style:%").select("*");
20504
+ const visible = [];
20505
+ for (const row of rows) {
20506
+ const rbac = await RBACResolver(
20507
+ db2,
20508
+ "platform_configurations",
20509
+ row.id,
20510
+ row.rights_mode || "private"
20511
+ );
20512
+ const hasAccess = await checkRecordAccess(
20513
+ { ...row, RBAC: rbac },
20514
+ "read",
20515
+ user
20516
+ );
20517
+ if (!hasAccess) continue;
20518
+ const value = typeof row.config_value === "string" ? safeJsonParse(row.config_value) : row.config_value;
20519
+ visible.push({
20520
+ id: row.id,
20521
+ name: value?.name ?? row.config_key.replace(/^image_generation_style:/, ""),
20522
+ description: row.description ?? null,
20523
+ owner: String(row.created_by) === String(user.id) ? "user" : "shared"
20524
+ });
20525
+ }
20526
+ return visible;
20527
+ };
20528
+ var safeJsonParse = (s) => {
20529
+ try {
20530
+ return JSON.parse(s);
20531
+ } catch {
20532
+ return null;
20533
+ }
20534
+ };
20535
+ var createImageGenerationWidgetTool = (models2) => {
20536
+ setCachedImageModels(models2);
20537
+ return new ExuluTool({
20538
+ id: "image_generation",
20539
+ name: "image_generation",
20540
+ description: "Open an in-chat image generation widget pre-filled with your prompt. The user picks the model, size, quality and count, optionally attaches reference images for editing, applies a saved style, generates one or more candidates, and selects the final image(s) to share back into the conversation. Use this whenever the user asks to create or edit an image.",
20541
+ needsApproval: false,
20542
+ type: "function",
20543
+ config: [],
20544
+ inputSchema: import_zod21.z.object({
20545
+ prompt: import_zod21.z.string().describe(
20546
+ "Initial image prompt. The user can edit it before generating."
20547
+ )
20548
+ }),
20549
+ execute: async ({ prompt, user, sessionID }, options) => {
20550
+ if (!isLiteLLMEnabled()) {
20551
+ throw new Error(
20552
+ "Image generation is not enabled on this deployment (EXULU_USE_LITELLM is not 'true')."
20553
+ );
20554
+ }
20555
+ if (!_cachedImageModels || _cachedImageModels.length === 0) {
20556
+ throw new Error(
20557
+ "No image-generation models are registered in config.litellm.yaml."
20558
+ );
18784
20559
  }
20560
+ const toolCallId = options?.toolCallId;
20561
+ const styles = await loadAvailableStyles(user);
20562
+ return {
20563
+ result: JSON.stringify({
20564
+ type: "image_generation_widget",
20565
+ toolCallId,
20566
+ sessionId: sessionID,
20567
+ initialPrompt: prompt,
20568
+ models: _cachedImageModels.map((m) => ({
20569
+ name: m.model_name,
20570
+ sizes: m.sizes,
20571
+ qualities: m.qualities,
20572
+ supportsEdit: m.supports_edit,
20573
+ maxN: m.max_n
20574
+ })),
20575
+ styles,
20576
+ defaults: buildDefaults(_cachedImageModels)
20577
+ })
20578
+ };
18785
20579
  }
18786
- await sendEmail(recipient, subject, html, text, EMAIL_CONFIG);
18787
- return {
18788
- result: "Email sent successfully to " + recipient + " with subject " + subject + "."
18789
- };
18790
- }
18791
- });
20580
+ });
20581
+ };
20582
+
20583
+ // src/exulu/app/index.ts
20584
+ var import_node_path7 = require("path");
18792
20585
 
18793
20586
  // src/validators/postgres-name.ts
18794
20587
  init_cjs_shims();
@@ -18806,209 +20599,69 @@ init_entitlements();
18806
20599
  init_system_dependencies();
18807
20600
  init_supervisor();
18808
20601
 
18809
- // src/utils/python-setup.ts
20602
+ // src/templates/contexts/index.ts
18810
20603
  init_cjs_shims();
18811
- var import_child_process = require("child_process");
18812
- var import_util = require("util");
18813
- var import_path = require("path");
18814
- var import_fs4 = require("fs");
18815
- var import_url = require("url");
18816
- var execAsync4 = (0, import_util.promisify)(import_child_process.exec);
18817
- function getPackageRoot() {
18818
- const currentFile = (0, import_url.fileURLToPath)(importMetaUrl);
18819
- let currentDir = (0, import_path.dirname)(currentFile);
18820
- let attempts = 0;
18821
- const maxAttempts = 10;
18822
- while (attempts < maxAttempts) {
18823
- const packageJsonPath = (0, import_path.join)(currentDir, "package.json");
18824
- if ((0, import_fs4.existsSync)(packageJsonPath)) {
18825
- try {
18826
- const packageJson = JSON.parse((0, import_fs4.readFileSync)(packageJsonPath, "utf-8"));
18827
- if (packageJson.name === "@exulu/backend") {
18828
- return currentDir;
18829
- }
18830
- } catch {
18831
- }
18832
- }
18833
- const parentDir = (0, import_path.resolve)(currentDir, "..");
18834
- if (parentDir === currentDir) {
18835
- break;
18836
- }
18837
- currentDir = parentDir;
18838
- attempts++;
18839
- }
18840
- const fallback = (0, import_path.resolve)((0, import_path.dirname)((0, import_url.fileURLToPath)(importMetaUrl)), "../..");
18841
- return fallback;
18842
- }
18843
- function getSetupScriptPath(packageRoot) {
18844
- return (0, import_path.resolve)(packageRoot, "ee/python/setup.sh");
18845
- }
18846
- function getVenvPath(packageRoot) {
18847
- return (0, import_path.resolve)(packageRoot, "ee/python/.venv");
18848
- }
18849
- function isPythonEnvironmentSetup(packageRoot) {
18850
- const root = packageRoot ?? getPackageRoot();
18851
- const venvPath = getVenvPath(root);
18852
- const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
18853
- return (0, import_fs4.existsSync)(venvPath) && (0, import_fs4.existsSync)(pythonPath);
18854
- }
18855
- async function setupPythonEnvironment(options = {}) {
18856
- const {
18857
- packageRoot = getPackageRoot(),
18858
- force = false,
18859
- verbose = false,
18860
- timeout = 6e5
18861
- // 10 minutes
18862
- } = options;
18863
- if (!force && isPythonEnvironmentSetup(packageRoot)) {
18864
- if (verbose) {
18865
- console.log("\u2713 Python environment already set up");
18866
- }
18867
- return {
18868
- success: true,
18869
- message: "Python environment already exists",
18870
- alreadyExists: true
18871
- };
18872
- }
18873
- const setupScriptPath = getSetupScriptPath(packageRoot);
18874
- if (!(0, import_fs4.existsSync)(setupScriptPath)) {
18875
- return {
18876
- success: false,
18877
- message: `Setup script not found at: ${setupScriptPath}`,
18878
- alreadyExists: false
18879
- };
18880
- }
18881
- try {
18882
- if (verbose) {
18883
- console.log("Setting up Python environment...");
18884
- }
18885
- const { stdout, stderr } = await execAsync4(`bash "${setupScriptPath}"`, {
18886
- cwd: packageRoot,
18887
- timeout,
18888
- env: {
18889
- ...process.env,
18890
- // Ensure script can write to the directory
18891
- PYTHONDONTWRITEBYTECODE: "1"
18892
- },
18893
- maxBuffer: 10 * 1024 * 1024
18894
- // 10MB buffer
18895
- });
18896
- const output = stdout + stderr;
18897
- const versionMatch = output.match(/Python (\d+\.\d+\.\d+)/);
18898
- const pythonVersion = versionMatch ? versionMatch[1] : void 0;
18899
- if (verbose) {
18900
- console.log(output);
18901
- }
18902
- return {
18903
- success: true,
18904
- message: "Python environment set up successfully",
18905
- alreadyExists: false,
18906
- pythonVersion,
18907
- output
18908
- };
18909
- } catch (error) {
18910
- const errorOutput = error.stdout + error.stderr;
18911
- return {
18912
- success: false,
18913
- message: `Setup failed: ${error.message}`,
18914
- alreadyExists: false,
18915
- output: errorOutput
18916
- };
18917
- }
18918
- }
18919
- function getPythonSetupInstructions() {
18920
- return `
18921
- Python environment not set up. Please run one of the following commands:
18922
-
18923
- Option 1 (Automatic):
18924
- import { setupPythonEnvironment } from '@exulu/backend';
18925
- await setupPythonEnvironment();
18926
-
18927
- Option 2 (Manual - for package consumers):
18928
- npx @exulu/backend setup-python
18929
-
18930
- Option 3 (Manual - for contributors):
18931
- npm run python:setup
18932
-
18933
- These commands will automatically create a Python virtual environment (.venv)
18934
- in the @exulu/backend package and install all required dependencies.
18935
-
18936
- Requirements:
18937
- - Python 3.10 or higher must be installed
18938
- - pip must be available
18939
- - venv module must be available (for creating virtual environments)
18940
-
18941
- If Python dependencies are not installed, install them first, then run one of the commands above:
18942
- - macOS: brew install python@3.12
18943
- - Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
18944
- - Alpine Linux: apk add python3 py3-pip python3-dev
18945
- - Windows: Download from https://www.python.org/downloads/
18946
20604
 
18947
- Note: In Docker containers, ensure you install all three components:
18948
- Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
18949
- Alpine: apk add python3 py3-pip python3-dev
18950
- `.trim();
18951
- }
18952
- async function validatePythonEnvironment(packageRoot, checkPackages = true) {
18953
- const root = packageRoot ?? getPackageRoot();
18954
- const venvPath = getVenvPath(root);
18955
- const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
18956
- if (!(0, import_fs4.existsSync)(venvPath)) {
18957
- return {
18958
- valid: false,
18959
- message: getPythonSetupInstructions()
18960
- };
18961
- }
18962
- if (!(0, import_fs4.existsSync)(pythonPath)) {
18963
- return {
18964
- valid: false,
18965
- message: "Python virtual environment is corrupted. Please run:\n await setupPythonEnvironment({ force: true })"
18966
- };
18967
- }
18968
- try {
18969
- await execAsync4(`"${pythonPath}" --version`, { cwd: root });
18970
- } catch {
18971
- return {
18972
- valid: false,
18973
- message: "Python executable is not working. Please run:\n await setupPythonEnvironment({ force: true })"
18974
- };
20605
+ // src/templates/contexts/transcriptions.ts
20606
+ init_cjs_shims();
20607
+ init_context();
20608
+ var transcriptionsContext = new ExuluContext2({
20609
+ id: "transcriptions",
20610
+ name: "Transcriptions",
20611
+ description: "Diarized audio transcripts",
20612
+ fields: [
20613
+ { name: "transcript_text", type: "longText", editable: true },
20614
+ { name: "audio", type: "file" },
20615
+ { name: "language", type: "text" },
20616
+ { name: "duration_seconds", type: "number" },
20617
+ { name: "speakers", type: "json" },
20618
+ { name: "raw_segments", type: "json", editable: false }
20619
+ ],
20620
+ sources: [],
20621
+ active: true,
20622
+ configuration: {
20623
+ calculateVectors: "onInsert",
20624
+ defaultRightsMode: "private"
18975
20625
  }
18976
- if (checkPackages) {
18977
- const criticalPackages = ["docling", "transformers"];
18978
- const missingPackages = [];
18979
- for (const pkg of criticalPackages) {
18980
- try {
18981
- await execAsync4(`"${pythonPath}" -c "import ${pkg}"`, {
18982
- cwd: root,
18983
- timeout: 1e4
18984
- // 10 second timeout per import check
18985
- });
18986
- } catch {
18987
- missingPackages.push(pkg);
18988
- }
18989
- }
18990
- if (missingPackages.length > 0) {
18991
- return {
18992
- valid: false,
18993
- message: `Python environment exists but required packages are not installed: ${missingPackages.join(", ")}
18994
-
18995
- This usually happens when:
18996
- 1. The .venv folder was copied but dependencies were not installed
18997
- 2. The package was installed via npm but setup script was not run
20626
+ });
18998
20627
 
18999
- Please run:
19000
- await setupPythonEnvironment({ force: true })
20628
+ // src/templates/contexts/index.ts
20629
+ var builtInContexts = {
20630
+ transcriptions: transcriptionsContext
20631
+ };
19001
20632
 
19002
- Or manually run the setup script:
19003
- bash ` + getSetupScriptPath(root)
19004
- };
20633
+ // src/exulu/transcription/polling-loop.ts
20634
+ init_cjs_shims();
20635
+ var POLL_INTERVAL_MS = 5e3;
20636
+ var MAX_PER_TICK = 50;
20637
+ var timer = null;
20638
+ var stopped = false;
20639
+ var tick = async () => {
20640
+ if (stopped) return;
20641
+ try {
20642
+ await transcriptionService.pollOnce(MAX_PER_TICK);
20643
+ } catch (err) {
20644
+ console.error(`[EXULU-TRANSCRIPTION] polling tick failed: ${err.message}`);
20645
+ } finally {
20646
+ if (!stopped) {
20647
+ timer = setTimeout(tick, POLL_INTERVAL_MS);
19005
20648
  }
19006
20649
  }
19007
- return {
19008
- valid: true,
19009
- message: "Python environment is valid"
20650
+ };
20651
+ var startTranscriptionPollingLoop = () => {
20652
+ if (timer) return;
20653
+ stopped = false;
20654
+ timer = setTimeout(tick, POLL_INTERVAL_MS);
20655
+ const stop = () => {
20656
+ stopped = true;
20657
+ if (timer) {
20658
+ clearTimeout(timer);
20659
+ timer = null;
20660
+ }
19010
20661
  };
19011
- }
20662
+ process.on("SIGINT", stop);
20663
+ process.on("SIGTERM", stop);
20664
+ };
19012
20665
 
19013
20666
  // src/exulu/app/index.ts
19014
20667
  var isDev = process.env.NODE_ENV !== "production";
@@ -19074,8 +20727,14 @@ var ExuluApp = class {
19074
20727
  rerankers
19075
20728
  }) => {
19076
20729
  this._evals = redisServer.host?.length && redisServer.port?.length ? [...getDefaultEvals(), ...evals ?? []] : [];
20730
+ if (contexts && "transcriptions" in contexts) {
20731
+ console.warn(
20732
+ "[EXULU] User-defined 'transcriptions' context overridden by built-in. Rename your context to avoid the collision."
20733
+ );
20734
+ }
19077
20735
  this._contexts = {
19078
- ...contexts
20736
+ ...contexts,
20737
+ ...builtInContexts
19079
20738
  };
19080
20739
  this._rerankers = [...rerankers ?? []];
19081
20740
  this._agents = [...agents ?? []];
@@ -19102,12 +20761,30 @@ var ExuluApp = class {
19102
20761
  ...providers ?? []
19103
20762
  ];
19104
20763
  this._config = config;
20764
+ const transcriptionTools = [];
20765
+ if (process.env.TRANSCRIPTION_MODEL && config?.fileUploads && config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
20766
+ transcriptionTools.push(transcribeTool);
20767
+ }
20768
+ const imageGenerationTools = [];
20769
+ const s3Configured = !!config?.fileUploads && !!config.fileUploads.s3region && !!config.fileUploads.s3key && !!config.fileUploads.s3secret && !!config.fileUploads.s3Bucket;
20770
+ if (isLiteLLMEnabled() && s3Configured) {
20771
+ const configPath = process.env.LITELLM_CONFIG_PATH ?? (0, import_node_path7.resolve)(getPackageRoot(), "./config.litellm.yaml");
20772
+ const imageModels = parseImageGenerationModels(configPath);
20773
+ if (imageModels.length > 0) {
20774
+ console.log(
20775
+ `[EXULU] Registering image_generation widget tool with ${imageModels.length} model(s): ${imageModels.map((m) => m.model_name).join(", ")}`
20776
+ );
20777
+ imageGenerationTools.push(createImageGenerationWidgetTool(imageModels));
20778
+ }
20779
+ }
19105
20780
  this._tools = [
19106
20781
  ...tools ?? [],
19107
20782
  ...todoTools,
19108
20783
  ...questionTools,
19109
20784
  ...perplexityTools,
19110
- emailTool
20785
+ emailTool,
20786
+ ...transcriptionTools,
20787
+ ...imageGenerationTools
19111
20788
  // Because agents are stored in the database, we add those as tools
19112
20789
  // at request time, not during ExuluApp initialization. We add them
19113
20790
  // in the grahql tools resolver.
@@ -19206,6 +20883,23 @@ var ExuluApp = class {
19206
20883
  );
19207
20884
  }
19208
20885
  }
20886
+ if (process.env.TRANSCRIPTION_SERVER) {
20887
+ try {
20888
+ const health = await transcriptionClient.health();
20889
+ console.log(
20890
+ `[EXULU] Transcription: enabled (server=${process.env.TRANSCRIPTION_SERVER}, device=${health.device}, GPU=${health.gpu.available ? "enabled" : "disabled"}, diarization=${health.diarization ? "enabled" : "disabled"})`
20891
+ );
20892
+ startTranscriptionPollingLoop();
20893
+ } catch (err) {
20894
+ console.warn(
20895
+ `[EXULU] TRANSCRIPTION_SERVER set but unreachable: ${err.message}. Transcriptions will fail until the server is up.`
20896
+ );
20897
+ }
20898
+ } else {
20899
+ console.log(
20900
+ "[EXULU] Transcription: disabled (TRANSCRIPTION_SERVER not set). Start a whisper server with `npx @exulu/backend exulu-start-whisper`."
20901
+ );
20902
+ }
19209
20903
  return this._expressApp;
19210
20904
  }
19211
20905
  };
@@ -21023,7 +22717,9 @@ var {
21023
22717
  promptLibrarySchema: promptLibrarySchema3,
21024
22718
  contextPresetsSchema: contextPresetsSchema3,
21025
22719
  embedderSettingsSchema: embedderSettingsSchema3,
21026
- promptFavoritesSchema: promptFavoritesSchema3
22720
+ promptFavoritesSchema: promptFavoritesSchema3,
22721
+ transcriptionJobsSchema: transcriptionJobsSchema3,
22722
+ imageGenerationsSchema: imageGenerationsSchema2
21027
22723
  } = coreSchemas.get();
21028
22724
  var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
21029
22725
  for (const field of fields) {
@@ -21063,6 +22759,8 @@ var up = async function(knex) {
21063
22759
  contextPresetsSchema3(),
21064
22760
  embedderSettingsSchema3(),
21065
22761
  promptFavoritesSchema3(),
22762
+ transcriptionJobsSchema3(),
22763
+ imageGenerationsSchema2(),
21066
22764
  rbacSchema3(),
21067
22765
  agentsSchema3(),
21068
22766
  feedbackSchema3(),
@@ -21273,17 +22971,17 @@ init_cjs_shims();
21273
22971
 
21274
22972
  // src/exulu/litellm/db-init.ts
21275
22973
  init_cjs_shims();
21276
- var import_node_fs6 = require("fs");
21277
- var import_node_path5 = require("path");
22974
+ var import_node_fs7 = require("fs");
22975
+ var import_node_path8 = require("path");
21278
22976
  var import_node_child_process5 = require("child_process");
21279
22977
  var import_pg = require("pg");
21280
22978
 
21281
22979
  // src/exulu/litellm/db-setup-check.ts
21282
22980
  init_cjs_shims();
21283
- var import_node_fs5 = require("fs");
22981
+ var import_node_fs6 = require("fs");
21284
22982
  var readLiteLLMDatabaseUrl = (configPath) => {
21285
- if (!(0, import_node_fs5.existsSync)(configPath)) return void 0;
21286
- const text = (0, import_node_fs5.readFileSync)(configPath, "utf8");
22983
+ if (!(0, import_node_fs6.existsSync)(configPath)) return void 0;
22984
+ const text = (0, import_node_fs6.readFileSync)(configPath, "utf8");
21287
22985
  const match = text.match(
21288
22986
  /^\s*database_url:\s*["']?([^"'\n#]+?)["']?\s*(#.*)?$/m
21289
22987
  );
@@ -21342,9 +23040,9 @@ ${WARNING_BANNER}`);
21342
23040
  console.warn(`${WARNING_BANNER}
21343
23041
  `);
21344
23042
  };
21345
- var log2 = (line) => console.log(`[EXULU-LITELLM] ${line}`);
23043
+ var log3 = (line) => console.log(`[EXULU-LITELLM] ${line}`);
21346
23044
  var initLiteLLMDatabase = async (packageRoot) => {
21347
- const configPath = process.env.LITELLM_CONFIG_PATH ?? (0, import_node_path5.resolve)(packageRoot, "./config.litellm.yaml");
23045
+ const configPath = process.env.LITELLM_CONFIG_PATH ?? (0, import_node_path8.resolve)(packageRoot, "./config.litellm.yaml");
21348
23046
  const safety = checkLiteLLMDatabaseSafety(configPath);
21349
23047
  if (safety.ok && safety.reason === "no-litellm-db-mode") return;
21350
23048
  if (!safety.ok && safety.reason === "unparseable-url") {
@@ -21376,7 +23074,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
21376
23074
  return;
21377
23075
  }
21378
23076
  const target = "litellmTarget" in safety ? safety.litellmTarget : void 0;
21379
- log2(
23077
+ log3(
21380
23078
  `LiteLLM database mode detected (${target?.host}:${target?.port}/${target?.database}).`
21381
23079
  );
21382
23080
  const ensureDatabaseExists2 = async () => {
@@ -21404,7 +23102,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
21404
23102
  return false;
21405
23103
  }
21406
23104
  url.pathname = "/postgres";
21407
- log2(`Target database "${targetDbName}" does not exist; creating it\u2026`);
23105
+ log3(`Target database "${targetDbName}" does not exist; creating it\u2026`);
21408
23106
  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(targetDbName)) {
21409
23107
  warn([
21410
23108
  `Refusing to auto-create database "${targetDbName}" \u2014 name`,
@@ -21417,7 +23115,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
21417
23115
  try {
21418
23116
  await admin.connect();
21419
23117
  await admin.query(`CREATE DATABASE "${targetDbName}"`);
21420
- log2(`\u2713 Created database "${targetDbName}".`);
23118
+ log3(`\u2713 Created database "${targetDbName}".`);
21421
23119
  return true;
21422
23120
  } catch (createErr) {
21423
23121
  warn([
@@ -21439,7 +23137,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
21439
23137
  }
21440
23138
  };
21441
23139
  if (!await ensureDatabaseExists2()) return;
21442
- log2("Checking that the target database is safe to push into\u2026");
23140
+ log3("Checking that the target database is safe to push into\u2026");
21443
23141
  const client2 = new import_pg.Client({ connectionString: litellmUrl });
21444
23142
  let foreignTables = [];
21445
23143
  try {
@@ -21482,14 +23180,14 @@ var initLiteLLMDatabase = async (packageRoot) => {
21482
23180
  ]);
21483
23181
  return;
21484
23182
  }
21485
- const venvBin = (0, import_node_path5.resolve)(packageRoot, "ee/python/.venv/bin");
21486
- const prismaCli = (0, import_node_path5.resolve)(venvBin, "prisma");
21487
- const litellmProxyDir = (0, import_node_path5.resolve)(
23183
+ const venvBin = (0, import_node_path8.resolve)(packageRoot, "ee/python/.venv/bin");
23184
+ const prismaCli = (0, import_node_path8.resolve)(venvBin, "prisma");
23185
+ const litellmProxyDir = (0, import_node_path8.resolve)(
21488
23186
  packageRoot,
21489
23187
  "ee/python/.venv/lib/python3.12/site-packages/litellm/proxy"
21490
23188
  );
21491
- const schemaPath = (0, import_node_path5.resolve)(litellmProxyDir, "schema.prisma");
21492
- if (!(0, import_node_fs6.existsSync)(prismaCli)) {
23189
+ const schemaPath = (0, import_node_path8.resolve)(litellmProxyDir, "schema.prisma");
23190
+ if (!(0, import_node_fs7.existsSync)(prismaCli)) {
21493
23191
  warn([
21494
23192
  `Prisma CLI not found at ${prismaCli}.`,
21495
23193
  `Run \`npm run python:setup\` to create the venv and install prisma.`,
@@ -21497,14 +23195,14 @@ var initLiteLLMDatabase = async (packageRoot) => {
21497
23195
  ]);
21498
23196
  return;
21499
23197
  }
21500
- if (!(0, import_node_fs6.existsSync)(schemaPath)) {
23198
+ if (!(0, import_node_fs7.existsSync)(schemaPath)) {
21501
23199
  warn([
21502
23200
  `LiteLLM Prisma schema not found at ${schemaPath}.`,
21503
23201
  `Re-run \`npm run python:setup\`. Skipping LiteLLM database setup.`
21504
23202
  ]);
21505
23203
  return;
21506
23204
  }
21507
- log2("Running `prisma db push` against LiteLLM's schema\u2026");
23205
+ log3("Running `prisma db push` against LiteLLM's schema\u2026");
21508
23206
  const result = (0, import_node_child_process5.spawnSync)(prismaCli, ["db", "push", "--skip-generate"], {
21509
23207
  cwd: litellmProxyDir,
21510
23208
  env: {
@@ -21534,7 +23232,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
21534
23232
  ]);
21535
23233
  return;
21536
23234
  }
21537
- log2("\u2713 LiteLLM database ready.");
23235
+ log3("\u2713 LiteLLM database ready.");
21538
23236
  };
21539
23237
 
21540
23238
  // src/postgres/init-litellm-db.ts
@@ -22078,7 +23776,7 @@ init_cjs_shims();
22078
23776
  var fs5 = __toESM(require("fs"), 1);
22079
23777
  var path2 = __toESM(require("path"), 1);
22080
23778
  var import_ai14 = require("ai");
22081
- var import_zod20 = require("zod");
23779
+ var import_zod22 = require("zod");
22082
23780
  var import_p_limit = __toESM(require("p-limit"), 1);
22083
23781
  var import_crypto = require("crypto");
22084
23782
  init_with_retry();
@@ -22433,15 +24131,15 @@ If the page contains a flow-chart, schematic, technical drawing or control board
22433
24131
  const result = await (0, import_ai14.generateText)({
22434
24132
  model,
22435
24133
  output: import_ai14.Output.object({
22436
- schema: import_zod20.z.object({
22437
- needs_correction: import_zod20.z.boolean(),
22438
- corrected_text: import_zod20.z.string().nullable(),
22439
- current_page_table: import_zod20.z.object({
22440
- headers: import_zod20.z.array(import_zod20.z.string()),
22441
- is_continuation: import_zod20.z.boolean()
24134
+ schema: import_zod22.z.object({
24135
+ needs_correction: import_zod22.z.boolean(),
24136
+ corrected_text: import_zod22.z.string().nullable(),
24137
+ current_page_table: import_zod22.z.object({
24138
+ headers: import_zod22.z.array(import_zod22.z.string()),
24139
+ is_continuation: import_zod22.z.boolean()
22442
24140
  }).nullable(),
22443
- confidence: import_zod20.z.enum(["high", "medium", "low"]),
22444
- reasoning: import_zod20.z.string()
24141
+ confidence: import_zod22.z.enum(["high", "medium", "low"]),
24142
+ reasoning: import_zod22.z.string()
22445
24143
  })
22446
24144
  }),
22447
24145
  messages: [
@@ -22521,7 +24219,7 @@ async function validateWithVLM(document2, model, verbose = false, concurrency =
22521
24219
  let correctedCount = 0;
22522
24220
  const validationTasks = document2.map(
22523
24221
  (page) => limit(async () => {
22524
- await new Promise((resolve6) => setImmediate(resolve6));
24222
+ await new Promise((resolve7) => setImmediate(resolve7));
22525
24223
  const imagePath = page.image;
22526
24224
  if (!imagePath) {
22527
24225
  console.warn(`[EXULU] Page ${page.page}: No image found, skipping validation`);
@@ -22712,7 +24410,7 @@ ${setupResult.output || ""}`);
22712
24410
  if (!MISTRAL_API_KEY) {
22713
24411
  throw new Error('[EXULU] MISTRAL_API_KEY is not set, please set it in the environment variable via process.env or via an Exulu variable named "MISTRAL_API_KEY".');
22714
24412
  }
22715
- await new Promise((resolve6) => setTimeout(resolve6, Math.floor(Math.random() * 4e3) + 1e3));
24413
+ await new Promise((resolve7) => setTimeout(resolve7, Math.floor(Math.random() * 4e3) + 1e3));
22716
24414
  const base64Pdf = buffer.toString("base64");
22717
24415
  const client2 = new import_mistralai.Mistral({ apiKey: MISTRAL_API_KEY });
22718
24416
  const ocrResponse = await withRetry(async () => {
@@ -22808,8 +24506,8 @@ ${setupResult.output || ""}`);
22808
24506
  markdownStream.write("\n\n\n<!-- END_OF_PAGE -->\n\n\n");
22809
24507
  }
22810
24508
  }
22811
- await new Promise((resolve6, reject) => {
22812
- markdownStream.end(() => resolve6());
24509
+ await new Promise((resolve7, reject) => {
24510
+ markdownStream.end(() => resolve7());
22813
24511
  markdownStream.on("error", reject);
22814
24512
  });
22815
24513
  console.log(`[EXULU] Validated output saved to: ${paths.json}`);