@exulu/backend 1.60.0 → 1.61.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/backend.cjs +60 -0
- package/dist/{catalog-EOKGOHTY.js → catalog-BWE6SLE2.js} +1 -1
- package/dist/chunk-IDHS2BZO.js +210 -0
- package/dist/{chunk-YS27XOXI.js → chunk-ILAHW4UT.js} +5 -1
- package/dist/{chunk-23YNGK3V.js → chunk-MPV7HBV6.js} +63 -2
- package/dist/cli/start-whisper.cjs +240 -0
- package/dist/cli/start-whisper.d.cts +1 -0
- package/dist/cli/start-whisper.d.ts +1 -0
- package/dist/cli/start-whisper.js +204 -0
- package/dist/{convert-exulu-tools-to-ai-sdk-tools-PLLM2CJL.js → convert-exulu-tools-to-ai-sdk-tools-CULC37U6.js} +1 -1
- package/dist/index.cjs +1827 -346
- package/dist/index.d.cts +2 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1447 -249
- package/ee/python/requirements.txt +18 -0
- package/ee/python/setup.sh +44 -0
- package/ee/python/transcription/__init__.py +0 -0
- package/ee/python/transcription/pipeline.py +232 -0
- package/ee/python/transcription/server.py +151 -0
- package/ee/python/transcription/tests/__init__.py +0 -0
- package/ee/python/transcription/tests/test_server.py +111 -0
- package/ee/python/transcription/worker.py +135 -0
- package/package.json +5 -2
package/dist/index.js
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getPackageRoot,
|
|
3
|
+
getPythonSetupInstructions,
|
|
4
|
+
isPythonEnvironmentSetup,
|
|
5
|
+
setupPythonEnvironment,
|
|
6
|
+
validatePythonEnvironment
|
|
7
|
+
} from "./chunk-IDHS2BZO.js";
|
|
1
8
|
import {
|
|
2
9
|
ExuluContext,
|
|
3
10
|
ExuluStorage,
|
|
@@ -46,10 +53,10 @@ import {
|
|
|
46
53
|
vectorSearch,
|
|
47
54
|
waitForLiteLLMReady,
|
|
48
55
|
withRetry
|
|
49
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-MPV7HBV6.js";
|
|
50
57
|
import {
|
|
51
58
|
findLiteLLMModel
|
|
52
|
-
} from "./chunk-
|
|
59
|
+
} from "./chunk-ILAHW4UT.js";
|
|
53
60
|
|
|
54
61
|
// src/index.ts
|
|
55
62
|
import "dotenv/config";
|
|
@@ -3358,6 +3365,365 @@ function getAverage(arr) {
|
|
|
3358
3365
|
|
|
3359
3366
|
// src/graphql/schemas/index.ts
|
|
3360
3367
|
import "fs";
|
|
3368
|
+
|
|
3369
|
+
// src/exulu/transcription/client.ts
|
|
3370
|
+
var TranscriptionServerUnavailable = class extends Error {
|
|
3371
|
+
constructor(message) {
|
|
3372
|
+
super(message);
|
|
3373
|
+
this.name = "TranscriptionServerUnavailable";
|
|
3374
|
+
}
|
|
3375
|
+
};
|
|
3376
|
+
var getBaseUrl = () => {
|
|
3377
|
+
const url = process.env.TRANSCRIPTION_SERVER;
|
|
3378
|
+
if (!url) {
|
|
3379
|
+
throw new TranscriptionServerUnavailable(
|
|
3380
|
+
"TRANSCRIPTION_SERVER env var is not set. Start a whisper server with `npx @exulu/backend exulu-start-whisper` and point TRANSCRIPTION_SERVER at it."
|
|
3381
|
+
);
|
|
3382
|
+
}
|
|
3383
|
+
return url.replace(/\/$/, "");
|
|
3384
|
+
};
|
|
3385
|
+
var request = async (path2, init = {}) => {
|
|
3386
|
+
const url = `${getBaseUrl()}${path2}`;
|
|
3387
|
+
let res;
|
|
3388
|
+
try {
|
|
3389
|
+
res = await fetch(url, init);
|
|
3390
|
+
} catch (err) {
|
|
3391
|
+
throw new TranscriptionServerUnavailable(
|
|
3392
|
+
`Unable to reach whisper server at ${url}: ${err.message}`
|
|
3393
|
+
);
|
|
3394
|
+
}
|
|
3395
|
+
if (res.status === 404) {
|
|
3396
|
+
const err = new Error(`whisper server returned 404 for ${path2}`);
|
|
3397
|
+
err.code = "JOB_NOT_FOUND";
|
|
3398
|
+
throw err;
|
|
3399
|
+
}
|
|
3400
|
+
if (!res.ok) {
|
|
3401
|
+
throw new Error(
|
|
3402
|
+
`whisper server returned ${res.status} for ${path2}: ${await res.text()}`
|
|
3403
|
+
);
|
|
3404
|
+
}
|
|
3405
|
+
return await res.json();
|
|
3406
|
+
};
|
|
3407
|
+
var transcriptionClient = {
|
|
3408
|
+
submitJob: (opts) => request("/jobs", {
|
|
3409
|
+
method: "POST",
|
|
3410
|
+
headers: { "content-type": "application/json" },
|
|
3411
|
+
body: JSON.stringify(opts)
|
|
3412
|
+
}),
|
|
3413
|
+
getJob: (jobId) => request(`/jobs/${jobId}`),
|
|
3414
|
+
cancelJob: (jobId) => request(`/jobs/${jobId}`, {
|
|
3415
|
+
method: "DELETE"
|
|
3416
|
+
}),
|
|
3417
|
+
health: () => request("/healthz"),
|
|
3418
|
+
isConfigured: () => Boolean(process.env.TRANSCRIPTION_SERVER)
|
|
3419
|
+
};
|
|
3420
|
+
|
|
3421
|
+
// src/exulu/transcription/transcript-text.ts
|
|
3422
|
+
var renderTranscript = (segments, speakers) => {
|
|
3423
|
+
if (!segments || segments.length === 0) return "";
|
|
3424
|
+
const blocks = [];
|
|
3425
|
+
for (const seg of segments) {
|
|
3426
|
+
const text = (seg.text ?? "").trim();
|
|
3427
|
+
if (!text) continue;
|
|
3428
|
+
const label = speakers[seg.speaker] ?? seg.speaker ?? "unknown";
|
|
3429
|
+
const last = blocks[blocks.length - 1];
|
|
3430
|
+
if (last && last.speaker === label) {
|
|
3431
|
+
last.text = `${last.text} ${text}`.trim();
|
|
3432
|
+
} else {
|
|
3433
|
+
blocks.push({ speaker: label, text });
|
|
3434
|
+
}
|
|
3435
|
+
}
|
|
3436
|
+
return blocks.map((b) => `${b.speaker}: ${b.text}`).join("\n");
|
|
3437
|
+
};
|
|
3438
|
+
|
|
3439
|
+
// src/exulu/transcription/service.ts
|
|
3440
|
+
var TABLE = "transcription_jobs";
|
|
3441
|
+
var log = (msg) => console.log(`[EXULU-TRANSCRIPTION] ${msg}`);
|
|
3442
|
+
var parseJsonField = (v) => {
|
|
3443
|
+
if (v == null) return null;
|
|
3444
|
+
if (typeof v === "string") {
|
|
3445
|
+
try {
|
|
3446
|
+
return JSON.parse(v);
|
|
3447
|
+
} catch {
|
|
3448
|
+
return null;
|
|
3449
|
+
}
|
|
3450
|
+
}
|
|
3451
|
+
return v;
|
|
3452
|
+
};
|
|
3453
|
+
var presignAudio = async (s3Key) => {
|
|
3454
|
+
const app = exuluApp.get();
|
|
3455
|
+
const config = app._config ?? app.config;
|
|
3456
|
+
const configuredBucket = config?.fileUploads?.s3Bucket;
|
|
3457
|
+
if (!configuredBucket) {
|
|
3458
|
+
throw new Error("File uploads are not configured (s3Bucket missing).");
|
|
3459
|
+
}
|
|
3460
|
+
const firstSlash = s3Key.indexOf("/");
|
|
3461
|
+
const bucket = firstSlash > 0 ? s3Key.slice(0, firstSlash) : configuredBucket;
|
|
3462
|
+
const objectKey = firstSlash > 0 ? s3Key.slice(firstSlash + 1) : s3Key;
|
|
3463
|
+
return getPresignedUrl(bucket, objectKey, config);
|
|
3464
|
+
};
|
|
3465
|
+
var transcriptionService = {
|
|
3466
|
+
/**
|
|
3467
|
+
* Create a transcription job row and dispatch it to the whisper server.
|
|
3468
|
+
* Throws TranscriptionServerUnavailable if the feature is off.
|
|
3469
|
+
*/
|
|
3470
|
+
async startJob(input) {
|
|
3471
|
+
if (!transcriptionClient.isConfigured()) {
|
|
3472
|
+
throw new TranscriptionServerUnavailable(
|
|
3473
|
+
"TRANSCRIPTION_SERVER is not set. Start a whisper server with `npx @exulu/backend exulu-start-whisper` and point TRANSCRIPTION_SERVER at it."
|
|
3474
|
+
);
|
|
3475
|
+
}
|
|
3476
|
+
const { db } = await postgresClient();
|
|
3477
|
+
const now = /* @__PURE__ */ new Date();
|
|
3478
|
+
const [inserted] = await db(TABLE).insert({
|
|
3479
|
+
audio_s3key: input.s3Key,
|
|
3480
|
+
title: input.title ?? input.filename,
|
|
3481
|
+
status: "queued",
|
|
3482
|
+
project_id: input.project_id ?? null,
|
|
3483
|
+
target_rights_mode: input.target_rights_mode ?? "private",
|
|
3484
|
+
target_rbac_users: input.target_rbac_users ? JSON.stringify(input.target_rbac_users) : null,
|
|
3485
|
+
target_rbac_roles: input.target_rbac_roles ? JSON.stringify(input.target_rbac_roles) : null,
|
|
3486
|
+
rights_mode: "private",
|
|
3487
|
+
created_by: input.userId,
|
|
3488
|
+
createdAt: now,
|
|
3489
|
+
updatedAt: now
|
|
3490
|
+
}).returning("*");
|
|
3491
|
+
const row = this._rowFromDb(inserted);
|
|
3492
|
+
try {
|
|
3493
|
+
const audioUrl = await presignAudio(input.s3Key);
|
|
3494
|
+
const submitted = await transcriptionClient.submitJob({
|
|
3495
|
+
audio_url: audioUrl,
|
|
3496
|
+
language: input.language ?? void 0,
|
|
3497
|
+
num_speakers: input.num_speakers ?? void 0,
|
|
3498
|
+
hotwords: input.hotwords
|
|
3499
|
+
});
|
|
3500
|
+
const [updated] = await db(TABLE).where({ id: row.id }).update({
|
|
3501
|
+
whisper_job_id: submitted.job_id,
|
|
3502
|
+
status: "transcribing",
|
|
3503
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3504
|
+
}).returning("*");
|
|
3505
|
+
return this._rowFromDb(updated);
|
|
3506
|
+
} catch (err) {
|
|
3507
|
+
const [failed] = await db(TABLE).where({ id: row.id }).update({
|
|
3508
|
+
status: "failed",
|
|
3509
|
+
error: err.message,
|
|
3510
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3511
|
+
}).returning("*");
|
|
3512
|
+
log(`Failed to dispatch job ${row.id}: ${err.message}`);
|
|
3513
|
+
return this._rowFromDb(failed);
|
|
3514
|
+
}
|
|
3515
|
+
},
|
|
3516
|
+
/**
|
|
3517
|
+
* Reconcile every transcribing row against the whisper server. Called from
|
|
3518
|
+
* the polling loop on a fixed interval. Caps how many rows we touch per
|
|
3519
|
+
* tick so a backlog can't starve the event loop.
|
|
3520
|
+
*/
|
|
3521
|
+
async pollOnce(maxPerTick = 50) {
|
|
3522
|
+
if (!transcriptionClient.isConfigured()) return;
|
|
3523
|
+
const { db } = await postgresClient();
|
|
3524
|
+
const rows = await db(TABLE).where({ status: "transcribing" }).whereNotNull("whisper_job_id").limit(maxPerTick);
|
|
3525
|
+
for (const dbRow of rows) {
|
|
3526
|
+
const row = this._rowFromDb(dbRow);
|
|
3527
|
+
if (!row.whisper_job_id) continue;
|
|
3528
|
+
try {
|
|
3529
|
+
const job = await transcriptionClient.getJob(row.whisper_job_id);
|
|
3530
|
+
await this._applyJobUpdate(row, job);
|
|
3531
|
+
} catch (err) {
|
|
3532
|
+
const code = err.code;
|
|
3533
|
+
if (code === "JOB_NOT_FOUND") {
|
|
3534
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3535
|
+
status: "failed",
|
|
3536
|
+
error: "lost on server restart",
|
|
3537
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3538
|
+
});
|
|
3539
|
+
} else if (err instanceof TranscriptionServerUnavailable) {
|
|
3540
|
+
log(`Whisper server unreachable while polling ${row.id}; will retry`);
|
|
3541
|
+
} else {
|
|
3542
|
+
log(`Error polling job ${row.id}: ${err.message}`);
|
|
3543
|
+
}
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
},
|
|
3547
|
+
async _applyJobUpdate(row, job) {
|
|
3548
|
+
const { db } = await postgresClient();
|
|
3549
|
+
if ((job.status === "queued" || job.status === "running") && job.duration_seconds != null && row.duration_seconds !== job.duration_seconds) {
|
|
3550
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3551
|
+
duration_seconds: job.duration_seconds,
|
|
3552
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3553
|
+
});
|
|
3554
|
+
}
|
|
3555
|
+
if (job.status === "running" || job.status === "queued") return;
|
|
3556
|
+
if (job.status === "completed") {
|
|
3557
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3558
|
+
status: "awaiting_review",
|
|
3559
|
+
raw_segments: JSON.stringify(job.segments ?? []),
|
|
3560
|
+
language: job.language ?? null,
|
|
3561
|
+
duration_seconds: job.duration_seconds ?? null,
|
|
3562
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3563
|
+
});
|
|
3564
|
+
return;
|
|
3565
|
+
}
|
|
3566
|
+
if (job.status === "failed") {
|
|
3567
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3568
|
+
status: "failed",
|
|
3569
|
+
error: job.error ?? "transcription failed",
|
|
3570
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3571
|
+
});
|
|
3572
|
+
return;
|
|
3573
|
+
}
|
|
3574
|
+
if (job.status === "cancelled") {
|
|
3575
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3576
|
+
status: "cancelled",
|
|
3577
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3578
|
+
});
|
|
3579
|
+
}
|
|
3580
|
+
},
|
|
3581
|
+
async cancelJob(id) {
|
|
3582
|
+
const { db } = await postgresClient();
|
|
3583
|
+
const dbRow = await db(TABLE).where({ id }).first();
|
|
3584
|
+
if (!dbRow) throw new Error(`transcription_job ${id} not found`);
|
|
3585
|
+
const row = this._rowFromDb(dbRow);
|
|
3586
|
+
if (row.whisper_job_id && transcriptionClient.isConfigured()) {
|
|
3587
|
+
try {
|
|
3588
|
+
await transcriptionClient.cancelJob(row.whisper_job_id);
|
|
3589
|
+
} catch (err) {
|
|
3590
|
+
const code = err.code;
|
|
3591
|
+
if (code !== "JOB_NOT_FOUND") {
|
|
3592
|
+
log(`Best-effort cancel of whisper job failed: ${err.message}`);
|
|
3593
|
+
}
|
|
3594
|
+
}
|
|
3595
|
+
}
|
|
3596
|
+
const [updated] = await db(TABLE).where({ id }).update({ status: "cancelled", updatedAt: /* @__PURE__ */ new Date() }).returning("*");
|
|
3597
|
+
return this._rowFromDb(updated);
|
|
3598
|
+
},
|
|
3599
|
+
/**
|
|
3600
|
+
* User clicked Save in the review panel.
|
|
3601
|
+
*
|
|
3602
|
+
* - From 'awaiting_review': render the speaker-labeled transcript, create a
|
|
3603
|
+
* new ExuluContext item, apply RBAC + optional project linkage, mark the
|
|
3604
|
+
* job saved.
|
|
3605
|
+
* - From 'saved': re-render the transcript with the (possibly updated)
|
|
3606
|
+
* speaker map and upsert the existing context item by id. Used by the
|
|
3607
|
+
* Completed-section re-edit flow.
|
|
3608
|
+
*/
|
|
3609
|
+
async finalize(id, input) {
|
|
3610
|
+
const { db } = await postgresClient();
|
|
3611
|
+
const dbRow = await db(TABLE).where({ id }).first();
|
|
3612
|
+
if (!dbRow) throw new Error(`transcription_job ${id} not found`);
|
|
3613
|
+
const row = this._rowFromDb(dbRow);
|
|
3614
|
+
if (row.status !== "awaiting_review" && row.status !== "saved") {
|
|
3615
|
+
throw new Error(
|
|
3616
|
+
`transcription_job ${id} is in status '${row.status}'; can only finalize from 'awaiting_review' or 'saved'`
|
|
3617
|
+
);
|
|
3618
|
+
}
|
|
3619
|
+
if (!row.raw_segments) {
|
|
3620
|
+
throw new Error(`transcription_job ${id} has no raw_segments to finalize`);
|
|
3621
|
+
}
|
|
3622
|
+
const app = exuluApp.get();
|
|
3623
|
+
const context = app.context("transcriptions");
|
|
3624
|
+
if (!context) {
|
|
3625
|
+
throw new Error("Built-in transcriptions context not registered");
|
|
3626
|
+
}
|
|
3627
|
+
const config = app._config ?? app.config;
|
|
3628
|
+
const transcriptText = renderTranscript(row.raw_segments, input.speakers);
|
|
3629
|
+
const rightsMode = input.target_rights_mode ?? row.target_rights_mode ?? "private";
|
|
3630
|
+
const isReSave = row.status === "saved" && !!row.saved_item_id;
|
|
3631
|
+
const itemInput = {
|
|
3632
|
+
// Carrying the id on re-save makes context.createItem upsert in place.
|
|
3633
|
+
...isReSave && row.saved_item_id ? { id: row.saved_item_id } : {},
|
|
3634
|
+
name: input.title ?? row.title ?? "Transcript",
|
|
3635
|
+
transcript_text: transcriptText,
|
|
3636
|
+
audio_s3key: row.audio_s3key,
|
|
3637
|
+
language: row.language ?? void 0,
|
|
3638
|
+
duration_seconds: row.duration_seconds ?? void 0,
|
|
3639
|
+
speakers: input.speakers,
|
|
3640
|
+
raw_segments: row.raw_segments,
|
|
3641
|
+
rights_mode: rightsMode,
|
|
3642
|
+
created_by: row.created_by
|
|
3643
|
+
};
|
|
3644
|
+
let item;
|
|
3645
|
+
try {
|
|
3646
|
+
const result = await context.createItem(
|
|
3647
|
+
itemInput,
|
|
3648
|
+
config,
|
|
3649
|
+
row.created_by,
|
|
3650
|
+
void 0,
|
|
3651
|
+
isReSave
|
|
3652
|
+
// upsert when re-saving
|
|
3653
|
+
);
|
|
3654
|
+
item = result.item;
|
|
3655
|
+
} catch (err) {
|
|
3656
|
+
await db(TABLE).where({ id }).update({
|
|
3657
|
+
speakers: JSON.stringify(input.speakers),
|
|
3658
|
+
error: `Failed to save: ${err.message}`,
|
|
3659
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3660
|
+
});
|
|
3661
|
+
throw err;
|
|
3662
|
+
}
|
|
3663
|
+
const itemId = item.id ?? row.saved_item_id ?? "";
|
|
3664
|
+
const users = input.target_rbac_users ?? row.target_rbac_users ?? [];
|
|
3665
|
+
const roles = input.target_rbac_roles ?? row.target_rbac_roles ?? [];
|
|
3666
|
+
if ((users.length || roles.length) && rightsMode !== "private") {
|
|
3667
|
+
try {
|
|
3668
|
+
await handleRBACUpdate(
|
|
3669
|
+
db,
|
|
3670
|
+
"transcriptions",
|
|
3671
|
+
itemId,
|
|
3672
|
+
{ users, roles },
|
|
3673
|
+
[]
|
|
3674
|
+
);
|
|
3675
|
+
} catch (err) {
|
|
3676
|
+
log(`RBAC update failed for item ${itemId}: ${err.message}`);
|
|
3677
|
+
}
|
|
3678
|
+
}
|
|
3679
|
+
const projectId = input.project_id ?? row.project_id ?? null;
|
|
3680
|
+
let projectWarning = null;
|
|
3681
|
+
if (projectId && !isReSave) {
|
|
3682
|
+
try {
|
|
3683
|
+
const project = await db("projects").where({ id: projectId }).first();
|
|
3684
|
+
if (!project) {
|
|
3685
|
+
projectWarning = `project ${projectId} not found`;
|
|
3686
|
+
} else {
|
|
3687
|
+
const existing = parseJsonField(project.project_items) ?? [];
|
|
3688
|
+
const globalId = `transcriptions/${itemId}`;
|
|
3689
|
+
if (!existing.includes(globalId)) {
|
|
3690
|
+
existing.push(globalId);
|
|
3691
|
+
await db("projects").where({ id: projectId }).update({
|
|
3692
|
+
project_items: JSON.stringify(existing),
|
|
3693
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3694
|
+
});
|
|
3695
|
+
}
|
|
3696
|
+
}
|
|
3697
|
+
} catch (err) {
|
|
3698
|
+
projectWarning = err.message;
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
3701
|
+
const [updated] = await db(TABLE).where({ id }).update({
|
|
3702
|
+
status: "saved",
|
|
3703
|
+
saved_item_id: itemId,
|
|
3704
|
+
title: input.title ?? row.title ?? null,
|
|
3705
|
+
speakers: JSON.stringify(input.speakers),
|
|
3706
|
+
error: projectWarning ? `Saved, but could not attach to project: ${projectWarning}` : null,
|
|
3707
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3708
|
+
}).returning("*");
|
|
3709
|
+
return { item, row: this._rowFromDb(updated) };
|
|
3710
|
+
},
|
|
3711
|
+
_rowFromDb(dbRow) {
|
|
3712
|
+
return {
|
|
3713
|
+
...dbRow,
|
|
3714
|
+
raw_segments: parseJsonField(dbRow.raw_segments),
|
|
3715
|
+
speakers: parseJsonField(dbRow.speakers),
|
|
3716
|
+
target_rbac_users: parseJsonField(
|
|
3717
|
+
dbRow.target_rbac_users
|
|
3718
|
+
),
|
|
3719
|
+
target_rbac_roles: parseJsonField(
|
|
3720
|
+
dbRow.target_rbac_roles
|
|
3721
|
+
)
|
|
3722
|
+
};
|
|
3723
|
+
}
|
|
3724
|
+
};
|
|
3725
|
+
|
|
3726
|
+
// src/graphql/schemas/index.ts
|
|
3361
3727
|
function createExuluContextsTypeDefs(table) {
|
|
3362
3728
|
const enumDefs = table.fields.filter((field) => field.type === "enum" && field.enumValues).map((field) => {
|
|
3363
3729
|
if (!field.enumValues) {
|
|
@@ -3775,6 +4141,39 @@ type PageInfo {
|
|
|
3775
4141
|
mutationDefs += `
|
|
3776
4142
|
deleteJob(queue: QueueEnum!, id: String!): JobActionReturnPayload
|
|
3777
4143
|
`;
|
|
4144
|
+
mutationDefs += `
|
|
4145
|
+
transcriptionJobStart(input: TranscriptionJobStartInput!): transcription_job
|
|
4146
|
+
transcriptionJobFinalize(id: ID!, input: TranscriptionJobFinalizeInput!): TranscriptionJobFinalizeResult
|
|
4147
|
+
transcriptionJobCancel(id: ID!): transcription_job
|
|
4148
|
+
`;
|
|
4149
|
+
modelDefs += `
|
|
4150
|
+
input TranscriptionJobStartInput {
|
|
4151
|
+
audio_s3key: String!
|
|
4152
|
+
filename: String!
|
|
4153
|
+
title: String
|
|
4154
|
+
language: String
|
|
4155
|
+
num_speakers: Int
|
|
4156
|
+
hotwords: [String!]
|
|
4157
|
+
project_id: ID
|
|
4158
|
+
target_rights_mode: String
|
|
4159
|
+
target_rbac_users: [RBACUserInput!]
|
|
4160
|
+
target_rbac_roles: [RBACRoleInput!]
|
|
4161
|
+
}
|
|
4162
|
+
|
|
4163
|
+
input TranscriptionJobFinalizeInput {
|
|
4164
|
+
title: String
|
|
4165
|
+
speakers: JSON!
|
|
4166
|
+
project_id: ID
|
|
4167
|
+
target_rights_mode: String
|
|
4168
|
+
target_rbac_users: [RBACUserInput!]
|
|
4169
|
+
target_rbac_roles: [RBACRoleInput!]
|
|
4170
|
+
}
|
|
4171
|
+
|
|
4172
|
+
type TranscriptionJobFinalizeResult {
|
|
4173
|
+
job: transcription_job!
|
|
4174
|
+
item_id: ID!
|
|
4175
|
+
}
|
|
4176
|
+
`;
|
|
3778
4177
|
typeDefs += `
|
|
3779
4178
|
tools(search: String, category: String, limit: Int, page: Int): ToolPaginationResult
|
|
3780
4179
|
toolCategories: [String!]!
|
|
@@ -3865,7 +4264,7 @@ type LiteLLMModel {
|
|
|
3865
4264
|
};
|
|
3866
4265
|
};
|
|
3867
4266
|
resolvers.Query["litellmCatalog"] = async () => {
|
|
3868
|
-
const { fetchLiteLLMCatalog } = await import("./catalog-
|
|
4267
|
+
const { fetchLiteLLMCatalog } = await import("./catalog-BWE6SLE2.js");
|
|
3869
4268
|
return fetchLiteLLMCatalog();
|
|
3870
4269
|
};
|
|
3871
4270
|
resolvers.Query["workflowSchedule"] = async (_, args, context, info) => {
|
|
@@ -4340,6 +4739,54 @@ type LiteLLMModel {
|
|
|
4340
4739
|
await config2.queue.remove(args.id);
|
|
4341
4740
|
return { success: true };
|
|
4342
4741
|
};
|
|
4742
|
+
const assertOwnsTranscriptionJob = async (id, context) => {
|
|
4743
|
+
const { db, user } = context;
|
|
4744
|
+
if (!user) throw new Error("Authentication required");
|
|
4745
|
+
if (user.super_admin === true) return;
|
|
4746
|
+
const row = await db.from("transcription_jobs").select(["created_by", "rights_mode"]).where({ id }).first();
|
|
4747
|
+
if (!row) throw new Error(`transcription_job ${id} not found`);
|
|
4748
|
+
if (row.rights_mode === "public") return;
|
|
4749
|
+
if (row.created_by === user.id) return;
|
|
4750
|
+
throw new Error("Not authorized to act on this transcription job");
|
|
4751
|
+
};
|
|
4752
|
+
resolvers.Mutation["transcriptionJobStart"] = async (_, args, context) => {
|
|
4753
|
+
const { user } = context;
|
|
4754
|
+
if (!user) throw new Error("Authentication required");
|
|
4755
|
+
if (!transcriptionClient.isConfigured()) {
|
|
4756
|
+
throw new Error(
|
|
4757
|
+
"TRANSCRIPTION_DISABLED: TRANSCRIPTION_SERVER not set on this server. Ask the operator to start a whisper server with `npx @exulu/backend exulu-start-whisper`."
|
|
4758
|
+
);
|
|
4759
|
+
}
|
|
4760
|
+
return transcriptionService.startJob({
|
|
4761
|
+
userId: user.id,
|
|
4762
|
+
s3Key: args.input.audio_s3key,
|
|
4763
|
+
filename: args.input.filename,
|
|
4764
|
+
title: args.input.title,
|
|
4765
|
+
language: args.input.language ?? void 0,
|
|
4766
|
+
num_speakers: args.input.num_speakers ?? void 0,
|
|
4767
|
+
hotwords: args.input.hotwords ?? void 0,
|
|
4768
|
+
project_id: args.input.project_id ?? null,
|
|
4769
|
+
target_rights_mode: args.input.target_rights_mode ?? null,
|
|
4770
|
+
target_rbac_users: args.input.target_rbac_users ?? void 0,
|
|
4771
|
+
target_rbac_roles: args.input.target_rbac_roles ?? void 0
|
|
4772
|
+
});
|
|
4773
|
+
};
|
|
4774
|
+
resolvers.Mutation["transcriptionJobFinalize"] = async (_, args, context) => {
|
|
4775
|
+
await assertOwnsTranscriptionJob(args.id, context);
|
|
4776
|
+
const { item, row } = await transcriptionService.finalize(args.id, {
|
|
4777
|
+
title: args.input.title,
|
|
4778
|
+
speakers: args.input.speakers,
|
|
4779
|
+
project_id: args.input.project_id ?? null,
|
|
4780
|
+
target_rights_mode: args.input.target_rights_mode ?? null,
|
|
4781
|
+
target_rbac_users: args.input.target_rbac_users ?? void 0,
|
|
4782
|
+
target_rbac_roles: args.input.target_rbac_roles ?? void 0
|
|
4783
|
+
});
|
|
4784
|
+
return { job: row, item_id: item.id };
|
|
4785
|
+
};
|
|
4786
|
+
resolvers.Mutation["transcriptionJobCancel"] = async (_, args, context) => {
|
|
4787
|
+
await assertOwnsTranscriptionJob(args.id, context);
|
|
4788
|
+
return transcriptionService.cancelJob(args.id);
|
|
4789
|
+
};
|
|
4343
4790
|
resolvers.Query["evals"] = async (_, args, context, info) => {
|
|
4344
4791
|
const requestedFields = getRequestedFields(info);
|
|
4345
4792
|
return {
|
|
@@ -5197,7 +5644,8 @@ import {
|
|
|
5197
5644
|
generateText as generateText2,
|
|
5198
5645
|
streamText,
|
|
5199
5646
|
validateUIMessages,
|
|
5200
|
-
stepCountIs
|
|
5647
|
+
stepCountIs,
|
|
5648
|
+
hasToolCall
|
|
5201
5649
|
} from "ai";
|
|
5202
5650
|
|
|
5203
5651
|
// src/utils/generate-slug.ts
|
|
@@ -5528,6 +5976,9 @@ var ExuluProvider = class {
|
|
|
5528
5976
|
If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume
|
|
5529
5977
|
they are talking with the current date in mind as a reference.`;
|
|
5530
5978
|
let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
|
|
5979
|
+
if (user?.personal_system_prompt?.trim()) {
|
|
5980
|
+
system += "\n\nUser preferences:\n" + user.personal_system_prompt.trim();
|
|
5981
|
+
}
|
|
5531
5982
|
system += "\n\n" + genericContext;
|
|
5532
5983
|
if (memoryContext) {
|
|
5533
5984
|
system += "\n\n" + memoryContext;
|
|
@@ -5627,7 +6078,10 @@ When a tool execution is not approved by the user, do not retry it unless explic
|
|
|
5627
6078
|
agent,
|
|
5628
6079
|
memoryItems
|
|
5629
6080
|
),
|
|
5630
|
-
|
|
6081
|
+
// Stop after the image_generation tool fires — the widget IS the
|
|
6082
|
+
// assistant's response, no follow-up text turn is wanted (same
|
|
6083
|
+
// reasoning as question_ask: the UI artifact is the message).
|
|
6084
|
+
stopWhen: [stepCountIs(maxStepCount || 5), hasToolCall("image_generation")]
|
|
5631
6085
|
// make configurable
|
|
5632
6086
|
});
|
|
5633
6087
|
console.log("[EXULU] Output: " + JSON.stringify(output, null, 2));
|
|
@@ -5708,7 +6162,7 @@ When a tool execution is not approved by the user, do not retry it unless explic
|
|
|
5708
6162
|
agent,
|
|
5709
6163
|
memoryItems
|
|
5710
6164
|
),
|
|
5711
|
-
stopWhen: [stepCountIs(maxStepCount || 5)]
|
|
6165
|
+
stopWhen: [stepCountIs(maxStepCount || 5), hasToolCall("image_generation")]
|
|
5712
6166
|
});
|
|
5713
6167
|
if (statistics) {
|
|
5714
6168
|
await Promise.all([
|
|
@@ -5933,6 +6387,9 @@ ${extractedText}
|
|
|
5933
6387
|
messages = await this.processFilePartsInMessages(messages);
|
|
5934
6388
|
const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
|
|
5935
6389
|
let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
|
|
6390
|
+
if (user?.personal_system_prompt?.trim()) {
|
|
6391
|
+
system += "\n\nUser preferences:\n" + user.personal_system_prompt.trim();
|
|
6392
|
+
}
|
|
5936
6393
|
system += "\n\n" + genericContext;
|
|
5937
6394
|
const includesContextSearchTool = currentTools?.some(
|
|
5938
6395
|
(tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search") || tool2.type === "context"
|
|
@@ -6089,7 +6546,7 @@ When a tool execution is not approved by the user, do not retry it unless explic
|
|
|
6089
6546
|
},
|
|
6090
6547
|
// provide more loops for skills because they are more complex to execute
|
|
6091
6548
|
// todo allow configuring this per skill
|
|
6092
|
-
stopWhen: [stepCountIs(maxStepCount || currentSkills?.length ? 10 : 5)]
|
|
6549
|
+
stopWhen: [stepCountIs(maxStepCount || currentSkills?.length ? 10 : 5), hasToolCall("image_generation")]
|
|
6093
6550
|
});
|
|
6094
6551
|
return {
|
|
6095
6552
|
stream: result,
|
|
@@ -6296,7 +6753,266 @@ async function synthesizeSpeech(args) {
|
|
|
6296
6753
|
return Buffer.from(arrayBuf);
|
|
6297
6754
|
}
|
|
6298
6755
|
|
|
6756
|
+
// src/exulu/image-generation.ts
|
|
6757
|
+
var ImageGenerationError = class extends Error {
|
|
6758
|
+
constructor(upstreamStatus, message) {
|
|
6759
|
+
super(message);
|
|
6760
|
+
this.upstreamStatus = upstreamStatus;
|
|
6761
|
+
this.name = "ImageGenerationError";
|
|
6762
|
+
}
|
|
6763
|
+
};
|
|
6764
|
+
var resolveProxyConfig = () => {
|
|
6765
|
+
const host = process.env.LITELLM_HOST ?? "127.0.0.1";
|
|
6766
|
+
const port = process.env.LITELLM_PORT ?? "4000";
|
|
6767
|
+
const masterKey = process.env.LITELLM_MASTER_KEY;
|
|
6768
|
+
if (!masterKey) throw new Error("LITELLM_MASTER_KEY is not set");
|
|
6769
|
+
return { host, port, masterKey };
|
|
6770
|
+
};
|
|
6771
|
+
var normalizeDataEntries = async (data) => {
|
|
6772
|
+
const out = [];
|
|
6773
|
+
for (const entry of data) {
|
|
6774
|
+
let buffer;
|
|
6775
|
+
let contentType = "image/png";
|
|
6776
|
+
let extension = "png";
|
|
6777
|
+
if (entry.b64_json) {
|
|
6778
|
+
buffer = Buffer.from(entry.b64_json, "base64");
|
|
6779
|
+
} else if (entry.url) {
|
|
6780
|
+
const upstream = await fetch(entry.url);
|
|
6781
|
+
if (!upstream.ok) {
|
|
6782
|
+
throw new ImageGenerationError(
|
|
6783
|
+
upstream.status,
|
|
6784
|
+
`Failed to download generated image from ${entry.url}: ${upstream.status} ${upstream.statusText}`
|
|
6785
|
+
);
|
|
6786
|
+
}
|
|
6787
|
+
const ct = upstream.headers.get("content-type");
|
|
6788
|
+
if (ct && ct.startsWith("image/")) {
|
|
6789
|
+
contentType = ct;
|
|
6790
|
+
const inferred = ct.split("/")[1]?.split(";")[0]?.trim();
|
|
6791
|
+
if (inferred) extension = inferred === "jpeg" ? "jpg" : inferred;
|
|
6792
|
+
}
|
|
6793
|
+
buffer = Buffer.from(await upstream.arrayBuffer());
|
|
6794
|
+
} else {
|
|
6795
|
+
throw new ImageGenerationError(
|
|
6796
|
+
0,
|
|
6797
|
+
"LiteLLM image response entry contained neither b64_json nor url."
|
|
6798
|
+
);
|
|
6799
|
+
}
|
|
6800
|
+
out.push({ buffer, contentType, extension, revisedPrompt: entry.revised_prompt });
|
|
6801
|
+
}
|
|
6802
|
+
return out;
|
|
6803
|
+
};
|
|
6804
|
+
async function generateImage(args) {
|
|
6805
|
+
if (!args.model) throw new Error("model is required");
|
|
6806
|
+
if (!args.prompt) throw new Error("prompt is required");
|
|
6807
|
+
const cfg = resolveProxyConfig();
|
|
6808
|
+
const body = {
|
|
6809
|
+
model: args.model,
|
|
6810
|
+
prompt: args.prompt
|
|
6811
|
+
};
|
|
6812
|
+
if (args.size) body.size = args.size;
|
|
6813
|
+
if (args.quality) body.quality = args.quality;
|
|
6814
|
+
if (args.n) body.n = args.n;
|
|
6815
|
+
const res = await fetch(`http://${cfg.host}:${cfg.port}/v1/images/generations`, {
|
|
6816
|
+
method: "POST",
|
|
6817
|
+
headers: {
|
|
6818
|
+
Authorization: `Bearer ${cfg.masterKey}`,
|
|
6819
|
+
"Content-Type": "application/json"
|
|
6820
|
+
},
|
|
6821
|
+
body: JSON.stringify(body),
|
|
6822
|
+
signal: args.signal
|
|
6823
|
+
});
|
|
6824
|
+
if (!res.ok) {
|
|
6825
|
+
const text = await res.text().catch(() => "");
|
|
6826
|
+
throw new ImageGenerationError(
|
|
6827
|
+
res.status,
|
|
6828
|
+
`LiteLLM image generation failed (status ${res.status}): ${text}`.trim()
|
|
6829
|
+
);
|
|
6830
|
+
}
|
|
6831
|
+
const json = await res.json();
|
|
6832
|
+
if (!json?.data || json.data.length === 0) {
|
|
6833
|
+
throw new ImageGenerationError(
|
|
6834
|
+
res.status,
|
|
6835
|
+
"LiteLLM returned no image data in the response."
|
|
6836
|
+
);
|
|
6837
|
+
}
|
|
6838
|
+
return normalizeDataEntries(json.data);
|
|
6839
|
+
}
|
|
6840
|
+
async function editImage(args) {
|
|
6841
|
+
if (!args.model) throw new Error("model is required");
|
|
6842
|
+
if (!args.prompt) throw new Error("prompt is required");
|
|
6843
|
+
if (!args.references || args.references.length === 0) {
|
|
6844
|
+
throw new Error("at least one reference image is required");
|
|
6845
|
+
}
|
|
6846
|
+
const cfg = resolveProxyConfig();
|
|
6847
|
+
const form = new FormData();
|
|
6848
|
+
form.append("model", args.model);
|
|
6849
|
+
form.append("prompt", args.prompt);
|
|
6850
|
+
if (args.n != null) form.append("n", String(args.n));
|
|
6851
|
+
if (args.size) form.append("size", args.size);
|
|
6852
|
+
if (args.quality) form.append("quality", args.quality);
|
|
6853
|
+
form.append("response_format", "b64_json");
|
|
6854
|
+
for (const ref of args.references) {
|
|
6855
|
+
form.append(
|
|
6856
|
+
"image",
|
|
6857
|
+
new Blob([ref.buffer], { type: ref.mimetype ?? "image/png" }),
|
|
6858
|
+
ref.filename
|
|
6859
|
+
);
|
|
6860
|
+
}
|
|
6861
|
+
if (args.mask) {
|
|
6862
|
+
form.append(
|
|
6863
|
+
"mask",
|
|
6864
|
+
new Blob([args.mask.buffer], { type: args.mask.mimetype ?? "image/png" }),
|
|
6865
|
+
args.mask.filename
|
|
6866
|
+
);
|
|
6867
|
+
}
|
|
6868
|
+
const res = await fetch(`http://${cfg.host}:${cfg.port}/v1/images/edits`, {
|
|
6869
|
+
method: "POST",
|
|
6870
|
+
headers: { Authorization: `Bearer ${cfg.masterKey}` },
|
|
6871
|
+
body: form,
|
|
6872
|
+
signal: args.signal
|
|
6873
|
+
});
|
|
6874
|
+
if (!res.ok) {
|
|
6875
|
+
const text = await res.text().catch(() => "");
|
|
6876
|
+
throw new ImageGenerationError(
|
|
6877
|
+
res.status,
|
|
6878
|
+
`LiteLLM image edit failed (status ${res.status}): ${text}`.trim()
|
|
6879
|
+
);
|
|
6880
|
+
}
|
|
6881
|
+
const json = await res.json();
|
|
6882
|
+
if (!json?.data || json.data.length === 0) {
|
|
6883
|
+
throw new ImageGenerationError(
|
|
6884
|
+
res.status,
|
|
6885
|
+
"LiteLLM returned no image data in the edit response."
|
|
6886
|
+
);
|
|
6887
|
+
}
|
|
6888
|
+
return normalizeDataEntries(json.data);
|
|
6889
|
+
}
|
|
6890
|
+
|
|
6891
|
+
// src/exulu/litellm/parse-image-models.ts
|
|
6892
|
+
import { existsSync as existsSync2, readFileSync } from "fs";
|
|
6893
|
+
var stripComment = (line) => {
|
|
6894
|
+
const idx = line.indexOf("#");
|
|
6895
|
+
return idx >= 0 ? line.slice(0, idx) : line;
|
|
6896
|
+
};
|
|
6897
|
+
var parseInlineArray = (raw) => {
|
|
6898
|
+
const m = raw.trim().match(/^\[(.*)\]$/);
|
|
6899
|
+
if (!m) return void 0;
|
|
6900
|
+
const inner = m[1] ?? "";
|
|
6901
|
+
if (!inner.trim()) return [];
|
|
6902
|
+
return inner.split(",").map((s) => s.trim().replace(/^["']|["']$/g, "")).filter((s) => s.length > 0);
|
|
6903
|
+
};
|
|
6904
|
+
var parseBool = (raw) => {
|
|
6905
|
+
const v = raw.trim().toLowerCase();
|
|
6906
|
+
if (v === "true" || v === "yes") return true;
|
|
6907
|
+
if (v === "false" || v === "no") return false;
|
|
6908
|
+
return void 0;
|
|
6909
|
+
};
|
|
6910
|
+
var parseInt10 = (raw) => {
|
|
6911
|
+
const n = Number(raw.trim());
|
|
6912
|
+
return Number.isInteger(n) ? n : void 0;
|
|
6913
|
+
};
|
|
6914
|
+
var parseImageGenerationModels = (configPath) => {
|
|
6915
|
+
if (!existsSync2(configPath)) return [];
|
|
6916
|
+
const text = readFileSync(configPath, "utf8");
|
|
6917
|
+
const lines = text.split("\n");
|
|
6918
|
+
const entries = [];
|
|
6919
|
+
let current;
|
|
6920
|
+
for (const rawLine of lines) {
|
|
6921
|
+
const noComment = stripComment(rawLine);
|
|
6922
|
+
if (!noComment.trim()) continue;
|
|
6923
|
+
const indent = (rawLine.match(/^\s*/)?.[0] ?? "").length;
|
|
6924
|
+
const modelNameMatch = noComment.match(
|
|
6925
|
+
/^\s*-\s*model_name\s*:\s*["']?([^"'\s#]+)["']?\s*$/
|
|
6926
|
+
);
|
|
6927
|
+
if (modelNameMatch) {
|
|
6928
|
+
if (current) entries.push(current);
|
|
6929
|
+
current = { model_name: modelNameMatch[1], indent };
|
|
6930
|
+
continue;
|
|
6931
|
+
}
|
|
6932
|
+
if (!current) continue;
|
|
6933
|
+
if (indent <= current.indent && !/^\s*-\s/.test(rawLine)) {
|
|
6934
|
+
entries.push(current);
|
|
6935
|
+
current = void 0;
|
|
6936
|
+
continue;
|
|
6937
|
+
}
|
|
6938
|
+
const kvMatch = noComment.match(/^\s*(\w+)\s*:\s*(.+?)\s*$/);
|
|
6939
|
+
if (!kvMatch) continue;
|
|
6940
|
+
const key2 = kvMatch[1] ?? "";
|
|
6941
|
+
const rawValue = kvMatch[2] ?? "";
|
|
6942
|
+
switch (key2) {
|
|
6943
|
+
case "type": {
|
|
6944
|
+
current.type = rawValue.replace(/^["']|["']$/g, "").trim();
|
|
6945
|
+
break;
|
|
6946
|
+
}
|
|
6947
|
+
case "sizes": {
|
|
6948
|
+
current.sizes = parseInlineArray(rawValue);
|
|
6949
|
+
break;
|
|
6950
|
+
}
|
|
6951
|
+
case "qualities": {
|
|
6952
|
+
current.qualities = parseInlineArray(rawValue);
|
|
6953
|
+
break;
|
|
6954
|
+
}
|
|
6955
|
+
case "supports_edit": {
|
|
6956
|
+
current.supports_edit = parseBool(rawValue);
|
|
6957
|
+
break;
|
|
6958
|
+
}
|
|
6959
|
+
case "max_n": {
|
|
6960
|
+
current.max_n = parseInt10(rawValue);
|
|
6961
|
+
break;
|
|
6962
|
+
}
|
|
6963
|
+
}
|
|
6964
|
+
}
|
|
6965
|
+
if (current) entries.push(current);
|
|
6966
|
+
const imageEntries = entries.filter((e) => e.type === "image_generation");
|
|
6967
|
+
const errors = [];
|
|
6968
|
+
const validated = [];
|
|
6969
|
+
for (const e of imageEntries) {
|
|
6970
|
+
const modelErrs = [];
|
|
6971
|
+
if (!Array.isArray(e.sizes) || e.sizes.length === 0) {
|
|
6972
|
+
modelErrs.push(
|
|
6973
|
+
'model_info.sizes must be a non-empty inline YAML array of strings, e.g. `sizes: ["1024x1024", "1024x1536"]`'
|
|
6974
|
+
);
|
|
6975
|
+
}
|
|
6976
|
+
if (!Array.isArray(e.qualities) || e.qualities.length === 0) {
|
|
6977
|
+
modelErrs.push(
|
|
6978
|
+
'model_info.qualities must be a non-empty inline YAML array of strings, e.g. `qualities: ["auto", "high"]`'
|
|
6979
|
+
);
|
|
6980
|
+
}
|
|
6981
|
+
if (typeof e.supports_edit !== "boolean") {
|
|
6982
|
+
modelErrs.push(
|
|
6983
|
+
"model_info.supports_edit must be a boolean (true/false)"
|
|
6984
|
+
);
|
|
6985
|
+
}
|
|
6986
|
+
if (typeof e.max_n !== "number" || !Number.isInteger(e.max_n) || e.max_n < 1) {
|
|
6987
|
+
modelErrs.push("model_info.max_n must be an integer \u2265 1");
|
|
6988
|
+
}
|
|
6989
|
+
if (modelErrs.length > 0) {
|
|
6990
|
+
errors.push(
|
|
6991
|
+
` - "${e.model_name}":
|
|
6992
|
+
- ${modelErrs.join("\n - ")}`
|
|
6993
|
+
);
|
|
6994
|
+
continue;
|
|
6995
|
+
}
|
|
6996
|
+
validated.push({
|
|
6997
|
+
model_name: e.model_name,
|
|
6998
|
+
sizes: e.sizes,
|
|
6999
|
+
qualities: e.qualities,
|
|
7000
|
+
supports_edit: e.supports_edit,
|
|
7001
|
+
max_n: e.max_n
|
|
7002
|
+
});
|
|
7003
|
+
}
|
|
7004
|
+
if (errors.length > 0) {
|
|
7005
|
+
throw new Error(
|
|
7006
|
+
`[EXULU] config.litellm.yaml has image-generation models with missing or invalid model_info keys. Fix and restart Exulu:
|
|
7007
|
+
${errors.join("\n")}
|
|
7008
|
+
See docs/superpowers/specs/2026-05-31-in-chat-image-generation-design.md for the required schema.`
|
|
7009
|
+
);
|
|
7010
|
+
}
|
|
7011
|
+
return validated;
|
|
7012
|
+
};
|
|
7013
|
+
|
|
6299
7014
|
// src/exulu/routes.ts
|
|
7015
|
+
import { resolve as resolvePath } from "path";
|
|
6300
7016
|
import multer from "multer";
|
|
6301
7017
|
|
|
6302
7018
|
// src/utils/check-provider-rate-limit.ts
|
|
@@ -6993,7 +7709,8 @@ var {
|
|
|
6993
7709
|
contextPresetsSchema,
|
|
6994
7710
|
embedderSettingsSchema,
|
|
6995
7711
|
promptFavoritesSchema,
|
|
6996
|
-
statisticsSchema
|
|
7712
|
+
statisticsSchema,
|
|
7713
|
+
transcriptionJobsSchema
|
|
6997
7714
|
} = coreSchemas.get();
|
|
6998
7715
|
var createExpressRoutes = async (app, providers, tools, contexts, config, evals, tracer, queues2, rerankers) => {
|
|
6999
7716
|
let corsOptions = {
|
|
@@ -7051,7 +7768,8 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
|
|
|
7051
7768
|
variablesSchema(),
|
|
7052
7769
|
workflowTemplatesSchema(),
|
|
7053
7770
|
statisticsSchema(),
|
|
7054
|
-
rbacSchema()
|
|
7771
|
+
rbacSchema(),
|
|
7772
|
+
transcriptionJobsSchema()
|
|
7055
7773
|
],
|
|
7056
7774
|
contexts ?? [],
|
|
7057
7775
|
providers,
|
|
@@ -7874,20 +8592,501 @@ ${customInstructions}` : agent.instructions;
|
|
|
7874
8592
|
}
|
|
7875
8593
|
}
|
|
7876
8594
|
);
|
|
7877
|
-
|
|
7878
|
-
if (!isLiteLLMEnabled())
|
|
7879
|
-
|
|
7880
|
-
|
|
7881
|
-
|
|
7882
|
-
return;
|
|
7883
|
-
}
|
|
7884
|
-
|
|
7885
|
-
|
|
7886
|
-
|
|
7887
|
-
|
|
8595
|
+
const imageModelsByName = (() => {
|
|
8596
|
+
if (!isLiteLLMEnabled() || !config?.fileUploads) return /* @__PURE__ */ new Map();
|
|
8597
|
+
try {
|
|
8598
|
+
const configPath = process.env.LITELLM_CONFIG_PATH ?? resolvePath(getPackageRoot(), "./config.litellm.yaml");
|
|
8599
|
+
const models2 = parseImageGenerationModels(configPath);
|
|
8600
|
+
return new Map(models2.map((m) => [m.model_name, m]));
|
|
8601
|
+
} catch (err) {
|
|
8602
|
+
console.error(
|
|
8603
|
+
"[EXULU] Skipping /images/* routes due to config.litellm.yaml error:",
|
|
8604
|
+
err.message
|
|
8605
|
+
);
|
|
8606
|
+
return /* @__PURE__ */ new Map();
|
|
7888
8607
|
}
|
|
7889
|
-
|
|
7890
|
-
|
|
8608
|
+
})();
|
|
8609
|
+
const imageRoutesEnabled = isLiteLLMEnabled() && !!config?.fileUploads?.s3region && !!config?.fileUploads?.s3key && !!config?.fileUploads?.s3secret && !!config?.fileUploads?.s3Bucket && imageModelsByName.size > 0;
|
|
8610
|
+
const respond503ImagesNotEnabled = (res) => {
|
|
8611
|
+
res.status(503).json({
|
|
8612
|
+
detail: "Image generation is not enabled on this deployment. Requires EXULU_USE_LITELLM=true, S3 fileUploads configuration, and at least one model in config.litellm.yaml with model_info.type=image_generation."
|
|
8613
|
+
});
|
|
8614
|
+
};
|
|
8615
|
+
const loadAuthedSession = async (req, res, sessionId, rights) => {
|
|
8616
|
+
const authResult = await requestValidators.authenticate(req);
|
|
8617
|
+
if (!authResult.user?.id) {
|
|
8618
|
+
res.status(authResult.code || 401).json({ detail: authResult.message });
|
|
8619
|
+
return null;
|
|
8620
|
+
}
|
|
8621
|
+
const { db } = await postgresClient();
|
|
8622
|
+
const session = await db.from("agent_sessions").where({ id: sessionId }).first();
|
|
8623
|
+
if (!session) {
|
|
8624
|
+
res.status(404).json({ detail: `Session ${sessionId} not found.` });
|
|
8625
|
+
return null;
|
|
8626
|
+
}
|
|
8627
|
+
const sessionRbac = await RBACResolver(
|
|
8628
|
+
db,
|
|
8629
|
+
"agent_sessions",
|
|
8630
|
+
session.id,
|
|
8631
|
+
session.rights_mode || "private"
|
|
8632
|
+
);
|
|
8633
|
+
const allowed = await checkRecordAccess(
|
|
8634
|
+
{ ...session, RBAC: sessionRbac },
|
|
8635
|
+
rights,
|
|
8636
|
+
authResult.user
|
|
8637
|
+
);
|
|
8638
|
+
if (!allowed) {
|
|
8639
|
+
res.status(403).json({ detail: `You don't have ${rights} access to this session.` });
|
|
8640
|
+
return null;
|
|
8641
|
+
}
|
|
8642
|
+
return { user: authResult.user, session, db };
|
|
8643
|
+
};
|
|
8644
|
+
const loadStyle = async (db, styleId, user, res) => {
|
|
8645
|
+
if (!styleId) return { markdown: null, id: null };
|
|
8646
|
+
const row = await db.from("platform_configurations").where({ id: styleId }).first();
|
|
8647
|
+
if (!row) {
|
|
8648
|
+
res.status(404).json({ detail: `Style ${styleId} not found.` });
|
|
8649
|
+
return "error";
|
|
8650
|
+
}
|
|
8651
|
+
const rbac = await RBACResolver(
|
|
8652
|
+
db,
|
|
8653
|
+
"platform_configurations",
|
|
8654
|
+
row.id,
|
|
8655
|
+
row.rights_mode || "private"
|
|
8656
|
+
);
|
|
8657
|
+
const allowed = await checkRecordAccess(
|
|
8658
|
+
{ ...row, RBAC: rbac },
|
|
8659
|
+
"read",
|
|
8660
|
+
user
|
|
8661
|
+
);
|
|
8662
|
+
if (!allowed) {
|
|
8663
|
+
res.status(403).json({ detail: "You don't have access to that style." });
|
|
8664
|
+
return "error";
|
|
8665
|
+
}
|
|
8666
|
+
const value = typeof row.config_value === "string" ? (() => {
|
|
8667
|
+
try {
|
|
8668
|
+
return JSON.parse(row.config_value);
|
|
8669
|
+
} catch {
|
|
8670
|
+
return null;
|
|
8671
|
+
}
|
|
8672
|
+
})() : row.config_value;
|
|
8673
|
+
return { markdown: value?.markdown ?? null, id: row.id };
|
|
8674
|
+
};
|
|
8675
|
+
const validateGenerationParams = (body, res) => {
|
|
8676
|
+
const { model: modelName, prompt, n, size, quality } = body || {};
|
|
8677
|
+
const model = typeof modelName === "string" ? imageModelsByName.get(modelName) : void 0;
|
|
8678
|
+
if (!model) {
|
|
8679
|
+
res.status(400).json({
|
|
8680
|
+
detail: `Unknown image-generation model "${modelName}". Available: ${[...imageModelsByName.keys()].join(", ")}.`
|
|
8681
|
+
});
|
|
8682
|
+
return null;
|
|
8683
|
+
}
|
|
8684
|
+
if (typeof prompt !== "string" || prompt.trim().length === 0) {
|
|
8685
|
+
res.status(400).json({ detail: "prompt must be a non-empty string." });
|
|
8686
|
+
return null;
|
|
8687
|
+
}
|
|
8688
|
+
const requestedN = typeof n === "number" ? n : 1;
|
|
8689
|
+
if (!Number.isInteger(requestedN) || requestedN < 1 || requestedN > model.max_n) {
|
|
8690
|
+
res.status(400).json({
|
|
8691
|
+
detail: `n must be an integer between 1 and ${model.max_n} for model ${model.model_name}.`
|
|
8692
|
+
});
|
|
8693
|
+
return null;
|
|
8694
|
+
}
|
|
8695
|
+
if (size && !model.sizes.includes(size)) {
|
|
8696
|
+
res.status(400).json({
|
|
8697
|
+
detail: `size "${size}" is not supported by ${model.model_name}. Allowed: ${model.sizes.join(", ")}.`
|
|
8698
|
+
});
|
|
8699
|
+
return null;
|
|
8700
|
+
}
|
|
8701
|
+
if (quality && !model.qualities.includes(quality)) {
|
|
8702
|
+
res.status(400).json({
|
|
8703
|
+
detail: `quality "${quality}" is not supported by ${model.model_name}. Allowed: ${model.qualities.join(", ")}.`
|
|
8704
|
+
});
|
|
8705
|
+
return null;
|
|
8706
|
+
}
|
|
8707
|
+
return { model, prompt, n: requestedN, size, quality };
|
|
8708
|
+
};
|
|
8709
|
+
const uploadGeneratedImages = async (images, sessionId, toolCallId, userId) => {
|
|
8710
|
+
if (!config?.fileUploads) {
|
|
8711
|
+
throw new Error("File uploads not configured.");
|
|
8712
|
+
}
|
|
8713
|
+
const keys = [];
|
|
8714
|
+
const revisedPrompts = [];
|
|
8715
|
+
for (const img of images) {
|
|
8716
|
+
const filename = `${randomUUID2()}.${img.extension}`;
|
|
8717
|
+
const key2 = `sessions/${sessionId}/images/${toolCallId}/${filename}`;
|
|
8718
|
+
const fullKey = await uploadFile(
|
|
8719
|
+
img.buffer,
|
|
8720
|
+
key2,
|
|
8721
|
+
config,
|
|
8722
|
+
{ contentType: img.contentType },
|
|
8723
|
+
userId
|
|
8724
|
+
);
|
|
8725
|
+
keys.push(fullKey);
|
|
8726
|
+
revisedPrompts.push(img.revisedPrompt ?? null);
|
|
8727
|
+
}
|
|
8728
|
+
const presignedUrls = await Promise.all(
|
|
8729
|
+
keys.map((fullKey) => {
|
|
8730
|
+
const slash = fullKey.indexOf("/");
|
|
8731
|
+
const bucket = slash > 0 ? fullKey.slice(0, slash) : config.fileUploads.s3Bucket;
|
|
8732
|
+
const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
|
|
8733
|
+
return getPresignedUrl(bucket, objectKey, config);
|
|
8734
|
+
})
|
|
8735
|
+
);
|
|
8736
|
+
return { keys, revisedPrompts, presignedUrls };
|
|
8737
|
+
};
|
|
8738
|
+
app.post("/images/generate", async (req, res) => {
|
|
8739
|
+
if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
|
|
8740
|
+
const { sessionId, toolCallId, styleId } = req.body || {};
|
|
8741
|
+
if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
|
|
8742
|
+
res.status(400).json({ detail: "sessionId and toolCallId are required." });
|
|
8743
|
+
return;
|
|
8744
|
+
}
|
|
8745
|
+
const authed = await loadAuthedSession(req, res, sessionId, "write");
|
|
8746
|
+
if (!authed) return;
|
|
8747
|
+
const params = validateGenerationParams(req.body, res);
|
|
8748
|
+
if (!params) return;
|
|
8749
|
+
const style = await loadStyle(authed.db, styleId, authed.user, res);
|
|
8750
|
+
if (style === "error") return;
|
|
8751
|
+
const finalPrompt = style.markdown ? `${params.prompt}
|
|
8752
|
+
|
|
8753
|
+
${style.markdown}` : params.prompt;
|
|
8754
|
+
try {
|
|
8755
|
+
await Promise.race([
|
|
8756
|
+
waitForLiteLLMReady(),
|
|
8757
|
+
new Promise(
|
|
8758
|
+
(_, reject) => setTimeout(() => reject(new Error("LiteLLM not ready")), 5e3)
|
|
8759
|
+
)
|
|
8760
|
+
]);
|
|
8761
|
+
} catch {
|
|
8762
|
+
res.status(503).json({ detail: "Image service is not ready. Try again shortly." });
|
|
8763
|
+
return;
|
|
8764
|
+
}
|
|
8765
|
+
const abortController = new AbortController();
|
|
8766
|
+
req.on("close", () => abortController.abort());
|
|
8767
|
+
try {
|
|
8768
|
+
const images = await generateImage({
|
|
8769
|
+
model: params.model.model_name,
|
|
8770
|
+
prompt: finalPrompt,
|
|
8771
|
+
n: params.n,
|
|
8772
|
+
size: params.size,
|
|
8773
|
+
quality: params.quality,
|
|
8774
|
+
signal: abortController.signal
|
|
8775
|
+
});
|
|
8776
|
+
const { keys, revisedPrompts, presignedUrls } = await uploadGeneratedImages(
|
|
8777
|
+
images,
|
|
8778
|
+
sessionId,
|
|
8779
|
+
toolCallId,
|
|
8780
|
+
authed.user.id
|
|
8781
|
+
);
|
|
8782
|
+
const [row] = await authed.db("image_generations").insert({
|
|
8783
|
+
session_id: sessionId,
|
|
8784
|
+
tool_call_id: toolCallId,
|
|
8785
|
+
user_id: authed.user.id,
|
|
8786
|
+
operation: "generate",
|
|
8787
|
+
model: params.model.model_name,
|
|
8788
|
+
prompt: params.prompt,
|
|
8789
|
+
applied_style_id: style.id,
|
|
8790
|
+
applied_style_markdown: style.markdown,
|
|
8791
|
+
size: params.size,
|
|
8792
|
+
quality: params.quality,
|
|
8793
|
+
n: params.n,
|
|
8794
|
+
image_keys: JSON.stringify(keys),
|
|
8795
|
+
revised_prompts: JSON.stringify(revisedPrompts),
|
|
8796
|
+
selected: false
|
|
8797
|
+
}).returning("*");
|
|
8798
|
+
res.status(200).json({
|
|
8799
|
+
generationId: row.id,
|
|
8800
|
+
images: keys.map((key2, i) => ({
|
|
8801
|
+
key: key2,
|
|
8802
|
+
presignedUrl: presignedUrls[i],
|
|
8803
|
+
revisedPrompt: revisedPrompts[i]
|
|
8804
|
+
}))
|
|
8805
|
+
});
|
|
8806
|
+
} catch (err) {
|
|
8807
|
+
if (abortController.signal.aborted) return;
|
|
8808
|
+
if (err instanceof ImageGenerationError) {
|
|
8809
|
+
const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
|
|
8810
|
+
res.status(code).json({ detail: err.message });
|
|
8811
|
+
return;
|
|
8812
|
+
}
|
|
8813
|
+
console.error("[EXULU] /images/generate failed", err);
|
|
8814
|
+
res.status(500).json({
|
|
8815
|
+
detail: err instanceof Error ? err.message : "Image generation failed."
|
|
8816
|
+
});
|
|
8817
|
+
}
|
|
8818
|
+
});
|
|
8819
|
+
app.post("/images/edit", async (req, res) => {
|
|
8820
|
+
if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
|
|
8821
|
+
const { sessionId, toolCallId, styleId, referenceImageKeys, maskKey } = req.body || {};
|
|
8822
|
+
if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
|
|
8823
|
+
res.status(400).json({ detail: "sessionId and toolCallId are required." });
|
|
8824
|
+
return;
|
|
8825
|
+
}
|
|
8826
|
+
if (!Array.isArray(referenceImageKeys) || referenceImageKeys.length === 0) {
|
|
8827
|
+
res.status(400).json({ detail: "referenceImageKeys must be a non-empty array." });
|
|
8828
|
+
return;
|
|
8829
|
+
}
|
|
8830
|
+
const authed = await loadAuthedSession(req, res, sessionId, "write");
|
|
8831
|
+
if (!authed) return;
|
|
8832
|
+
const params = validateGenerationParams(req.body, res);
|
|
8833
|
+
if (!params) return;
|
|
8834
|
+
if (!params.model.supports_edit) {
|
|
8835
|
+
res.status(400).json({
|
|
8836
|
+
detail: `Model ${params.model.model_name} does not support image editing.`
|
|
8837
|
+
});
|
|
8838
|
+
return;
|
|
8839
|
+
}
|
|
8840
|
+
const userPrefix = `user_${authed.user.id}/`;
|
|
8841
|
+
const sessionPrefix = `sessions/${sessionId}/`;
|
|
8842
|
+
const ownsKey = (k) => k.includes(userPrefix) || k.includes(sessionPrefix);
|
|
8843
|
+
if (!referenceImageKeys.every((k) => typeof k === "string" && ownsKey(k))) {
|
|
8844
|
+
res.status(403).json({ detail: "One or more reference image keys are not accessible." });
|
|
8845
|
+
return;
|
|
8846
|
+
}
|
|
8847
|
+
if (maskKey && (typeof maskKey !== "string" || !ownsKey(maskKey))) {
|
|
8848
|
+
res.status(403).json({ detail: "Mask image is not accessible." });
|
|
8849
|
+
return;
|
|
8850
|
+
}
|
|
8851
|
+
const style = await loadStyle(authed.db, styleId, authed.user, res);
|
|
8852
|
+
if (style === "error") return;
|
|
8853
|
+
const finalPrompt = style.markdown ? `${params.prompt}
|
|
8854
|
+
|
|
8855
|
+
${style.markdown}` : params.prompt;
|
|
8856
|
+
try {
|
|
8857
|
+
await Promise.race([
|
|
8858
|
+
waitForLiteLLMReady(),
|
|
8859
|
+
new Promise(
|
|
8860
|
+
(_, reject) => setTimeout(() => reject(new Error("LiteLLM not ready")), 5e3)
|
|
8861
|
+
)
|
|
8862
|
+
]);
|
|
8863
|
+
} catch {
|
|
8864
|
+
res.status(503).json({ detail: "Image service is not ready. Try again shortly." });
|
|
8865
|
+
return;
|
|
8866
|
+
}
|
|
8867
|
+
const fetchRef = async (fullKey) => {
|
|
8868
|
+
const slash = fullKey.indexOf("/");
|
|
8869
|
+
const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
|
|
8870
|
+
const buf = await getS3ObjectBytes(objectKey, config);
|
|
8871
|
+
const filename = fullKey.split("/").pop() ?? "image.png";
|
|
8872
|
+
const ext = filename.split(".").pop()?.toLowerCase() ?? "png";
|
|
8873
|
+
const mimetype = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : `image/${ext}`;
|
|
8874
|
+
return { buffer: buf, filename, mimetype };
|
|
8875
|
+
};
|
|
8876
|
+
const abortController = new AbortController();
|
|
8877
|
+
req.on("close", () => abortController.abort());
|
|
8878
|
+
try {
|
|
8879
|
+
const references = await Promise.all(referenceImageKeys.map(fetchRef));
|
|
8880
|
+
const mask = maskKey ? await fetchRef(maskKey) : void 0;
|
|
8881
|
+
const images = await editImage({
|
|
8882
|
+
model: params.model.model_name,
|
|
8883
|
+
prompt: finalPrompt,
|
|
8884
|
+
references,
|
|
8885
|
+
mask,
|
|
8886
|
+
n: params.n,
|
|
8887
|
+
size: params.size,
|
|
8888
|
+
quality: params.quality,
|
|
8889
|
+
signal: abortController.signal
|
|
8890
|
+
});
|
|
8891
|
+
const { keys, revisedPrompts, presignedUrls } = await uploadGeneratedImages(
|
|
8892
|
+
images,
|
|
8893
|
+
sessionId,
|
|
8894
|
+
toolCallId,
|
|
8895
|
+
authed.user.id
|
|
8896
|
+
);
|
|
8897
|
+
const [row] = await authed.db("image_generations").insert({
|
|
8898
|
+
session_id: sessionId,
|
|
8899
|
+
tool_call_id: toolCallId,
|
|
8900
|
+
user_id: authed.user.id,
|
|
8901
|
+
operation: "edit",
|
|
8902
|
+
model: params.model.model_name,
|
|
8903
|
+
prompt: params.prompt,
|
|
8904
|
+
applied_style_id: style.id,
|
|
8905
|
+
applied_style_markdown: style.markdown,
|
|
8906
|
+
size: params.size,
|
|
8907
|
+
quality: params.quality,
|
|
8908
|
+
n: params.n,
|
|
8909
|
+
reference_image_keys: JSON.stringify(referenceImageKeys),
|
|
8910
|
+
mask_image_key: maskKey,
|
|
8911
|
+
image_keys: JSON.stringify(keys),
|
|
8912
|
+
revised_prompts: JSON.stringify(revisedPrompts),
|
|
8913
|
+
selected: false
|
|
8914
|
+
}).returning("*");
|
|
8915
|
+
res.status(200).json({
|
|
8916
|
+
generationId: row.id,
|
|
8917
|
+
images: keys.map((key2, i) => ({
|
|
8918
|
+
key: key2,
|
|
8919
|
+
presignedUrl: presignedUrls[i],
|
|
8920
|
+
revisedPrompt: revisedPrompts[i]
|
|
8921
|
+
}))
|
|
8922
|
+
});
|
|
8923
|
+
} catch (err) {
|
|
8924
|
+
if (abortController.signal.aborted) return;
|
|
8925
|
+
if (err instanceof ImageGenerationError) {
|
|
8926
|
+
const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
|
|
8927
|
+
res.status(code).json({ detail: err.message });
|
|
8928
|
+
return;
|
|
8929
|
+
}
|
|
8930
|
+
console.error("[EXULU] /images/edit failed", err);
|
|
8931
|
+
res.status(500).json({
|
|
8932
|
+
detail: err instanceof Error ? err.message : "Image edit failed."
|
|
8933
|
+
});
|
|
8934
|
+
}
|
|
8935
|
+
});
|
|
8936
|
+
app.post("/images/select", async (req, res) => {
|
|
8937
|
+
if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
|
|
8938
|
+
const { sessionId, toolCallId, selections } = req.body || {};
|
|
8939
|
+
if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
|
|
8940
|
+
res.status(400).json({ detail: "sessionId and toolCallId are required." });
|
|
8941
|
+
return;
|
|
8942
|
+
}
|
|
8943
|
+
if (!Array.isArray(selections) || selections.length === 0) {
|
|
8944
|
+
res.status(400).json({ detail: "selections must be a non-empty array." });
|
|
8945
|
+
return;
|
|
8946
|
+
}
|
|
8947
|
+
const authed = await loadAuthedSession(req, res, sessionId, "write");
|
|
8948
|
+
if (!authed) return;
|
|
8949
|
+
const rows = await authed.db("image_generations").where({ session_id: sessionId, tool_call_id: toolCallId }).select("*");
|
|
8950
|
+
const rowsById = new Map(rows.map((r) => [r.id, r]));
|
|
8951
|
+
const selectedDetails = [];
|
|
8952
|
+
const rowsToMarkSelected = /* @__PURE__ */ new Set();
|
|
8953
|
+
for (const sel of selections) {
|
|
8954
|
+
if (typeof sel?.generationId !== "string" || typeof sel?.imageKey !== "string") {
|
|
8955
|
+
res.status(400).json({ detail: "Each selection needs generationId + imageKey." });
|
|
8956
|
+
return;
|
|
8957
|
+
}
|
|
8958
|
+
const row = rowsById.get(sel.generationId);
|
|
8959
|
+
if (!row) {
|
|
8960
|
+
res.status(404).json({ detail: `Generation ${sel.generationId} not found in this tool call.` });
|
|
8961
|
+
return;
|
|
8962
|
+
}
|
|
8963
|
+
const keys = Array.isArray(row.image_keys) ? row.image_keys : (() => {
|
|
8964
|
+
try {
|
|
8965
|
+
return JSON.parse(row.image_keys);
|
|
8966
|
+
} catch {
|
|
8967
|
+
return [];
|
|
8968
|
+
}
|
|
8969
|
+
})();
|
|
8970
|
+
if (!keys.includes(sel.imageKey)) {
|
|
8971
|
+
res.status(400).json({ detail: `imageKey not part of generation ${sel.generationId}.` });
|
|
8972
|
+
return;
|
|
8973
|
+
}
|
|
8974
|
+
const slash = sel.imageKey.indexOf("/");
|
|
8975
|
+
const bucket = slash > 0 ? sel.imageKey.slice(0, slash) : config.fileUploads.s3Bucket;
|
|
8976
|
+
const objectKey = slash > 0 ? sel.imageKey.slice(slash + 1) : sel.imageKey;
|
|
8977
|
+
const presignedUrl = await getPresignedUrl(bucket, objectKey, config);
|
|
8978
|
+
let styleName = null;
|
|
8979
|
+
if (row.applied_style_id) {
|
|
8980
|
+
const styleRow = await authed.db("platform_configurations").where({ id: row.applied_style_id }).first();
|
|
8981
|
+
const parsed = styleRow?.config_value && typeof styleRow.config_value === "string" ? (() => {
|
|
8982
|
+
try {
|
|
8983
|
+
return JSON.parse(styleRow.config_value);
|
|
8984
|
+
} catch {
|
|
8985
|
+
return null;
|
|
8986
|
+
}
|
|
8987
|
+
})() : styleRow?.config_value;
|
|
8988
|
+
styleName = parsed?.name ?? null;
|
|
8989
|
+
}
|
|
8990
|
+
selectedDetails.push({
|
|
8991
|
+
key: sel.imageKey,
|
|
8992
|
+
presignedUrl,
|
|
8993
|
+
prompt: row.prompt,
|
|
8994
|
+
model: row.model,
|
|
8995
|
+
styleName
|
|
8996
|
+
});
|
|
8997
|
+
rowsToMarkSelected.add(row.id);
|
|
8998
|
+
}
|
|
8999
|
+
await authed.db("image_generations").whereIn("id", [...rowsToMarkSelected]).update({ selected: true });
|
|
9000
|
+
const lines = selectedDetails.map(
|
|
9001
|
+
(d) => `- ${d.presignedUrl} (prompt: "${d.prompt}", model: ${d.model}${d.styleName ? `, style: ${d.styleName}` : ""})`
|
|
9002
|
+
);
|
|
9003
|
+
const messageText = "The user generated and selected the following image(s) in this chat:\n" + lines.join("\n");
|
|
9004
|
+
const messageId = randomUUID2();
|
|
9005
|
+
const uiMessage = {
|
|
9006
|
+
id: messageId,
|
|
9007
|
+
role: "system",
|
|
9008
|
+
parts: [{ type: "text", text: messageText }]
|
|
9009
|
+
};
|
|
9010
|
+
await authed.db("agent_messages").insert({
|
|
9011
|
+
content: JSON.stringify(uiMessage),
|
|
9012
|
+
message_id: messageId,
|
|
9013
|
+
session: sessionId,
|
|
9014
|
+
user: authed.user.id
|
|
9015
|
+
});
|
|
9016
|
+
res.status(200).json({ ok: true, systemMessage: uiMessage, selectedImages: selectedDetails });
|
|
9017
|
+
});
|
|
9018
|
+
app.get("/images/history", async (req, res) => {
|
|
9019
|
+
if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
|
|
9020
|
+
const sessionId = typeof req.query.sessionId === "string" ? req.query.sessionId : "";
|
|
9021
|
+
const toolCallId = typeof req.query.toolCallId === "string" ? req.query.toolCallId : "";
|
|
9022
|
+
if (!sessionId || !toolCallId) {
|
|
9023
|
+
res.status(400).json({ detail: "sessionId and toolCallId query params are required." });
|
|
9024
|
+
return;
|
|
9025
|
+
}
|
|
9026
|
+
const authed = await loadAuthedSession(req, res, sessionId, "read");
|
|
9027
|
+
if (!authed) return;
|
|
9028
|
+
const rows = await authed.db("image_generations").where({ session_id: sessionId, tool_call_id: toolCallId }).orderBy("createdAt", "asc").select("*");
|
|
9029
|
+
const parseList = (v) => {
|
|
9030
|
+
if (!v) return [];
|
|
9031
|
+
if (Array.isArray(v)) return v;
|
|
9032
|
+
try {
|
|
9033
|
+
return JSON.parse(v);
|
|
9034
|
+
} catch {
|
|
9035
|
+
return [];
|
|
9036
|
+
}
|
|
9037
|
+
};
|
|
9038
|
+
const sign = async (fullKey) => {
|
|
9039
|
+
const slash = fullKey.indexOf("/");
|
|
9040
|
+
const bucket = slash > 0 ? fullKey.slice(0, slash) : config.fileUploads.s3Bucket;
|
|
9041
|
+
const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
|
|
9042
|
+
return getPresignedUrl(bucket, objectKey, config);
|
|
9043
|
+
};
|
|
9044
|
+
const history = await Promise.all(rows.map(async (r) => {
|
|
9045
|
+
const keys = parseList(r.image_keys);
|
|
9046
|
+
const refs = parseList(r.reference_image_keys);
|
|
9047
|
+
const revisedPrompts = parseList(r.revised_prompts);
|
|
9048
|
+
const [imageUrls, referenceUrls] = await Promise.all([
|
|
9049
|
+
Promise.all(keys.map(sign)),
|
|
9050
|
+
Promise.all(refs.map(sign))
|
|
9051
|
+
]);
|
|
9052
|
+
return {
|
|
9053
|
+
generationId: r.id,
|
|
9054
|
+
createdAt: r.createdAt,
|
|
9055
|
+
operation: r.operation,
|
|
9056
|
+
model: r.model,
|
|
9057
|
+
prompt: r.prompt,
|
|
9058
|
+
appliedStyleId: r.applied_style_id ?? null,
|
|
9059
|
+
appliedStyleMarkdown: r.applied_style_markdown ?? null,
|
|
9060
|
+
size: r.size ?? null,
|
|
9061
|
+
quality: r.quality ?? null,
|
|
9062
|
+
n: r.n ?? 1,
|
|
9063
|
+
selected: !!r.selected,
|
|
9064
|
+
error: r.error ?? null,
|
|
9065
|
+
maskImageKey: r.mask_image_key ?? null,
|
|
9066
|
+
images: keys.map((key2, i) => ({
|
|
9067
|
+
key: key2,
|
|
9068
|
+
presignedUrl: imageUrls[i],
|
|
9069
|
+
revisedPrompt: revisedPrompts[i] ?? null
|
|
9070
|
+
})),
|
|
9071
|
+
references: refs.map((key2, i) => ({ key: key2, presignedUrl: referenceUrls[i] }))
|
|
9072
|
+
};
|
|
9073
|
+
}));
|
|
9074
|
+
res.status(200).json({ history });
|
|
9075
|
+
});
|
|
9076
|
+
app.use("/litellm/:project", async (req, res) => {
|
|
9077
|
+
if (!isLiteLLMEnabled()) {
|
|
9078
|
+
res.status(503).json({
|
|
9079
|
+
detail: "LiteLLM is not enabled on this deployment. Set EXULU_USE_LITELLM=true."
|
|
9080
|
+
});
|
|
9081
|
+
return;
|
|
9082
|
+
}
|
|
9083
|
+
const masterKey = process.env.LITELLM_MASTER_KEY;
|
|
9084
|
+
if (!masterKey) {
|
|
9085
|
+
res.status(503).json({ detail: "LITELLM_MASTER_KEY is not configured." });
|
|
9086
|
+
return;
|
|
9087
|
+
}
|
|
9088
|
+
const authenticationResult = await requestValidators.authenticate(req);
|
|
9089
|
+
if (!authenticationResult.user?.id) {
|
|
7891
9090
|
console.log("[EXULU] /litellm failed authentication", authenticationResult);
|
|
7892
9091
|
res.status(authenticationResult.code || 401).json({ detail: authenticationResult.message });
|
|
7893
9092
|
return;
|
|
@@ -11211,216 +12410,175 @@ var transcribeTool = new ExuluTool({
|
|
|
11211
12410
|
}
|
|
11212
12411
|
});
|
|
11213
12412
|
|
|
11214
|
-
// src/
|
|
11215
|
-
|
|
11216
|
-
|
|
11217
|
-
|
|
11218
|
-
|
|
11219
|
-
return isValid && length <= 80 && length > 2;
|
|
12413
|
+
// src/templates/tools/image-generation.ts
|
|
12414
|
+
import { z as z11 } from "zod";
|
|
12415
|
+
var _cachedImageModels;
|
|
12416
|
+
var setCachedImageModels = (models2) => {
|
|
12417
|
+
_cachedImageModels = models2;
|
|
11220
12418
|
};
|
|
11221
|
-
|
|
11222
|
-
|
|
11223
|
-
|
|
11224
|
-
|
|
11225
|
-
import { resolve, join as join3, dirname as dirname2 } from "path";
|
|
11226
|
-
import { existsSync as existsSync2, readFileSync } from "fs";
|
|
11227
|
-
import { fileURLToPath } from "url";
|
|
11228
|
-
var execAsync2 = promisify2(exec2);
|
|
11229
|
-
function getPackageRoot() {
|
|
11230
|
-
const currentFile = fileURLToPath(import.meta.url);
|
|
11231
|
-
let currentDir = dirname2(currentFile);
|
|
11232
|
-
let attempts = 0;
|
|
11233
|
-
const maxAttempts = 10;
|
|
11234
|
-
while (attempts < maxAttempts) {
|
|
11235
|
-
const packageJsonPath = join3(currentDir, "package.json");
|
|
11236
|
-
if (existsSync2(packageJsonPath)) {
|
|
11237
|
-
try {
|
|
11238
|
-
const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
|
|
11239
|
-
if (packageJson.name === "@exulu/backend") {
|
|
11240
|
-
return currentDir;
|
|
11241
|
-
}
|
|
11242
|
-
} catch {
|
|
11243
|
-
}
|
|
11244
|
-
}
|
|
11245
|
-
const parentDir = resolve(currentDir, "..");
|
|
11246
|
-
if (parentDir === currentDir) {
|
|
11247
|
-
break;
|
|
11248
|
-
}
|
|
11249
|
-
currentDir = parentDir;
|
|
11250
|
-
attempts++;
|
|
11251
|
-
}
|
|
11252
|
-
const fallback = resolve(dirname2(fileURLToPath(import.meta.url)), "../..");
|
|
11253
|
-
return fallback;
|
|
11254
|
-
}
|
|
11255
|
-
function getSetupScriptPath(packageRoot) {
|
|
11256
|
-
return resolve(packageRoot, "ee/python/setup.sh");
|
|
11257
|
-
}
|
|
11258
|
-
function getVenvPath(packageRoot) {
|
|
11259
|
-
return resolve(packageRoot, "ee/python/.venv");
|
|
11260
|
-
}
|
|
11261
|
-
function isPythonEnvironmentSetup(packageRoot) {
|
|
11262
|
-
const root = packageRoot ?? getPackageRoot();
|
|
11263
|
-
const venvPath = getVenvPath(root);
|
|
11264
|
-
const pythonPath = join3(venvPath, "bin", "python");
|
|
11265
|
-
return existsSync2(venvPath) && existsSync2(pythonPath);
|
|
11266
|
-
}
|
|
11267
|
-
async function setupPythonEnvironment(options = {}) {
|
|
11268
|
-
const {
|
|
11269
|
-
packageRoot = getPackageRoot(),
|
|
11270
|
-
force = false,
|
|
11271
|
-
verbose = false,
|
|
11272
|
-
timeout = 6e5
|
|
11273
|
-
// 10 minutes
|
|
11274
|
-
} = options;
|
|
11275
|
-
if (!force && isPythonEnvironmentSetup(packageRoot)) {
|
|
11276
|
-
if (verbose) {
|
|
11277
|
-
console.log("\u2713 Python environment already set up");
|
|
11278
|
-
}
|
|
11279
|
-
return {
|
|
11280
|
-
success: true,
|
|
11281
|
-
message: "Python environment already exists",
|
|
11282
|
-
alreadyExists: true
|
|
11283
|
-
};
|
|
12419
|
+
var buildDefaults = (models2) => {
|
|
12420
|
+
const m = models2[0];
|
|
12421
|
+
if (!m) {
|
|
12422
|
+
return { model: "", size: "1024x1024", quality: "auto", n: 1 };
|
|
11284
12423
|
}
|
|
11285
|
-
|
|
11286
|
-
|
|
11287
|
-
|
|
11288
|
-
|
|
11289
|
-
|
|
11290
|
-
|
|
11291
|
-
|
|
11292
|
-
|
|
11293
|
-
|
|
11294
|
-
|
|
11295
|
-
|
|
11296
|
-
|
|
11297
|
-
|
|
11298
|
-
|
|
11299
|
-
|
|
11300
|
-
|
|
11301
|
-
|
|
11302
|
-
|
|
11303
|
-
|
|
11304
|
-
|
|
11305
|
-
|
|
11306
|
-
|
|
12424
|
+
return {
|
|
12425
|
+
model: m.model_name,
|
|
12426
|
+
size: m.sizes.includes("1024x1024") ? "1024x1024" : m.sizes[0],
|
|
12427
|
+
quality: m.qualities.includes("auto") ? "auto" : m.qualities[0],
|
|
12428
|
+
n: 1
|
|
12429
|
+
};
|
|
12430
|
+
};
|
|
12431
|
+
var loadAvailableStyles = async (user) => {
|
|
12432
|
+
if (!user?.id) return [];
|
|
12433
|
+
const { db } = await postgresClient();
|
|
12434
|
+
const rows = await db.from("platform_configurations").where("config_key", "like", "image_generation_style:%").select("*");
|
|
12435
|
+
const visible = [];
|
|
12436
|
+
for (const row of rows) {
|
|
12437
|
+
const rbac = await RBACResolver(
|
|
12438
|
+
db,
|
|
12439
|
+
"platform_configurations",
|
|
12440
|
+
row.id,
|
|
12441
|
+
row.rights_mode || "private"
|
|
12442
|
+
);
|
|
12443
|
+
const hasAccess = await checkRecordAccess(
|
|
12444
|
+
{ ...row, RBAC: rbac },
|
|
12445
|
+
"read",
|
|
12446
|
+
user
|
|
12447
|
+
);
|
|
12448
|
+
if (!hasAccess) continue;
|
|
12449
|
+
const value = typeof row.config_value === "string" ? safeJsonParse(row.config_value) : row.config_value;
|
|
12450
|
+
visible.push({
|
|
12451
|
+
id: row.id,
|
|
12452
|
+
name: value?.name ?? row.config_key.replace(/^image_generation_style:/, ""),
|
|
12453
|
+
description: row.description ?? null,
|
|
12454
|
+
owner: String(row.created_by) === String(user.id) ? "user" : "shared"
|
|
11307
12455
|
});
|
|
11308
|
-
const output = stdout + stderr;
|
|
11309
|
-
const versionMatch = output.match(/Python (\d+\.\d+\.\d+)/);
|
|
11310
|
-
const pythonVersion = versionMatch ? versionMatch[1] : void 0;
|
|
11311
|
-
if (verbose) {
|
|
11312
|
-
console.log(output);
|
|
11313
|
-
}
|
|
11314
|
-
return {
|
|
11315
|
-
success: true,
|
|
11316
|
-
message: "Python environment set up successfully",
|
|
11317
|
-
alreadyExists: false,
|
|
11318
|
-
pythonVersion,
|
|
11319
|
-
output
|
|
11320
|
-
};
|
|
11321
|
-
} catch (error) {
|
|
11322
|
-
const errorOutput = error.stdout + error.stderr;
|
|
11323
|
-
return {
|
|
11324
|
-
success: false,
|
|
11325
|
-
message: `Setup failed: ${error.message}`,
|
|
11326
|
-
alreadyExists: false,
|
|
11327
|
-
output: errorOutput
|
|
11328
|
-
};
|
|
11329
|
-
}
|
|
11330
|
-
}
|
|
11331
|
-
function getPythonSetupInstructions() {
|
|
11332
|
-
return `
|
|
11333
|
-
Python environment not set up. Please run one of the following commands:
|
|
11334
|
-
|
|
11335
|
-
Option 1 (Automatic):
|
|
11336
|
-
import { setupPythonEnvironment } from '@exulu/backend';
|
|
11337
|
-
await setupPythonEnvironment();
|
|
11338
|
-
|
|
11339
|
-
Option 2 (Manual - for package consumers):
|
|
11340
|
-
npx @exulu/backend setup-python
|
|
11341
|
-
|
|
11342
|
-
Option 3 (Manual - for contributors):
|
|
11343
|
-
npm run python:setup
|
|
11344
|
-
|
|
11345
|
-
These commands will automatically create a Python virtual environment (.venv)
|
|
11346
|
-
in the @exulu/backend package and install all required dependencies.
|
|
11347
|
-
|
|
11348
|
-
Requirements:
|
|
11349
|
-
- Python 3.10 or higher must be installed
|
|
11350
|
-
- pip must be available
|
|
11351
|
-
- venv module must be available (for creating virtual environments)
|
|
11352
|
-
|
|
11353
|
-
If Python dependencies are not installed, install them first, then run one of the commands above:
|
|
11354
|
-
- macOS: brew install python@3.12
|
|
11355
|
-
- Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
|
|
11356
|
-
- Alpine Linux: apk add python3 py3-pip python3-dev
|
|
11357
|
-
- Windows: Download from https://www.python.org/downloads/
|
|
11358
|
-
|
|
11359
|
-
Note: In Docker containers, ensure you install all three components:
|
|
11360
|
-
Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
|
|
11361
|
-
Alpine: apk add python3 py3-pip python3-dev
|
|
11362
|
-
`.trim();
|
|
11363
|
-
}
|
|
11364
|
-
async function validatePythonEnvironment(packageRoot, checkPackages = true) {
|
|
11365
|
-
const root = packageRoot ?? getPackageRoot();
|
|
11366
|
-
const venvPath = getVenvPath(root);
|
|
11367
|
-
const pythonPath = join3(venvPath, "bin", "python");
|
|
11368
|
-
if (!existsSync2(venvPath)) {
|
|
11369
|
-
return {
|
|
11370
|
-
valid: false,
|
|
11371
|
-
message: getPythonSetupInstructions()
|
|
11372
|
-
};
|
|
11373
|
-
}
|
|
11374
|
-
if (!existsSync2(pythonPath)) {
|
|
11375
|
-
return {
|
|
11376
|
-
valid: false,
|
|
11377
|
-
message: "Python virtual environment is corrupted. Please run:\n await setupPythonEnvironment({ force: true })"
|
|
11378
|
-
};
|
|
11379
12456
|
}
|
|
12457
|
+
return visible;
|
|
12458
|
+
};
|
|
12459
|
+
var safeJsonParse = (s) => {
|
|
11380
12460
|
try {
|
|
11381
|
-
|
|
12461
|
+
return JSON.parse(s);
|
|
11382
12462
|
} catch {
|
|
11383
|
-
return
|
|
11384
|
-
valid: false,
|
|
11385
|
-
message: "Python executable is not working. Please run:\n await setupPythonEnvironment({ force: true })"
|
|
11386
|
-
};
|
|
12463
|
+
return null;
|
|
11387
12464
|
}
|
|
11388
|
-
|
|
11389
|
-
|
|
11390
|
-
|
|
11391
|
-
|
|
11392
|
-
|
|
11393
|
-
|
|
11394
|
-
|
|
11395
|
-
|
|
11396
|
-
|
|
11397
|
-
|
|
11398
|
-
|
|
11399
|
-
|
|
12465
|
+
};
|
|
12466
|
+
var createImageGenerationWidgetTool = (models2) => {
|
|
12467
|
+
setCachedImageModels(models2);
|
|
12468
|
+
return new ExuluTool({
|
|
12469
|
+
id: "image_generation",
|
|
12470
|
+
name: "image_generation",
|
|
12471
|
+
description: "Open an in-chat image generation widget pre-filled with your prompt. The user picks the model, size, quality and count, optionally attaches reference images for editing, applies a saved style, generates one or more candidates, and selects the final image(s) to share back into the conversation. Use this whenever the user asks to create or edit an image.",
|
|
12472
|
+
needsApproval: false,
|
|
12473
|
+
type: "function",
|
|
12474
|
+
config: [],
|
|
12475
|
+
inputSchema: z11.object({
|
|
12476
|
+
prompt: z11.string().describe(
|
|
12477
|
+
"Initial image prompt. The user can edit it before generating."
|
|
12478
|
+
)
|
|
12479
|
+
}),
|
|
12480
|
+
execute: async ({ prompt, user, sessionID }, options) => {
|
|
12481
|
+
if (!isLiteLLMEnabled()) {
|
|
12482
|
+
throw new Error(
|
|
12483
|
+
"Image generation is not enabled on this deployment (EXULU_USE_LITELLM is not 'true')."
|
|
12484
|
+
);
|
|
11400
12485
|
}
|
|
11401
|
-
|
|
11402
|
-
|
|
12486
|
+
if (!_cachedImageModels || _cachedImageModels.length === 0) {
|
|
12487
|
+
throw new Error(
|
|
12488
|
+
"No image-generation models are registered in config.litellm.yaml."
|
|
12489
|
+
);
|
|
12490
|
+
}
|
|
12491
|
+
const toolCallId = options?.toolCallId;
|
|
12492
|
+
const styles = await loadAvailableStyles(user);
|
|
11403
12493
|
return {
|
|
11404
|
-
|
|
11405
|
-
|
|
12494
|
+
result: JSON.stringify({
|
|
12495
|
+
type: "image_generation_widget",
|
|
12496
|
+
toolCallId,
|
|
12497
|
+
sessionId: sessionID,
|
|
12498
|
+
initialPrompt: prompt,
|
|
12499
|
+
models: _cachedImageModels.map((m) => ({
|
|
12500
|
+
name: m.model_name,
|
|
12501
|
+
sizes: m.sizes,
|
|
12502
|
+
qualities: m.qualities,
|
|
12503
|
+
supportsEdit: m.supports_edit,
|
|
12504
|
+
maxN: m.max_n
|
|
12505
|
+
})),
|
|
12506
|
+
styles,
|
|
12507
|
+
defaults: buildDefaults(_cachedImageModels)
|
|
12508
|
+
})
|
|
12509
|
+
};
|
|
12510
|
+
}
|
|
12511
|
+
});
|
|
12512
|
+
};
|
|
11406
12513
|
|
|
11407
|
-
|
|
11408
|
-
|
|
11409
|
-
|
|
12514
|
+
// src/exulu/app/index.ts
|
|
12515
|
+
import { resolve } from "path";
|
|
12516
|
+
|
|
12517
|
+
// src/validators/postgres-name.ts
|
|
12518
|
+
var isValidPostgresName = (id) => {
|
|
12519
|
+
const regex = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
12520
|
+
const isValid = regex.test(id);
|
|
12521
|
+
const length = id.length;
|
|
12522
|
+
return isValid && length <= 80 && length > 2;
|
|
12523
|
+
};
|
|
11410
12524
|
|
|
11411
|
-
|
|
11412
|
-
|
|
12525
|
+
// src/templates/contexts/transcriptions.ts
|
|
12526
|
+
var transcriptionsContext = new ExuluContext({
|
|
12527
|
+
id: "transcriptions",
|
|
12528
|
+
name: "Transcriptions",
|
|
12529
|
+
description: "Diarized audio transcripts",
|
|
12530
|
+
fields: [
|
|
12531
|
+
{ name: "transcript_text", type: "longText", editable: true },
|
|
12532
|
+
{ name: "audio", type: "file" },
|
|
12533
|
+
{ name: "language", type: "text" },
|
|
12534
|
+
{ name: "duration_seconds", type: "number" },
|
|
12535
|
+
{ name: "speakers", type: "json" },
|
|
12536
|
+
{ name: "raw_segments", type: "json", editable: false }
|
|
12537
|
+
],
|
|
12538
|
+
sources: [],
|
|
12539
|
+
active: true,
|
|
12540
|
+
configuration: {
|
|
12541
|
+
calculateVectors: "onInsert",
|
|
12542
|
+
defaultRightsMode: "private"
|
|
12543
|
+
}
|
|
12544
|
+
});
|
|
11413
12545
|
|
|
11414
|
-
|
|
11415
|
-
|
|
11416
|
-
|
|
12546
|
+
// src/templates/contexts/index.ts
|
|
12547
|
+
var builtInContexts = {
|
|
12548
|
+
transcriptions: transcriptionsContext
|
|
12549
|
+
};
|
|
12550
|
+
|
|
12551
|
+
// src/exulu/transcription/polling-loop.ts
|
|
12552
|
+
var POLL_INTERVAL_MS = 5e3;
|
|
12553
|
+
var MAX_PER_TICK = 50;
|
|
12554
|
+
var timer = null;
|
|
12555
|
+
var stopped = false;
|
|
12556
|
+
var tick = async () => {
|
|
12557
|
+
if (stopped) return;
|
|
12558
|
+
try {
|
|
12559
|
+
await transcriptionService.pollOnce(MAX_PER_TICK);
|
|
12560
|
+
} catch (err) {
|
|
12561
|
+
console.error(`[EXULU-TRANSCRIPTION] polling tick failed: ${err.message}`);
|
|
12562
|
+
} finally {
|
|
12563
|
+
if (!stopped) {
|
|
12564
|
+
timer = setTimeout(tick, POLL_INTERVAL_MS);
|
|
11417
12565
|
}
|
|
11418
12566
|
}
|
|
11419
|
-
|
|
11420
|
-
|
|
11421
|
-
|
|
12567
|
+
};
|
|
12568
|
+
var startTranscriptionPollingLoop = () => {
|
|
12569
|
+
if (timer) return;
|
|
12570
|
+
stopped = false;
|
|
12571
|
+
timer = setTimeout(tick, POLL_INTERVAL_MS);
|
|
12572
|
+
const stop = () => {
|
|
12573
|
+
stopped = true;
|
|
12574
|
+
if (timer) {
|
|
12575
|
+
clearTimeout(timer);
|
|
12576
|
+
timer = null;
|
|
12577
|
+
}
|
|
11422
12578
|
};
|
|
11423
|
-
|
|
12579
|
+
process.on("SIGINT", stop);
|
|
12580
|
+
process.on("SIGTERM", stop);
|
|
12581
|
+
};
|
|
11424
12582
|
|
|
11425
12583
|
// src/exulu/app/index.ts
|
|
11426
12584
|
var isDev = process.env.NODE_ENV !== "production";
|
|
@@ -11486,8 +12644,14 @@ var ExuluApp = class {
|
|
|
11486
12644
|
rerankers
|
|
11487
12645
|
}) => {
|
|
11488
12646
|
this._evals = redisServer.host?.length && redisServer.port?.length ? [...getDefaultEvals(), ...evals ?? []] : [];
|
|
12647
|
+
if (contexts && "transcriptions" in contexts) {
|
|
12648
|
+
console.warn(
|
|
12649
|
+
"[EXULU] User-defined 'transcriptions' context overridden by built-in. Rename your context to avoid the collision."
|
|
12650
|
+
);
|
|
12651
|
+
}
|
|
11489
12652
|
this._contexts = {
|
|
11490
|
-
...contexts
|
|
12653
|
+
...contexts,
|
|
12654
|
+
...builtInContexts
|
|
11491
12655
|
};
|
|
11492
12656
|
this._rerankers = [...rerankers ?? []];
|
|
11493
12657
|
this._agents = [...agents ?? []];
|
|
@@ -11518,13 +12682,26 @@ var ExuluApp = class {
|
|
|
11518
12682
|
if (process.env.TRANSCRIPTION_MODEL && config?.fileUploads && config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
|
|
11519
12683
|
transcriptionTools.push(transcribeTool);
|
|
11520
12684
|
}
|
|
12685
|
+
const imageGenerationTools = [];
|
|
12686
|
+
const s3Configured = !!config?.fileUploads && !!config.fileUploads.s3region && !!config.fileUploads.s3key && !!config.fileUploads.s3secret && !!config.fileUploads.s3Bucket;
|
|
12687
|
+
if (isLiteLLMEnabled() && s3Configured) {
|
|
12688
|
+
const configPath = process.env.LITELLM_CONFIG_PATH ?? resolve(getPackageRoot(), "./config.litellm.yaml");
|
|
12689
|
+
const imageModels = parseImageGenerationModels(configPath);
|
|
12690
|
+
if (imageModels.length > 0) {
|
|
12691
|
+
console.log(
|
|
12692
|
+
`[EXULU] Registering image_generation widget tool with ${imageModels.length} model(s): ${imageModels.map((m) => m.model_name).join(", ")}`
|
|
12693
|
+
);
|
|
12694
|
+
imageGenerationTools.push(createImageGenerationWidgetTool(imageModels));
|
|
12695
|
+
}
|
|
12696
|
+
}
|
|
11521
12697
|
this._tools = [
|
|
11522
12698
|
...tools ?? [],
|
|
11523
12699
|
...todoTools,
|
|
11524
12700
|
...questionTools,
|
|
11525
12701
|
...perplexityTools,
|
|
11526
12702
|
emailTool,
|
|
11527
|
-
...transcriptionTools
|
|
12703
|
+
...transcriptionTools,
|
|
12704
|
+
...imageGenerationTools
|
|
11528
12705
|
// Because agents are stored in the database, we add those as tools
|
|
11529
12706
|
// at request time, not during ExuluApp initialization. We add them
|
|
11530
12707
|
// in the grahql tools resolver.
|
|
@@ -11623,6 +12800,23 @@ var ExuluApp = class {
|
|
|
11623
12800
|
);
|
|
11624
12801
|
}
|
|
11625
12802
|
}
|
|
12803
|
+
if (process.env.TRANSCRIPTION_SERVER) {
|
|
12804
|
+
try {
|
|
12805
|
+
const health = await transcriptionClient.health();
|
|
12806
|
+
console.log(
|
|
12807
|
+
`[EXULU] Transcription: enabled (server=${process.env.TRANSCRIPTION_SERVER}, device=${health.device}, GPU=${health.gpu.available ? "enabled" : "disabled"}, diarization=${health.diarization ? "enabled" : "disabled"})`
|
|
12808
|
+
);
|
|
12809
|
+
startTranscriptionPollingLoop();
|
|
12810
|
+
} catch (err) {
|
|
12811
|
+
console.warn(
|
|
12812
|
+
`[EXULU] TRANSCRIPTION_SERVER set but unreachable: ${err.message}. Transcriptions will fail until the server is up.`
|
|
12813
|
+
);
|
|
12814
|
+
}
|
|
12815
|
+
} else {
|
|
12816
|
+
console.log(
|
|
12817
|
+
"[EXULU] Transcription: disabled (TRANSCRIPTION_SERVER not set). Start a whisper server with `npx @exulu/backend exulu-start-whisper`."
|
|
12818
|
+
);
|
|
12819
|
+
}
|
|
11626
12820
|
return this._expressApp;
|
|
11627
12821
|
}
|
|
11628
12822
|
};
|
|
@@ -13406,7 +14600,9 @@ var {
|
|
|
13406
14600
|
promptLibrarySchema: promptLibrarySchema2,
|
|
13407
14601
|
contextPresetsSchema: contextPresetsSchema2,
|
|
13408
14602
|
embedderSettingsSchema: embedderSettingsSchema2,
|
|
13409
|
-
promptFavoritesSchema: promptFavoritesSchema2
|
|
14603
|
+
promptFavoritesSchema: promptFavoritesSchema2,
|
|
14604
|
+
transcriptionJobsSchema: transcriptionJobsSchema2,
|
|
14605
|
+
imageGenerationsSchema
|
|
13410
14606
|
} = coreSchemas.get();
|
|
13411
14607
|
var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
|
|
13412
14608
|
for (const field of fields) {
|
|
@@ -13446,6 +14642,8 @@ var up = async function(knex) {
|
|
|
13446
14642
|
contextPresetsSchema2(),
|
|
13447
14643
|
embedderSettingsSchema2(),
|
|
13448
14644
|
promptFavoritesSchema2(),
|
|
14645
|
+
transcriptionJobsSchema2(),
|
|
14646
|
+
imageGenerationsSchema(),
|
|
13449
14647
|
rbacSchema2(),
|
|
13450
14648
|
agentsSchema2(),
|
|
13451
14649
|
feedbackSchema2(),
|
|
@@ -13720,7 +14918,7 @@ ${WARNING_BANNER}`);
|
|
|
13720
14918
|
console.warn(`${WARNING_BANNER}
|
|
13721
14919
|
`);
|
|
13722
14920
|
};
|
|
13723
|
-
var
|
|
14921
|
+
var log2 = (line) => console.log(`[EXULU-LITELLM] ${line}`);
|
|
13724
14922
|
var initLiteLLMDatabase = async (packageRoot) => {
|
|
13725
14923
|
const configPath = process.env.LITELLM_CONFIG_PATH ?? resolve2(packageRoot, "./config.litellm.yaml");
|
|
13726
14924
|
const safety = checkLiteLLMDatabaseSafety(configPath);
|
|
@@ -13754,7 +14952,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13754
14952
|
return;
|
|
13755
14953
|
}
|
|
13756
14954
|
const target = "litellmTarget" in safety ? safety.litellmTarget : void 0;
|
|
13757
|
-
|
|
14955
|
+
log2(
|
|
13758
14956
|
`LiteLLM database mode detected (${target?.host}:${target?.port}/${target?.database}).`
|
|
13759
14957
|
);
|
|
13760
14958
|
const ensureDatabaseExists = async () => {
|
|
@@ -13782,7 +14980,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13782
14980
|
return false;
|
|
13783
14981
|
}
|
|
13784
14982
|
url.pathname = "/postgres";
|
|
13785
|
-
|
|
14983
|
+
log2(`Target database "${targetDbName}" does not exist; creating it\u2026`);
|
|
13786
14984
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(targetDbName)) {
|
|
13787
14985
|
warn([
|
|
13788
14986
|
`Refusing to auto-create database "${targetDbName}" \u2014 name`,
|
|
@@ -13795,7 +14993,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13795
14993
|
try {
|
|
13796
14994
|
await admin.connect();
|
|
13797
14995
|
await admin.query(`CREATE DATABASE "${targetDbName}"`);
|
|
13798
|
-
|
|
14996
|
+
log2(`\u2713 Created database "${targetDbName}".`);
|
|
13799
14997
|
return true;
|
|
13800
14998
|
} catch (createErr) {
|
|
13801
14999
|
warn([
|
|
@@ -13817,7 +15015,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13817
15015
|
}
|
|
13818
15016
|
};
|
|
13819
15017
|
if (!await ensureDatabaseExists()) return;
|
|
13820
|
-
|
|
15018
|
+
log2("Checking that the target database is safe to push into\u2026");
|
|
13821
15019
|
const client2 = new Client({ connectionString: litellmUrl });
|
|
13822
15020
|
let foreignTables = [];
|
|
13823
15021
|
try {
|
|
@@ -13882,7 +15080,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13882
15080
|
]);
|
|
13883
15081
|
return;
|
|
13884
15082
|
}
|
|
13885
|
-
|
|
15083
|
+
log2("Running `prisma db push` against LiteLLM's schema\u2026");
|
|
13886
15084
|
const result = spawnSync(prismaCli, ["db", "push", "--skip-generate"], {
|
|
13887
15085
|
cwd: litellmProxyDir,
|
|
13888
15086
|
env: {
|
|
@@ -13912,7 +15110,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13912
15110
|
]);
|
|
13913
15111
|
return;
|
|
13914
15112
|
}
|
|
13915
|
-
|
|
15113
|
+
log2("\u2713 LiteLLM database ready.");
|
|
13916
15114
|
};
|
|
13917
15115
|
|
|
13918
15116
|
// src/postgres/init-litellm-db.ts
|
|
@@ -14449,7 +15647,7 @@ var MarkdownChunker = class {
|
|
|
14449
15647
|
import * as fs4 from "fs";
|
|
14450
15648
|
import * as path from "path";
|
|
14451
15649
|
import { generateText as generateText5, Output as Output2 } from "ai";
|
|
14452
|
-
import { z as
|
|
15650
|
+
import { z as z12 } from "zod";
|
|
14453
15651
|
import pLimit from "p-limit";
|
|
14454
15652
|
import { randomUUID as randomUUID6 } from "crypto";
|
|
14455
15653
|
import * as mammoth from "mammoth";
|
|
@@ -14458,19 +15656,19 @@ import WordExtractor from "word-extractor";
|
|
|
14458
15656
|
import { parseOfficeAsync as parseOfficeAsync2 } from "officeparser";
|
|
14459
15657
|
|
|
14460
15658
|
// src/utils/python-executor.ts
|
|
14461
|
-
import { exec as
|
|
14462
|
-
import { promisify as
|
|
14463
|
-
import { resolve as resolve3, join as
|
|
15659
|
+
import { exec as exec2 } from "child_process";
|
|
15660
|
+
import { promisify as promisify2 } from "util";
|
|
15661
|
+
import { resolve as resolve3, join as join3, dirname as dirname2 } from "path";
|
|
14464
15662
|
import { existsSync as existsSync5, readFileSync as readFileSync3 } from "fs";
|
|
14465
|
-
import { fileURLToPath
|
|
14466
|
-
var
|
|
15663
|
+
import { fileURLToPath } from "url";
|
|
15664
|
+
var execAsync2 = promisify2(exec2);
|
|
14467
15665
|
function getPackageRoot2() {
|
|
14468
|
-
const currentFile =
|
|
14469
|
-
let currentDir =
|
|
15666
|
+
const currentFile = fileURLToPath(import.meta.url);
|
|
15667
|
+
let currentDir = dirname2(currentFile);
|
|
14470
15668
|
let attempts = 0;
|
|
14471
15669
|
const maxAttempts = 10;
|
|
14472
15670
|
while (attempts < maxAttempts) {
|
|
14473
|
-
const packageJsonPath =
|
|
15671
|
+
const packageJsonPath = join3(currentDir, "package.json");
|
|
14474
15672
|
if (existsSync5(packageJsonPath)) {
|
|
14475
15673
|
try {
|
|
14476
15674
|
const packageJson = JSON.parse(readFileSync3(packageJsonPath, "utf-8"));
|
|
@@ -14487,7 +15685,7 @@ function getPackageRoot2() {
|
|
|
14487
15685
|
currentDir = parentDir;
|
|
14488
15686
|
attempts++;
|
|
14489
15687
|
}
|
|
14490
|
-
return resolve3(
|
|
15688
|
+
return resolve3(dirname2(fileURLToPath(import.meta.url)), "../..");
|
|
14491
15689
|
}
|
|
14492
15690
|
var PythonEnvironmentError = class extends Error {
|
|
14493
15691
|
constructor(message) {
|
|
@@ -14507,12 +15705,12 @@ var PythonExecutionError = class extends Error {
|
|
|
14507
15705
|
this.exitCode = exitCode;
|
|
14508
15706
|
}
|
|
14509
15707
|
};
|
|
14510
|
-
function
|
|
15708
|
+
function getVenvPath(packageRoot) {
|
|
14511
15709
|
return resolve3(packageRoot, "ee/python/.venv");
|
|
14512
15710
|
}
|
|
14513
15711
|
function getPythonExecutable(packageRoot) {
|
|
14514
|
-
const venvPath =
|
|
14515
|
-
return
|
|
15712
|
+
const venvPath = getVenvPath(packageRoot);
|
|
15713
|
+
return join3(venvPath, "bin", "python");
|
|
14516
15714
|
}
|
|
14517
15715
|
async function validatePythonEnvironmentForExecution(packageRoot) {
|
|
14518
15716
|
const validation = await validatePythonEnvironment(packageRoot);
|
|
@@ -14549,7 +15747,7 @@ async function executePythonScript(config) {
|
|
|
14549
15747
|
});
|
|
14550
15748
|
const command = `${pythonExecutable} "${resolvedScriptPath}" ${quotedArgs.join(" ")}`;
|
|
14551
15749
|
try {
|
|
14552
|
-
const { stdout, stderr } = await
|
|
15750
|
+
const { stdout, stderr } = await execAsync2(command, {
|
|
14553
15751
|
cwd,
|
|
14554
15752
|
timeout,
|
|
14555
15753
|
env: {
|
|
@@ -14801,15 +15999,15 @@ If the page contains a flow-chart, schematic, technical drawing or control board
|
|
|
14801
15999
|
const result = await generateText5({
|
|
14802
16000
|
model,
|
|
14803
16001
|
output: Output2.object({
|
|
14804
|
-
schema:
|
|
14805
|
-
needs_correction:
|
|
14806
|
-
corrected_text:
|
|
14807
|
-
current_page_table:
|
|
14808
|
-
headers:
|
|
14809
|
-
is_continuation:
|
|
16002
|
+
schema: z12.object({
|
|
16003
|
+
needs_correction: z12.boolean(),
|
|
16004
|
+
corrected_text: z12.string().nullable(),
|
|
16005
|
+
current_page_table: z12.object({
|
|
16006
|
+
headers: z12.array(z12.string()),
|
|
16007
|
+
is_continuation: z12.boolean()
|
|
14810
16008
|
}).nullable(),
|
|
14811
|
-
confidence:
|
|
14812
|
-
reasoning:
|
|
16009
|
+
confidence: z12.enum(["high", "medium", "low"]),
|
|
16010
|
+
reasoning: z12.string()
|
|
14813
16011
|
})
|
|
14814
16012
|
}),
|
|
14815
16013
|
messages: [
|