@exulu/backend 1.59.0 → 1.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{catalog-EOKGOHTY.js → catalog-BWE6SLE2.js} +1 -1
- package/dist/chunk-IDHS2BZO.js +210 -0
- package/dist/{chunk-YS27XOXI.js → chunk-ILAHW4UT.js} +5 -1
- package/dist/{chunk-U36VJDZ7.js → chunk-MPV7HBV6.js} +66 -4
- package/dist/cli/start-whisper.cjs +240 -0
- package/dist/cli/start-whisper.d.cts +1 -0
- package/dist/cli/start-whisper.d.ts +1 -0
- package/dist/cli/start-whisper.js +204 -0
- package/dist/{convert-exulu-tools-to-ai-sdk-tools-ZEECMX43.js → convert-exulu-tools-to-ai-sdk-tools-CULC37U6.js} +1 -1
- package/dist/index.cjs +2110 -412
- package/dist/index.d.cts +2 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1647 -237
- package/ee/python/requirements.txt +18 -0
- package/ee/python/setup.sh +44 -0
- package/ee/python/transcription/__init__.py +0 -0
- package/ee/python/transcription/pipeline.py +232 -0
- package/ee/python/transcription/server.py +151 -0
- package/ee/python/transcription/tests/__init__.py +0 -0
- package/ee/python/transcription/tests/test_server.py +111 -0
- package/ee/python/transcription/worker.py +135 -0
- package/package.json +4 -2
package/dist/index.js
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getPackageRoot,
|
|
3
|
+
getPythonSetupInstructions,
|
|
4
|
+
isPythonEnvironmentSetup,
|
|
5
|
+
setupPythonEnvironment,
|
|
6
|
+
validatePythonEnvironment
|
|
7
|
+
} from "./chunk-IDHS2BZO.js";
|
|
1
8
|
import {
|
|
2
9
|
ExuluContext,
|
|
3
10
|
ExuluStorage,
|
|
@@ -46,10 +53,10 @@ import {
|
|
|
46
53
|
vectorSearch,
|
|
47
54
|
waitForLiteLLMReady,
|
|
48
55
|
withRetry
|
|
49
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-MPV7HBV6.js";
|
|
50
57
|
import {
|
|
51
58
|
findLiteLLMModel
|
|
52
|
-
} from "./chunk-
|
|
59
|
+
} from "./chunk-ILAHW4UT.js";
|
|
53
60
|
|
|
54
61
|
// src/index.ts
|
|
55
62
|
import "dotenv/config";
|
|
@@ -3358,6 +3365,365 @@ function getAverage(arr) {
|
|
|
3358
3365
|
|
|
3359
3366
|
// src/graphql/schemas/index.ts
|
|
3360
3367
|
import "fs";
|
|
3368
|
+
|
|
3369
|
+
// src/exulu/transcription/client.ts
|
|
3370
|
+
var TranscriptionServerUnavailable = class extends Error {
|
|
3371
|
+
constructor(message) {
|
|
3372
|
+
super(message);
|
|
3373
|
+
this.name = "TranscriptionServerUnavailable";
|
|
3374
|
+
}
|
|
3375
|
+
};
|
|
3376
|
+
var getBaseUrl = () => {
|
|
3377
|
+
const url = process.env.TRANSCRIPTION_SERVER;
|
|
3378
|
+
if (!url) {
|
|
3379
|
+
throw new TranscriptionServerUnavailable(
|
|
3380
|
+
"TRANSCRIPTION_SERVER env var is not set. Start a whisper server with `npx @exulu/backend exulu-start-whisper` and point TRANSCRIPTION_SERVER at it."
|
|
3381
|
+
);
|
|
3382
|
+
}
|
|
3383
|
+
return url.replace(/\/$/, "");
|
|
3384
|
+
};
|
|
3385
|
+
var request = async (path2, init = {}) => {
|
|
3386
|
+
const url = `${getBaseUrl()}${path2}`;
|
|
3387
|
+
let res;
|
|
3388
|
+
try {
|
|
3389
|
+
res = await fetch(url, init);
|
|
3390
|
+
} catch (err) {
|
|
3391
|
+
throw new TranscriptionServerUnavailable(
|
|
3392
|
+
`Unable to reach whisper server at ${url}: ${err.message}`
|
|
3393
|
+
);
|
|
3394
|
+
}
|
|
3395
|
+
if (res.status === 404) {
|
|
3396
|
+
const err = new Error(`whisper server returned 404 for ${path2}`);
|
|
3397
|
+
err.code = "JOB_NOT_FOUND";
|
|
3398
|
+
throw err;
|
|
3399
|
+
}
|
|
3400
|
+
if (!res.ok) {
|
|
3401
|
+
throw new Error(
|
|
3402
|
+
`whisper server returned ${res.status} for ${path2}: ${await res.text()}`
|
|
3403
|
+
);
|
|
3404
|
+
}
|
|
3405
|
+
return await res.json();
|
|
3406
|
+
};
|
|
3407
|
+
var transcriptionClient = {
|
|
3408
|
+
submitJob: (opts) => request("/jobs", {
|
|
3409
|
+
method: "POST",
|
|
3410
|
+
headers: { "content-type": "application/json" },
|
|
3411
|
+
body: JSON.stringify(opts)
|
|
3412
|
+
}),
|
|
3413
|
+
getJob: (jobId) => request(`/jobs/${jobId}`),
|
|
3414
|
+
cancelJob: (jobId) => request(`/jobs/${jobId}`, {
|
|
3415
|
+
method: "DELETE"
|
|
3416
|
+
}),
|
|
3417
|
+
health: () => request("/healthz"),
|
|
3418
|
+
isConfigured: () => Boolean(process.env.TRANSCRIPTION_SERVER)
|
|
3419
|
+
};
|
|
3420
|
+
|
|
3421
|
+
// src/exulu/transcription/transcript-text.ts
|
|
3422
|
+
var renderTranscript = (segments, speakers) => {
|
|
3423
|
+
if (!segments || segments.length === 0) return "";
|
|
3424
|
+
const blocks = [];
|
|
3425
|
+
for (const seg of segments) {
|
|
3426
|
+
const text = (seg.text ?? "").trim();
|
|
3427
|
+
if (!text) continue;
|
|
3428
|
+
const label = speakers[seg.speaker] ?? seg.speaker ?? "unknown";
|
|
3429
|
+
const last = blocks[blocks.length - 1];
|
|
3430
|
+
if (last && last.speaker === label) {
|
|
3431
|
+
last.text = `${last.text} ${text}`.trim();
|
|
3432
|
+
} else {
|
|
3433
|
+
blocks.push({ speaker: label, text });
|
|
3434
|
+
}
|
|
3435
|
+
}
|
|
3436
|
+
return blocks.map((b) => `${b.speaker}: ${b.text}`).join("\n");
|
|
3437
|
+
};
|
|
3438
|
+
|
|
3439
|
+
// src/exulu/transcription/service.ts
|
|
3440
|
+
var TABLE = "transcription_jobs";
|
|
3441
|
+
var log = (msg) => console.log(`[EXULU-TRANSCRIPTION] ${msg}`);
|
|
3442
|
+
var parseJsonField = (v) => {
|
|
3443
|
+
if (v == null) return null;
|
|
3444
|
+
if (typeof v === "string") {
|
|
3445
|
+
try {
|
|
3446
|
+
return JSON.parse(v);
|
|
3447
|
+
} catch {
|
|
3448
|
+
return null;
|
|
3449
|
+
}
|
|
3450
|
+
}
|
|
3451
|
+
return v;
|
|
3452
|
+
};
|
|
3453
|
+
var presignAudio = async (s3Key) => {
|
|
3454
|
+
const app = exuluApp.get();
|
|
3455
|
+
const config = app._config ?? app.config;
|
|
3456
|
+
const configuredBucket = config?.fileUploads?.s3Bucket;
|
|
3457
|
+
if (!configuredBucket) {
|
|
3458
|
+
throw new Error("File uploads are not configured (s3Bucket missing).");
|
|
3459
|
+
}
|
|
3460
|
+
const firstSlash = s3Key.indexOf("/");
|
|
3461
|
+
const bucket = firstSlash > 0 ? s3Key.slice(0, firstSlash) : configuredBucket;
|
|
3462
|
+
const objectKey = firstSlash > 0 ? s3Key.slice(firstSlash + 1) : s3Key;
|
|
3463
|
+
return getPresignedUrl(bucket, objectKey, config);
|
|
3464
|
+
};
|
|
3465
|
+
var transcriptionService = {
|
|
3466
|
+
/**
|
|
3467
|
+
* Create a transcription job row and dispatch it to the whisper server.
|
|
3468
|
+
* Throws TranscriptionServerUnavailable if the feature is off.
|
|
3469
|
+
*/
|
|
3470
|
+
async startJob(input) {
|
|
3471
|
+
if (!transcriptionClient.isConfigured()) {
|
|
3472
|
+
throw new TranscriptionServerUnavailable(
|
|
3473
|
+
"TRANSCRIPTION_SERVER is not set. Start a whisper server with `npx @exulu/backend exulu-start-whisper` and point TRANSCRIPTION_SERVER at it."
|
|
3474
|
+
);
|
|
3475
|
+
}
|
|
3476
|
+
const { db } = await postgresClient();
|
|
3477
|
+
const now = /* @__PURE__ */ new Date();
|
|
3478
|
+
const [inserted] = await db(TABLE).insert({
|
|
3479
|
+
audio_s3key: input.s3Key,
|
|
3480
|
+
title: input.title ?? input.filename,
|
|
3481
|
+
status: "queued",
|
|
3482
|
+
project_id: input.project_id ?? null,
|
|
3483
|
+
target_rights_mode: input.target_rights_mode ?? "private",
|
|
3484
|
+
target_rbac_users: input.target_rbac_users ? JSON.stringify(input.target_rbac_users) : null,
|
|
3485
|
+
target_rbac_roles: input.target_rbac_roles ? JSON.stringify(input.target_rbac_roles) : null,
|
|
3486
|
+
rights_mode: "private",
|
|
3487
|
+
created_by: input.userId,
|
|
3488
|
+
createdAt: now,
|
|
3489
|
+
updatedAt: now
|
|
3490
|
+
}).returning("*");
|
|
3491
|
+
const row = this._rowFromDb(inserted);
|
|
3492
|
+
try {
|
|
3493
|
+
const audioUrl = await presignAudio(input.s3Key);
|
|
3494
|
+
const submitted = await transcriptionClient.submitJob({
|
|
3495
|
+
audio_url: audioUrl,
|
|
3496
|
+
language: input.language ?? void 0,
|
|
3497
|
+
num_speakers: input.num_speakers ?? void 0,
|
|
3498
|
+
hotwords: input.hotwords
|
|
3499
|
+
});
|
|
3500
|
+
const [updated] = await db(TABLE).where({ id: row.id }).update({
|
|
3501
|
+
whisper_job_id: submitted.job_id,
|
|
3502
|
+
status: "transcribing",
|
|
3503
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3504
|
+
}).returning("*");
|
|
3505
|
+
return this._rowFromDb(updated);
|
|
3506
|
+
} catch (err) {
|
|
3507
|
+
const [failed] = await db(TABLE).where({ id: row.id }).update({
|
|
3508
|
+
status: "failed",
|
|
3509
|
+
error: err.message,
|
|
3510
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3511
|
+
}).returning("*");
|
|
3512
|
+
log(`Failed to dispatch job ${row.id}: ${err.message}`);
|
|
3513
|
+
return this._rowFromDb(failed);
|
|
3514
|
+
}
|
|
3515
|
+
},
|
|
3516
|
+
/**
|
|
3517
|
+
* Reconcile every transcribing row against the whisper server. Called from
|
|
3518
|
+
* the polling loop on a fixed interval. Caps how many rows we touch per
|
|
3519
|
+
* tick so a backlog can't starve the event loop.
|
|
3520
|
+
*/
|
|
3521
|
+
async pollOnce(maxPerTick = 50) {
|
|
3522
|
+
if (!transcriptionClient.isConfigured()) return;
|
|
3523
|
+
const { db } = await postgresClient();
|
|
3524
|
+
const rows = await db(TABLE).where({ status: "transcribing" }).whereNotNull("whisper_job_id").limit(maxPerTick);
|
|
3525
|
+
for (const dbRow of rows) {
|
|
3526
|
+
const row = this._rowFromDb(dbRow);
|
|
3527
|
+
if (!row.whisper_job_id) continue;
|
|
3528
|
+
try {
|
|
3529
|
+
const job = await transcriptionClient.getJob(row.whisper_job_id);
|
|
3530
|
+
await this._applyJobUpdate(row, job);
|
|
3531
|
+
} catch (err) {
|
|
3532
|
+
const code = err.code;
|
|
3533
|
+
if (code === "JOB_NOT_FOUND") {
|
|
3534
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3535
|
+
status: "failed",
|
|
3536
|
+
error: "lost on server restart",
|
|
3537
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3538
|
+
});
|
|
3539
|
+
} else if (err instanceof TranscriptionServerUnavailable) {
|
|
3540
|
+
log(`Whisper server unreachable while polling ${row.id}; will retry`);
|
|
3541
|
+
} else {
|
|
3542
|
+
log(`Error polling job ${row.id}: ${err.message}`);
|
|
3543
|
+
}
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
},
|
|
3547
|
+
async _applyJobUpdate(row, job) {
|
|
3548
|
+
const { db } = await postgresClient();
|
|
3549
|
+
if ((job.status === "queued" || job.status === "running") && job.duration_seconds != null && row.duration_seconds !== job.duration_seconds) {
|
|
3550
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3551
|
+
duration_seconds: job.duration_seconds,
|
|
3552
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3553
|
+
});
|
|
3554
|
+
}
|
|
3555
|
+
if (job.status === "running" || job.status === "queued") return;
|
|
3556
|
+
if (job.status === "completed") {
|
|
3557
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3558
|
+
status: "awaiting_review",
|
|
3559
|
+
raw_segments: JSON.stringify(job.segments ?? []),
|
|
3560
|
+
language: job.language ?? null,
|
|
3561
|
+
duration_seconds: job.duration_seconds ?? null,
|
|
3562
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3563
|
+
});
|
|
3564
|
+
return;
|
|
3565
|
+
}
|
|
3566
|
+
if (job.status === "failed") {
|
|
3567
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3568
|
+
status: "failed",
|
|
3569
|
+
error: job.error ?? "transcription failed",
|
|
3570
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3571
|
+
});
|
|
3572
|
+
return;
|
|
3573
|
+
}
|
|
3574
|
+
if (job.status === "cancelled") {
|
|
3575
|
+
await db(TABLE).where({ id: row.id }).update({
|
|
3576
|
+
status: "cancelled",
|
|
3577
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3578
|
+
});
|
|
3579
|
+
}
|
|
3580
|
+
},
|
|
3581
|
+
async cancelJob(id) {
|
|
3582
|
+
const { db } = await postgresClient();
|
|
3583
|
+
const dbRow = await db(TABLE).where({ id }).first();
|
|
3584
|
+
if (!dbRow) throw new Error(`transcription_job ${id} not found`);
|
|
3585
|
+
const row = this._rowFromDb(dbRow);
|
|
3586
|
+
if (row.whisper_job_id && transcriptionClient.isConfigured()) {
|
|
3587
|
+
try {
|
|
3588
|
+
await transcriptionClient.cancelJob(row.whisper_job_id);
|
|
3589
|
+
} catch (err) {
|
|
3590
|
+
const code = err.code;
|
|
3591
|
+
if (code !== "JOB_NOT_FOUND") {
|
|
3592
|
+
log(`Best-effort cancel of whisper job failed: ${err.message}`);
|
|
3593
|
+
}
|
|
3594
|
+
}
|
|
3595
|
+
}
|
|
3596
|
+
const [updated] = await db(TABLE).where({ id }).update({ status: "cancelled", updatedAt: /* @__PURE__ */ new Date() }).returning("*");
|
|
3597
|
+
return this._rowFromDb(updated);
|
|
3598
|
+
},
|
|
3599
|
+
/**
|
|
3600
|
+
* User clicked Save in the review panel.
|
|
3601
|
+
*
|
|
3602
|
+
* - From 'awaiting_review': render the speaker-labeled transcript, create a
|
|
3603
|
+
* new ExuluContext item, apply RBAC + optional project linkage, mark the
|
|
3604
|
+
* job saved.
|
|
3605
|
+
* - From 'saved': re-render the transcript with the (possibly updated)
|
|
3606
|
+
* speaker map and upsert the existing context item by id. Used by the
|
|
3607
|
+
* Completed-section re-edit flow.
|
|
3608
|
+
*/
|
|
3609
|
+
async finalize(id, input) {
|
|
3610
|
+
const { db } = await postgresClient();
|
|
3611
|
+
const dbRow = await db(TABLE).where({ id }).first();
|
|
3612
|
+
if (!dbRow) throw new Error(`transcription_job ${id} not found`);
|
|
3613
|
+
const row = this._rowFromDb(dbRow);
|
|
3614
|
+
if (row.status !== "awaiting_review" && row.status !== "saved") {
|
|
3615
|
+
throw new Error(
|
|
3616
|
+
`transcription_job ${id} is in status '${row.status}'; can only finalize from 'awaiting_review' or 'saved'`
|
|
3617
|
+
);
|
|
3618
|
+
}
|
|
3619
|
+
if (!row.raw_segments) {
|
|
3620
|
+
throw new Error(`transcription_job ${id} has no raw_segments to finalize`);
|
|
3621
|
+
}
|
|
3622
|
+
const app = exuluApp.get();
|
|
3623
|
+
const context = app.context("transcriptions");
|
|
3624
|
+
if (!context) {
|
|
3625
|
+
throw new Error("Built-in transcriptions context not registered");
|
|
3626
|
+
}
|
|
3627
|
+
const config = app._config ?? app.config;
|
|
3628
|
+
const transcriptText = renderTranscript(row.raw_segments, input.speakers);
|
|
3629
|
+
const rightsMode = input.target_rights_mode ?? row.target_rights_mode ?? "private";
|
|
3630
|
+
const isReSave = row.status === "saved" && !!row.saved_item_id;
|
|
3631
|
+
const itemInput = {
|
|
3632
|
+
// Carrying the id on re-save makes context.createItem upsert in place.
|
|
3633
|
+
...isReSave && row.saved_item_id ? { id: row.saved_item_id } : {},
|
|
3634
|
+
name: input.title ?? row.title ?? "Transcript",
|
|
3635
|
+
transcript_text: transcriptText,
|
|
3636
|
+
audio_s3key: row.audio_s3key,
|
|
3637
|
+
language: row.language ?? void 0,
|
|
3638
|
+
duration_seconds: row.duration_seconds ?? void 0,
|
|
3639
|
+
speakers: input.speakers,
|
|
3640
|
+
raw_segments: row.raw_segments,
|
|
3641
|
+
rights_mode: rightsMode,
|
|
3642
|
+
created_by: row.created_by
|
|
3643
|
+
};
|
|
3644
|
+
let item;
|
|
3645
|
+
try {
|
|
3646
|
+
const result = await context.createItem(
|
|
3647
|
+
itemInput,
|
|
3648
|
+
config,
|
|
3649
|
+
row.created_by,
|
|
3650
|
+
void 0,
|
|
3651
|
+
isReSave
|
|
3652
|
+
// upsert when re-saving
|
|
3653
|
+
);
|
|
3654
|
+
item = result.item;
|
|
3655
|
+
} catch (err) {
|
|
3656
|
+
await db(TABLE).where({ id }).update({
|
|
3657
|
+
speakers: JSON.stringify(input.speakers),
|
|
3658
|
+
error: `Failed to save: ${err.message}`,
|
|
3659
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3660
|
+
});
|
|
3661
|
+
throw err;
|
|
3662
|
+
}
|
|
3663
|
+
const itemId = item.id ?? row.saved_item_id ?? "";
|
|
3664
|
+
const users = input.target_rbac_users ?? row.target_rbac_users ?? [];
|
|
3665
|
+
const roles = input.target_rbac_roles ?? row.target_rbac_roles ?? [];
|
|
3666
|
+
if ((users.length || roles.length) && rightsMode !== "private") {
|
|
3667
|
+
try {
|
|
3668
|
+
await handleRBACUpdate(
|
|
3669
|
+
db,
|
|
3670
|
+
"transcriptions",
|
|
3671
|
+
itemId,
|
|
3672
|
+
{ users, roles },
|
|
3673
|
+
[]
|
|
3674
|
+
);
|
|
3675
|
+
} catch (err) {
|
|
3676
|
+
log(`RBAC update failed for item ${itemId}: ${err.message}`);
|
|
3677
|
+
}
|
|
3678
|
+
}
|
|
3679
|
+
const projectId = input.project_id ?? row.project_id ?? null;
|
|
3680
|
+
let projectWarning = null;
|
|
3681
|
+
if (projectId && !isReSave) {
|
|
3682
|
+
try {
|
|
3683
|
+
const project = await db("projects").where({ id: projectId }).first();
|
|
3684
|
+
if (!project) {
|
|
3685
|
+
projectWarning = `project ${projectId} not found`;
|
|
3686
|
+
} else {
|
|
3687
|
+
const existing = parseJsonField(project.project_items) ?? [];
|
|
3688
|
+
const globalId = `transcriptions/${itemId}`;
|
|
3689
|
+
if (!existing.includes(globalId)) {
|
|
3690
|
+
existing.push(globalId);
|
|
3691
|
+
await db("projects").where({ id: projectId }).update({
|
|
3692
|
+
project_items: JSON.stringify(existing),
|
|
3693
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3694
|
+
});
|
|
3695
|
+
}
|
|
3696
|
+
}
|
|
3697
|
+
} catch (err) {
|
|
3698
|
+
projectWarning = err.message;
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
3701
|
+
const [updated] = await db(TABLE).where({ id }).update({
|
|
3702
|
+
status: "saved",
|
|
3703
|
+
saved_item_id: itemId,
|
|
3704
|
+
title: input.title ?? row.title ?? null,
|
|
3705
|
+
speakers: JSON.stringify(input.speakers),
|
|
3706
|
+
error: projectWarning ? `Saved, but could not attach to project: ${projectWarning}` : null,
|
|
3707
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3708
|
+
}).returning("*");
|
|
3709
|
+
return { item, row: this._rowFromDb(updated) };
|
|
3710
|
+
},
|
|
3711
|
+
_rowFromDb(dbRow) {
|
|
3712
|
+
return {
|
|
3713
|
+
...dbRow,
|
|
3714
|
+
raw_segments: parseJsonField(dbRow.raw_segments),
|
|
3715
|
+
speakers: parseJsonField(dbRow.speakers),
|
|
3716
|
+
target_rbac_users: parseJsonField(
|
|
3717
|
+
dbRow.target_rbac_users
|
|
3718
|
+
),
|
|
3719
|
+
target_rbac_roles: parseJsonField(
|
|
3720
|
+
dbRow.target_rbac_roles
|
|
3721
|
+
)
|
|
3722
|
+
};
|
|
3723
|
+
}
|
|
3724
|
+
};
|
|
3725
|
+
|
|
3726
|
+
// src/graphql/schemas/index.ts
|
|
3361
3727
|
function createExuluContextsTypeDefs(table) {
|
|
3362
3728
|
const enumDefs = table.fields.filter((field) => field.type === "enum" && field.enumValues).map((field) => {
|
|
3363
3729
|
if (!field.enumValues) {
|
|
@@ -3775,6 +4141,39 @@ type PageInfo {
|
|
|
3775
4141
|
mutationDefs += `
|
|
3776
4142
|
deleteJob(queue: QueueEnum!, id: String!): JobActionReturnPayload
|
|
3777
4143
|
`;
|
|
4144
|
+
mutationDefs += `
|
|
4145
|
+
transcriptionJobStart(input: TranscriptionJobStartInput!): transcription_job
|
|
4146
|
+
transcriptionJobFinalize(id: ID!, input: TranscriptionJobFinalizeInput!): TranscriptionJobFinalizeResult
|
|
4147
|
+
transcriptionJobCancel(id: ID!): transcription_job
|
|
4148
|
+
`;
|
|
4149
|
+
modelDefs += `
|
|
4150
|
+
input TranscriptionJobStartInput {
|
|
4151
|
+
audio_s3key: String!
|
|
4152
|
+
filename: String!
|
|
4153
|
+
title: String
|
|
4154
|
+
language: String
|
|
4155
|
+
num_speakers: Int
|
|
4156
|
+
hotwords: [String!]
|
|
4157
|
+
project_id: ID
|
|
4158
|
+
target_rights_mode: String
|
|
4159
|
+
target_rbac_users: [RBACUserInput!]
|
|
4160
|
+
target_rbac_roles: [RBACRoleInput!]
|
|
4161
|
+
}
|
|
4162
|
+
|
|
4163
|
+
input TranscriptionJobFinalizeInput {
|
|
4164
|
+
title: String
|
|
4165
|
+
speakers: JSON!
|
|
4166
|
+
project_id: ID
|
|
4167
|
+
target_rights_mode: String
|
|
4168
|
+
target_rbac_users: [RBACUserInput!]
|
|
4169
|
+
target_rbac_roles: [RBACRoleInput!]
|
|
4170
|
+
}
|
|
4171
|
+
|
|
4172
|
+
type TranscriptionJobFinalizeResult {
|
|
4173
|
+
job: transcription_job!
|
|
4174
|
+
item_id: ID!
|
|
4175
|
+
}
|
|
4176
|
+
`;
|
|
3778
4177
|
typeDefs += `
|
|
3779
4178
|
tools(search: String, category: String, limit: Int, page: Int): ToolPaginationResult
|
|
3780
4179
|
toolCategories: [String!]!
|
|
@@ -3865,7 +4264,7 @@ type LiteLLMModel {
|
|
|
3865
4264
|
};
|
|
3866
4265
|
};
|
|
3867
4266
|
resolvers.Query["litellmCatalog"] = async () => {
|
|
3868
|
-
const { fetchLiteLLMCatalog } = await import("./catalog-
|
|
4267
|
+
const { fetchLiteLLMCatalog } = await import("./catalog-BWE6SLE2.js");
|
|
3869
4268
|
return fetchLiteLLMCatalog();
|
|
3870
4269
|
};
|
|
3871
4270
|
resolvers.Query["workflowSchedule"] = async (_, args, context, info) => {
|
|
@@ -4340,6 +4739,54 @@ type LiteLLMModel {
|
|
|
4340
4739
|
await config2.queue.remove(args.id);
|
|
4341
4740
|
return { success: true };
|
|
4342
4741
|
};
|
|
4742
|
+
const assertOwnsTranscriptionJob = async (id, context) => {
|
|
4743
|
+
const { db, user } = context;
|
|
4744
|
+
if (!user) throw new Error("Authentication required");
|
|
4745
|
+
if (user.super_admin === true) return;
|
|
4746
|
+
const row = await db.from("transcription_jobs").select(["created_by", "rights_mode"]).where({ id }).first();
|
|
4747
|
+
if (!row) throw new Error(`transcription_job ${id} not found`);
|
|
4748
|
+
if (row.rights_mode === "public") return;
|
|
4749
|
+
if (row.created_by === user.id) return;
|
|
4750
|
+
throw new Error("Not authorized to act on this transcription job");
|
|
4751
|
+
};
|
|
4752
|
+
resolvers.Mutation["transcriptionJobStart"] = async (_, args, context) => {
|
|
4753
|
+
const { user } = context;
|
|
4754
|
+
if (!user) throw new Error("Authentication required");
|
|
4755
|
+
if (!transcriptionClient.isConfigured()) {
|
|
4756
|
+
throw new Error(
|
|
4757
|
+
"TRANSCRIPTION_DISABLED: TRANSCRIPTION_SERVER not set on this server. Ask the operator to start a whisper server with `npx @exulu/backend exulu-start-whisper`."
|
|
4758
|
+
);
|
|
4759
|
+
}
|
|
4760
|
+
return transcriptionService.startJob({
|
|
4761
|
+
userId: user.id,
|
|
4762
|
+
s3Key: args.input.audio_s3key,
|
|
4763
|
+
filename: args.input.filename,
|
|
4764
|
+
title: args.input.title,
|
|
4765
|
+
language: args.input.language ?? void 0,
|
|
4766
|
+
num_speakers: args.input.num_speakers ?? void 0,
|
|
4767
|
+
hotwords: args.input.hotwords ?? void 0,
|
|
4768
|
+
project_id: args.input.project_id ?? null,
|
|
4769
|
+
target_rights_mode: args.input.target_rights_mode ?? null,
|
|
4770
|
+
target_rbac_users: args.input.target_rbac_users ?? void 0,
|
|
4771
|
+
target_rbac_roles: args.input.target_rbac_roles ?? void 0
|
|
4772
|
+
});
|
|
4773
|
+
};
|
|
4774
|
+
resolvers.Mutation["transcriptionJobFinalize"] = async (_, args, context) => {
|
|
4775
|
+
await assertOwnsTranscriptionJob(args.id, context);
|
|
4776
|
+
const { item, row } = await transcriptionService.finalize(args.id, {
|
|
4777
|
+
title: args.input.title,
|
|
4778
|
+
speakers: args.input.speakers,
|
|
4779
|
+
project_id: args.input.project_id ?? null,
|
|
4780
|
+
target_rights_mode: args.input.target_rights_mode ?? null,
|
|
4781
|
+
target_rbac_users: args.input.target_rbac_users ?? void 0,
|
|
4782
|
+
target_rbac_roles: args.input.target_rbac_roles ?? void 0
|
|
4783
|
+
});
|
|
4784
|
+
return { job: row, item_id: item.id };
|
|
4785
|
+
};
|
|
4786
|
+
resolvers.Mutation["transcriptionJobCancel"] = async (_, args, context) => {
|
|
4787
|
+
await assertOwnsTranscriptionJob(args.id, context);
|
|
4788
|
+
return transcriptionService.cancelJob(args.id);
|
|
4789
|
+
};
|
|
4343
4790
|
resolvers.Query["evals"] = async (_, args, context, info) => {
|
|
4344
4791
|
const requestedFields = getRequestedFields(info);
|
|
4345
4792
|
return {
|
|
@@ -5197,7 +5644,8 @@ import {
|
|
|
5197
5644
|
generateText as generateText2,
|
|
5198
5645
|
streamText,
|
|
5199
5646
|
validateUIMessages,
|
|
5200
|
-
stepCountIs
|
|
5647
|
+
stepCountIs,
|
|
5648
|
+
hasToolCall
|
|
5201
5649
|
} from "ai";
|
|
5202
5650
|
|
|
5203
5651
|
// src/utils/generate-slug.ts
|
|
@@ -5528,6 +5976,9 @@ var ExuluProvider = class {
|
|
|
5528
5976
|
If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume
|
|
5529
5977
|
they are talking with the current date in mind as a reference.`;
|
|
5530
5978
|
let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
|
|
5979
|
+
if (user?.personal_system_prompt?.trim()) {
|
|
5980
|
+
system += "\n\nUser preferences:\n" + user.personal_system_prompt.trim();
|
|
5981
|
+
}
|
|
5531
5982
|
system += "\n\n" + genericContext;
|
|
5532
5983
|
if (memoryContext) {
|
|
5533
5984
|
system += "\n\n" + memoryContext;
|
|
@@ -5627,7 +6078,10 @@ When a tool execution is not approved by the user, do not retry it unless explic
|
|
|
5627
6078
|
agent,
|
|
5628
6079
|
memoryItems
|
|
5629
6080
|
),
|
|
5630
|
-
|
|
6081
|
+
// Stop after the image_generation tool fires — the widget IS the
|
|
6082
|
+
// assistant's response, no follow-up text turn is wanted (same
|
|
6083
|
+
// reasoning as question_ask: the UI artifact is the message).
|
|
6084
|
+
stopWhen: [stepCountIs(maxStepCount || 5), hasToolCall("image_generation")]
|
|
5631
6085
|
// make configurable
|
|
5632
6086
|
});
|
|
5633
6087
|
console.log("[EXULU] Output: " + JSON.stringify(output, null, 2));
|
|
@@ -5708,7 +6162,7 @@ When a tool execution is not approved by the user, do not retry it unless explic
|
|
|
5708
6162
|
agent,
|
|
5709
6163
|
memoryItems
|
|
5710
6164
|
),
|
|
5711
|
-
stopWhen: [stepCountIs(maxStepCount || 5)]
|
|
6165
|
+
stopWhen: [stepCountIs(maxStepCount || 5), hasToolCall("image_generation")]
|
|
5712
6166
|
});
|
|
5713
6167
|
if (statistics) {
|
|
5714
6168
|
await Promise.all([
|
|
@@ -5933,6 +6387,9 @@ ${extractedText}
|
|
|
5933
6387
|
messages = await this.processFilePartsInMessages(messages);
|
|
5934
6388
|
const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
|
|
5935
6389
|
let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
|
|
6390
|
+
if (user?.personal_system_prompt?.trim()) {
|
|
6391
|
+
system += "\n\nUser preferences:\n" + user.personal_system_prompt.trim();
|
|
6392
|
+
}
|
|
5936
6393
|
system += "\n\n" + genericContext;
|
|
5937
6394
|
const includesContextSearchTool = currentTools?.some(
|
|
5938
6395
|
(tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search") || tool2.type === "context"
|
|
@@ -6089,7 +6546,7 @@ When a tool execution is not approved by the user, do not retry it unless explic
|
|
|
6089
6546
|
},
|
|
6090
6547
|
// provide more loops for skills because they are more complex to execute
|
|
6091
6548
|
// todo allow configuring this per skill
|
|
6092
|
-
stopWhen: [stepCountIs(maxStepCount || currentSkills?.length ? 10 : 5)]
|
|
6549
|
+
stopWhen: [stepCountIs(maxStepCount || currentSkills?.length ? 10 : 5), hasToolCall("image_generation")]
|
|
6093
6550
|
});
|
|
6094
6551
|
return {
|
|
6095
6552
|
stream: result,
|
|
@@ -6296,7 +6753,266 @@ async function synthesizeSpeech(args) {
|
|
|
6296
6753
|
return Buffer.from(arrayBuf);
|
|
6297
6754
|
}
|
|
6298
6755
|
|
|
6756
|
+
// src/exulu/image-generation.ts
|
|
6757
|
+
var ImageGenerationError = class extends Error {
|
|
6758
|
+
constructor(upstreamStatus, message) {
|
|
6759
|
+
super(message);
|
|
6760
|
+
this.upstreamStatus = upstreamStatus;
|
|
6761
|
+
this.name = "ImageGenerationError";
|
|
6762
|
+
}
|
|
6763
|
+
};
|
|
6764
|
+
var resolveProxyConfig = () => {
|
|
6765
|
+
const host = process.env.LITELLM_HOST ?? "127.0.0.1";
|
|
6766
|
+
const port = process.env.LITELLM_PORT ?? "4000";
|
|
6767
|
+
const masterKey = process.env.LITELLM_MASTER_KEY;
|
|
6768
|
+
if (!masterKey) throw new Error("LITELLM_MASTER_KEY is not set");
|
|
6769
|
+
return { host, port, masterKey };
|
|
6770
|
+
};
|
|
6771
|
+
var normalizeDataEntries = async (data) => {
|
|
6772
|
+
const out = [];
|
|
6773
|
+
for (const entry of data) {
|
|
6774
|
+
let buffer;
|
|
6775
|
+
let contentType = "image/png";
|
|
6776
|
+
let extension = "png";
|
|
6777
|
+
if (entry.b64_json) {
|
|
6778
|
+
buffer = Buffer.from(entry.b64_json, "base64");
|
|
6779
|
+
} else if (entry.url) {
|
|
6780
|
+
const upstream = await fetch(entry.url);
|
|
6781
|
+
if (!upstream.ok) {
|
|
6782
|
+
throw new ImageGenerationError(
|
|
6783
|
+
upstream.status,
|
|
6784
|
+
`Failed to download generated image from ${entry.url}: ${upstream.status} ${upstream.statusText}`
|
|
6785
|
+
);
|
|
6786
|
+
}
|
|
6787
|
+
const ct = upstream.headers.get("content-type");
|
|
6788
|
+
if (ct && ct.startsWith("image/")) {
|
|
6789
|
+
contentType = ct;
|
|
6790
|
+
const inferred = ct.split("/")[1]?.split(";")[0]?.trim();
|
|
6791
|
+
if (inferred) extension = inferred === "jpeg" ? "jpg" : inferred;
|
|
6792
|
+
}
|
|
6793
|
+
buffer = Buffer.from(await upstream.arrayBuffer());
|
|
6794
|
+
} else {
|
|
6795
|
+
throw new ImageGenerationError(
|
|
6796
|
+
0,
|
|
6797
|
+
"LiteLLM image response entry contained neither b64_json nor url."
|
|
6798
|
+
);
|
|
6799
|
+
}
|
|
6800
|
+
out.push({ buffer, contentType, extension, revisedPrompt: entry.revised_prompt });
|
|
6801
|
+
}
|
|
6802
|
+
return out;
|
|
6803
|
+
};
|
|
6804
|
+
async function generateImage(args) {
|
|
6805
|
+
if (!args.model) throw new Error("model is required");
|
|
6806
|
+
if (!args.prompt) throw new Error("prompt is required");
|
|
6807
|
+
const cfg = resolveProxyConfig();
|
|
6808
|
+
const body = {
|
|
6809
|
+
model: args.model,
|
|
6810
|
+
prompt: args.prompt
|
|
6811
|
+
};
|
|
6812
|
+
if (args.size) body.size = args.size;
|
|
6813
|
+
if (args.quality) body.quality = args.quality;
|
|
6814
|
+
if (args.n) body.n = args.n;
|
|
6815
|
+
const res = await fetch(`http://${cfg.host}:${cfg.port}/v1/images/generations`, {
|
|
6816
|
+
method: "POST",
|
|
6817
|
+
headers: {
|
|
6818
|
+
Authorization: `Bearer ${cfg.masterKey}`,
|
|
6819
|
+
"Content-Type": "application/json"
|
|
6820
|
+
},
|
|
6821
|
+
body: JSON.stringify(body),
|
|
6822
|
+
signal: args.signal
|
|
6823
|
+
});
|
|
6824
|
+
if (!res.ok) {
|
|
6825
|
+
const text = await res.text().catch(() => "");
|
|
6826
|
+
throw new ImageGenerationError(
|
|
6827
|
+
res.status,
|
|
6828
|
+
`LiteLLM image generation failed (status ${res.status}): ${text}`.trim()
|
|
6829
|
+
);
|
|
6830
|
+
}
|
|
6831
|
+
const json = await res.json();
|
|
6832
|
+
if (!json?.data || json.data.length === 0) {
|
|
6833
|
+
throw new ImageGenerationError(
|
|
6834
|
+
res.status,
|
|
6835
|
+
"LiteLLM returned no image data in the response."
|
|
6836
|
+
);
|
|
6837
|
+
}
|
|
6838
|
+
return normalizeDataEntries(json.data);
|
|
6839
|
+
}
|
|
6840
|
+
async function editImage(args) {
|
|
6841
|
+
if (!args.model) throw new Error("model is required");
|
|
6842
|
+
if (!args.prompt) throw new Error("prompt is required");
|
|
6843
|
+
if (!args.references || args.references.length === 0) {
|
|
6844
|
+
throw new Error("at least one reference image is required");
|
|
6845
|
+
}
|
|
6846
|
+
const cfg = resolveProxyConfig();
|
|
6847
|
+
const form = new FormData();
|
|
6848
|
+
form.append("model", args.model);
|
|
6849
|
+
form.append("prompt", args.prompt);
|
|
6850
|
+
if (args.n != null) form.append("n", String(args.n));
|
|
6851
|
+
if (args.size) form.append("size", args.size);
|
|
6852
|
+
if (args.quality) form.append("quality", args.quality);
|
|
6853
|
+
form.append("response_format", "b64_json");
|
|
6854
|
+
for (const ref of args.references) {
|
|
6855
|
+
form.append(
|
|
6856
|
+
"image",
|
|
6857
|
+
new Blob([ref.buffer], { type: ref.mimetype ?? "image/png" }),
|
|
6858
|
+
ref.filename
|
|
6859
|
+
);
|
|
6860
|
+
}
|
|
6861
|
+
if (args.mask) {
|
|
6862
|
+
form.append(
|
|
6863
|
+
"mask",
|
|
6864
|
+
new Blob([args.mask.buffer], { type: args.mask.mimetype ?? "image/png" }),
|
|
6865
|
+
args.mask.filename
|
|
6866
|
+
);
|
|
6867
|
+
}
|
|
6868
|
+
const res = await fetch(`http://${cfg.host}:${cfg.port}/v1/images/edits`, {
|
|
6869
|
+
method: "POST",
|
|
6870
|
+
headers: { Authorization: `Bearer ${cfg.masterKey}` },
|
|
6871
|
+
body: form,
|
|
6872
|
+
signal: args.signal
|
|
6873
|
+
});
|
|
6874
|
+
if (!res.ok) {
|
|
6875
|
+
const text = await res.text().catch(() => "");
|
|
6876
|
+
throw new ImageGenerationError(
|
|
6877
|
+
res.status,
|
|
6878
|
+
`LiteLLM image edit failed (status ${res.status}): ${text}`.trim()
|
|
6879
|
+
);
|
|
6880
|
+
}
|
|
6881
|
+
const json = await res.json();
|
|
6882
|
+
if (!json?.data || json.data.length === 0) {
|
|
6883
|
+
throw new ImageGenerationError(
|
|
6884
|
+
res.status,
|
|
6885
|
+
"LiteLLM returned no image data in the edit response."
|
|
6886
|
+
);
|
|
6887
|
+
}
|
|
6888
|
+
return normalizeDataEntries(json.data);
|
|
6889
|
+
}
|
|
6890
|
+
|
|
6891
|
+
// src/exulu/litellm/parse-image-models.ts
|
|
6892
|
+
import { existsSync as existsSync2, readFileSync } from "fs";
|
|
6893
|
+
var stripComment = (line) => {
|
|
6894
|
+
const idx = line.indexOf("#");
|
|
6895
|
+
return idx >= 0 ? line.slice(0, idx) : line;
|
|
6896
|
+
};
|
|
6897
|
+
var parseInlineArray = (raw) => {
|
|
6898
|
+
const m = raw.trim().match(/^\[(.*)\]$/);
|
|
6899
|
+
if (!m) return void 0;
|
|
6900
|
+
const inner = m[1] ?? "";
|
|
6901
|
+
if (!inner.trim()) return [];
|
|
6902
|
+
return inner.split(",").map((s) => s.trim().replace(/^["']|["']$/g, "")).filter((s) => s.length > 0);
|
|
6903
|
+
};
|
|
6904
|
+
var parseBool = (raw) => {
|
|
6905
|
+
const v = raw.trim().toLowerCase();
|
|
6906
|
+
if (v === "true" || v === "yes") return true;
|
|
6907
|
+
if (v === "false" || v === "no") return false;
|
|
6908
|
+
return void 0;
|
|
6909
|
+
};
|
|
6910
|
+
var parseInt10 = (raw) => {
|
|
6911
|
+
const n = Number(raw.trim());
|
|
6912
|
+
return Number.isInteger(n) ? n : void 0;
|
|
6913
|
+
};
|
|
6914
|
+
var parseImageGenerationModels = (configPath) => {
|
|
6915
|
+
if (!existsSync2(configPath)) return [];
|
|
6916
|
+
const text = readFileSync(configPath, "utf8");
|
|
6917
|
+
const lines = text.split("\n");
|
|
6918
|
+
const entries = [];
|
|
6919
|
+
let current;
|
|
6920
|
+
for (const rawLine of lines) {
|
|
6921
|
+
const noComment = stripComment(rawLine);
|
|
6922
|
+
if (!noComment.trim()) continue;
|
|
6923
|
+
const indent = (rawLine.match(/^\s*/)?.[0] ?? "").length;
|
|
6924
|
+
const modelNameMatch = noComment.match(
|
|
6925
|
+
/^\s*-\s*model_name\s*:\s*["']?([^"'\s#]+)["']?\s*$/
|
|
6926
|
+
);
|
|
6927
|
+
if (modelNameMatch) {
|
|
6928
|
+
if (current) entries.push(current);
|
|
6929
|
+
current = { model_name: modelNameMatch[1], indent };
|
|
6930
|
+
continue;
|
|
6931
|
+
}
|
|
6932
|
+
if (!current) continue;
|
|
6933
|
+
if (indent <= current.indent && !/^\s*-\s/.test(rawLine)) {
|
|
6934
|
+
entries.push(current);
|
|
6935
|
+
current = void 0;
|
|
6936
|
+
continue;
|
|
6937
|
+
}
|
|
6938
|
+
const kvMatch = noComment.match(/^\s*(\w+)\s*:\s*(.+?)\s*$/);
|
|
6939
|
+
if (!kvMatch) continue;
|
|
6940
|
+
const key2 = kvMatch[1] ?? "";
|
|
6941
|
+
const rawValue = kvMatch[2] ?? "";
|
|
6942
|
+
switch (key2) {
|
|
6943
|
+
case "type": {
|
|
6944
|
+
current.type = rawValue.replace(/^["']|["']$/g, "").trim();
|
|
6945
|
+
break;
|
|
6946
|
+
}
|
|
6947
|
+
case "sizes": {
|
|
6948
|
+
current.sizes = parseInlineArray(rawValue);
|
|
6949
|
+
break;
|
|
6950
|
+
}
|
|
6951
|
+
case "qualities": {
|
|
6952
|
+
current.qualities = parseInlineArray(rawValue);
|
|
6953
|
+
break;
|
|
6954
|
+
}
|
|
6955
|
+
case "supports_edit": {
|
|
6956
|
+
current.supports_edit = parseBool(rawValue);
|
|
6957
|
+
break;
|
|
6958
|
+
}
|
|
6959
|
+
case "max_n": {
|
|
6960
|
+
current.max_n = parseInt10(rawValue);
|
|
6961
|
+
break;
|
|
6962
|
+
}
|
|
6963
|
+
}
|
|
6964
|
+
}
|
|
6965
|
+
if (current) entries.push(current);
|
|
6966
|
+
const imageEntries = entries.filter((e) => e.type === "image_generation");
|
|
6967
|
+
const errors = [];
|
|
6968
|
+
const validated = [];
|
|
6969
|
+
for (const e of imageEntries) {
|
|
6970
|
+
const modelErrs = [];
|
|
6971
|
+
if (!Array.isArray(e.sizes) || e.sizes.length === 0) {
|
|
6972
|
+
modelErrs.push(
|
|
6973
|
+
'model_info.sizes must be a non-empty inline YAML array of strings, e.g. `sizes: ["1024x1024", "1024x1536"]`'
|
|
6974
|
+
);
|
|
6975
|
+
}
|
|
6976
|
+
if (!Array.isArray(e.qualities) || e.qualities.length === 0) {
|
|
6977
|
+
modelErrs.push(
|
|
6978
|
+
'model_info.qualities must be a non-empty inline YAML array of strings, e.g. `qualities: ["auto", "high"]`'
|
|
6979
|
+
);
|
|
6980
|
+
}
|
|
6981
|
+
if (typeof e.supports_edit !== "boolean") {
|
|
6982
|
+
modelErrs.push(
|
|
6983
|
+
"model_info.supports_edit must be a boolean (true/false)"
|
|
6984
|
+
);
|
|
6985
|
+
}
|
|
6986
|
+
if (typeof e.max_n !== "number" || !Number.isInteger(e.max_n) || e.max_n < 1) {
|
|
6987
|
+
modelErrs.push("model_info.max_n must be an integer \u2265 1");
|
|
6988
|
+
}
|
|
6989
|
+
if (modelErrs.length > 0) {
|
|
6990
|
+
errors.push(
|
|
6991
|
+
` - "${e.model_name}":
|
|
6992
|
+
- ${modelErrs.join("\n - ")}`
|
|
6993
|
+
);
|
|
6994
|
+
continue;
|
|
6995
|
+
}
|
|
6996
|
+
validated.push({
|
|
6997
|
+
model_name: e.model_name,
|
|
6998
|
+
sizes: e.sizes,
|
|
6999
|
+
qualities: e.qualities,
|
|
7000
|
+
supports_edit: e.supports_edit,
|
|
7001
|
+
max_n: e.max_n
|
|
7002
|
+
});
|
|
7003
|
+
}
|
|
7004
|
+
if (errors.length > 0) {
|
|
7005
|
+
throw new Error(
|
|
7006
|
+
`[EXULU] config.litellm.yaml has image-generation models with missing or invalid model_info keys. Fix and restart Exulu:
|
|
7007
|
+
${errors.join("\n")}
|
|
7008
|
+
See docs/superpowers/specs/2026-05-31-in-chat-image-generation-design.md for the required schema.`
|
|
7009
|
+
);
|
|
7010
|
+
}
|
|
7011
|
+
return validated;
|
|
7012
|
+
};
|
|
7013
|
+
|
|
6299
7014
|
// src/exulu/routes.ts
|
|
7015
|
+
import { resolve as resolvePath } from "path";
|
|
6300
7016
|
import multer from "multer";
|
|
6301
7017
|
|
|
6302
7018
|
// src/utils/check-provider-rate-limit.ts
|
|
@@ -6993,7 +7709,8 @@ var {
|
|
|
6993
7709
|
contextPresetsSchema,
|
|
6994
7710
|
embedderSettingsSchema,
|
|
6995
7711
|
promptFavoritesSchema,
|
|
6996
|
-
statisticsSchema
|
|
7712
|
+
statisticsSchema,
|
|
7713
|
+
transcriptionJobsSchema
|
|
6997
7714
|
} = coreSchemas.get();
|
|
6998
7715
|
var createExpressRoutes = async (app, providers, tools, contexts, config, evals, tracer, queues2, rerankers) => {
|
|
6999
7716
|
let corsOptions = {
|
|
@@ -7051,7 +7768,8 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
|
|
|
7051
7768
|
variablesSchema(),
|
|
7052
7769
|
workflowTemplatesSchema(),
|
|
7053
7770
|
statisticsSchema(),
|
|
7054
|
-
rbacSchema()
|
|
7771
|
+
rbacSchema(),
|
|
7772
|
+
transcriptionJobsSchema()
|
|
7055
7773
|
],
|
|
7056
7774
|
contexts ?? [],
|
|
7057
7775
|
providers,
|
|
@@ -7874,17 +8592,498 @@ ${customInstructions}` : agent.instructions;
|
|
|
7874
8592
|
}
|
|
7875
8593
|
}
|
|
7876
8594
|
);
|
|
7877
|
-
|
|
7878
|
-
if (!isLiteLLMEnabled())
|
|
7879
|
-
|
|
7880
|
-
|
|
7881
|
-
|
|
7882
|
-
return;
|
|
8595
|
+
const imageModelsByName = (() => {
|
|
8596
|
+
if (!isLiteLLMEnabled() || !config?.fileUploads) return /* @__PURE__ */ new Map();
|
|
8597
|
+
try {
|
|
8598
|
+
const configPath = process.env.LITELLM_CONFIG_PATH ?? resolvePath(getPackageRoot(), "./config.litellm.yaml");
|
|
8599
|
+
const models2 = parseImageGenerationModels(configPath);
|
|
8600
|
+
return new Map(models2.map((m) => [m.model_name, m]));
|
|
8601
|
+
} catch (err) {
|
|
8602
|
+
console.error(
|
|
8603
|
+
"[EXULU] Skipping /images/* routes due to config.litellm.yaml error:",
|
|
8604
|
+
err.message
|
|
8605
|
+
);
|
|
8606
|
+
return /* @__PURE__ */ new Map();
|
|
7883
8607
|
}
|
|
7884
|
-
|
|
7885
|
-
|
|
7886
|
-
|
|
7887
|
-
|
|
8608
|
+
})();
|
|
8609
|
+
const imageRoutesEnabled = isLiteLLMEnabled() && !!config?.fileUploads?.s3region && !!config?.fileUploads?.s3key && !!config?.fileUploads?.s3secret && !!config?.fileUploads?.s3Bucket && imageModelsByName.size > 0;
|
|
8610
|
+
const respond503ImagesNotEnabled = (res) => {
|
|
8611
|
+
res.status(503).json({
|
|
8612
|
+
detail: "Image generation is not enabled on this deployment. Requires EXULU_USE_LITELLM=true, S3 fileUploads configuration, and at least one model in config.litellm.yaml with model_info.type=image_generation."
|
|
8613
|
+
});
|
|
8614
|
+
};
|
|
8615
|
+
const loadAuthedSession = async (req, res, sessionId, rights) => {
|
|
8616
|
+
const authResult = await requestValidators.authenticate(req);
|
|
8617
|
+
if (!authResult.user?.id) {
|
|
8618
|
+
res.status(authResult.code || 401).json({ detail: authResult.message });
|
|
8619
|
+
return null;
|
|
8620
|
+
}
|
|
8621
|
+
const { db } = await postgresClient();
|
|
8622
|
+
const session = await db.from("agent_sessions").where({ id: sessionId }).first();
|
|
8623
|
+
if (!session) {
|
|
8624
|
+
res.status(404).json({ detail: `Session ${sessionId} not found.` });
|
|
8625
|
+
return null;
|
|
8626
|
+
}
|
|
8627
|
+
const sessionRbac = await RBACResolver(
|
|
8628
|
+
db,
|
|
8629
|
+
"agent_sessions",
|
|
8630
|
+
session.id,
|
|
8631
|
+
session.rights_mode || "private"
|
|
8632
|
+
);
|
|
8633
|
+
const allowed = await checkRecordAccess(
|
|
8634
|
+
{ ...session, RBAC: sessionRbac },
|
|
8635
|
+
rights,
|
|
8636
|
+
authResult.user
|
|
8637
|
+
);
|
|
8638
|
+
if (!allowed) {
|
|
8639
|
+
res.status(403).json({ detail: `You don't have ${rights} access to this session.` });
|
|
8640
|
+
return null;
|
|
8641
|
+
}
|
|
8642
|
+
return { user: authResult.user, session, db };
|
|
8643
|
+
};
|
|
8644
|
+
const loadStyle = async (db, styleId, user, res) => {
|
|
8645
|
+
if (!styleId) return { markdown: null, id: null };
|
|
8646
|
+
const row = await db.from("platform_configurations").where({ id: styleId }).first();
|
|
8647
|
+
if (!row) {
|
|
8648
|
+
res.status(404).json({ detail: `Style ${styleId} not found.` });
|
|
8649
|
+
return "error";
|
|
8650
|
+
}
|
|
8651
|
+
const rbac = await RBACResolver(
|
|
8652
|
+
db,
|
|
8653
|
+
"platform_configurations",
|
|
8654
|
+
row.id,
|
|
8655
|
+
row.rights_mode || "private"
|
|
8656
|
+
);
|
|
8657
|
+
const allowed = await checkRecordAccess(
|
|
8658
|
+
{ ...row, RBAC: rbac },
|
|
8659
|
+
"read",
|
|
8660
|
+
user
|
|
8661
|
+
);
|
|
8662
|
+
if (!allowed) {
|
|
8663
|
+
res.status(403).json({ detail: "You don't have access to that style." });
|
|
8664
|
+
return "error";
|
|
8665
|
+
}
|
|
8666
|
+
const value = typeof row.config_value === "string" ? (() => {
|
|
8667
|
+
try {
|
|
8668
|
+
return JSON.parse(row.config_value);
|
|
8669
|
+
} catch {
|
|
8670
|
+
return null;
|
|
8671
|
+
}
|
|
8672
|
+
})() : row.config_value;
|
|
8673
|
+
return { markdown: value?.markdown ?? null, id: row.id };
|
|
8674
|
+
};
|
|
8675
|
+
const validateGenerationParams = (body, res) => {
|
|
8676
|
+
const { model: modelName, prompt, n, size, quality } = body || {};
|
|
8677
|
+
const model = typeof modelName === "string" ? imageModelsByName.get(modelName) : void 0;
|
|
8678
|
+
if (!model) {
|
|
8679
|
+
res.status(400).json({
|
|
8680
|
+
detail: `Unknown image-generation model "${modelName}". Available: ${[...imageModelsByName.keys()].join(", ")}.`
|
|
8681
|
+
});
|
|
8682
|
+
return null;
|
|
8683
|
+
}
|
|
8684
|
+
if (typeof prompt !== "string" || prompt.trim().length === 0) {
|
|
8685
|
+
res.status(400).json({ detail: "prompt must be a non-empty string." });
|
|
8686
|
+
return null;
|
|
8687
|
+
}
|
|
8688
|
+
const requestedN = typeof n === "number" ? n : 1;
|
|
8689
|
+
if (!Number.isInteger(requestedN) || requestedN < 1 || requestedN > model.max_n) {
|
|
8690
|
+
res.status(400).json({
|
|
8691
|
+
detail: `n must be an integer between 1 and ${model.max_n} for model ${model.model_name}.`
|
|
8692
|
+
});
|
|
8693
|
+
return null;
|
|
8694
|
+
}
|
|
8695
|
+
if (size && !model.sizes.includes(size)) {
|
|
8696
|
+
res.status(400).json({
|
|
8697
|
+
detail: `size "${size}" is not supported by ${model.model_name}. Allowed: ${model.sizes.join(", ")}.`
|
|
8698
|
+
});
|
|
8699
|
+
return null;
|
|
8700
|
+
}
|
|
8701
|
+
if (quality && !model.qualities.includes(quality)) {
|
|
8702
|
+
res.status(400).json({
|
|
8703
|
+
detail: `quality "${quality}" is not supported by ${model.model_name}. Allowed: ${model.qualities.join(", ")}.`
|
|
8704
|
+
});
|
|
8705
|
+
return null;
|
|
8706
|
+
}
|
|
8707
|
+
return { model, prompt, n: requestedN, size, quality };
|
|
8708
|
+
};
|
|
8709
|
+
const uploadGeneratedImages = async (images, sessionId, toolCallId, userId) => {
|
|
8710
|
+
if (!config?.fileUploads) {
|
|
8711
|
+
throw new Error("File uploads not configured.");
|
|
8712
|
+
}
|
|
8713
|
+
const keys = [];
|
|
8714
|
+
const revisedPrompts = [];
|
|
8715
|
+
for (const img of images) {
|
|
8716
|
+
const filename = `${randomUUID2()}.${img.extension}`;
|
|
8717
|
+
const key2 = `sessions/${sessionId}/images/${toolCallId}/${filename}`;
|
|
8718
|
+
const fullKey = await uploadFile(
|
|
8719
|
+
img.buffer,
|
|
8720
|
+
key2,
|
|
8721
|
+
config,
|
|
8722
|
+
{ contentType: img.contentType },
|
|
8723
|
+
userId
|
|
8724
|
+
);
|
|
8725
|
+
keys.push(fullKey);
|
|
8726
|
+
revisedPrompts.push(img.revisedPrompt ?? null);
|
|
8727
|
+
}
|
|
8728
|
+
const presignedUrls = await Promise.all(
|
|
8729
|
+
keys.map((fullKey) => {
|
|
8730
|
+
const slash = fullKey.indexOf("/");
|
|
8731
|
+
const bucket = slash > 0 ? fullKey.slice(0, slash) : config.fileUploads.s3Bucket;
|
|
8732
|
+
const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
|
|
8733
|
+
return getPresignedUrl(bucket, objectKey, config);
|
|
8734
|
+
})
|
|
8735
|
+
);
|
|
8736
|
+
return { keys, revisedPrompts, presignedUrls };
|
|
8737
|
+
};
|
|
8738
|
+
app.post("/images/generate", async (req, res) => {
|
|
8739
|
+
if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
|
|
8740
|
+
const { sessionId, toolCallId, styleId } = req.body || {};
|
|
8741
|
+
if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
|
|
8742
|
+
res.status(400).json({ detail: "sessionId and toolCallId are required." });
|
|
8743
|
+
return;
|
|
8744
|
+
}
|
|
8745
|
+
const authed = await loadAuthedSession(req, res, sessionId, "write");
|
|
8746
|
+
if (!authed) return;
|
|
8747
|
+
const params = validateGenerationParams(req.body, res);
|
|
8748
|
+
if (!params) return;
|
|
8749
|
+
const style = await loadStyle(authed.db, styleId, authed.user, res);
|
|
8750
|
+
if (style === "error") return;
|
|
8751
|
+
const finalPrompt = style.markdown ? `${params.prompt}
|
|
8752
|
+
|
|
8753
|
+
${style.markdown}` : params.prompt;
|
|
8754
|
+
try {
|
|
8755
|
+
await Promise.race([
|
|
8756
|
+
waitForLiteLLMReady(),
|
|
8757
|
+
new Promise(
|
|
8758
|
+
(_, reject) => setTimeout(() => reject(new Error("LiteLLM not ready")), 5e3)
|
|
8759
|
+
)
|
|
8760
|
+
]);
|
|
8761
|
+
} catch {
|
|
8762
|
+
res.status(503).json({ detail: "Image service is not ready. Try again shortly." });
|
|
8763
|
+
return;
|
|
8764
|
+
}
|
|
8765
|
+
const abortController = new AbortController();
|
|
8766
|
+
req.on("close", () => abortController.abort());
|
|
8767
|
+
try {
|
|
8768
|
+
const images = await generateImage({
|
|
8769
|
+
model: params.model.model_name,
|
|
8770
|
+
prompt: finalPrompt,
|
|
8771
|
+
n: params.n,
|
|
8772
|
+
size: params.size,
|
|
8773
|
+
quality: params.quality,
|
|
8774
|
+
signal: abortController.signal
|
|
8775
|
+
});
|
|
8776
|
+
const { keys, revisedPrompts, presignedUrls } = await uploadGeneratedImages(
|
|
8777
|
+
images,
|
|
8778
|
+
sessionId,
|
|
8779
|
+
toolCallId,
|
|
8780
|
+
authed.user.id
|
|
8781
|
+
);
|
|
8782
|
+
const [row] = await authed.db("image_generations").insert({
|
|
8783
|
+
session_id: sessionId,
|
|
8784
|
+
tool_call_id: toolCallId,
|
|
8785
|
+
user_id: authed.user.id,
|
|
8786
|
+
operation: "generate",
|
|
8787
|
+
model: params.model.model_name,
|
|
8788
|
+
prompt: params.prompt,
|
|
8789
|
+
applied_style_id: style.id,
|
|
8790
|
+
applied_style_markdown: style.markdown,
|
|
8791
|
+
size: params.size,
|
|
8792
|
+
quality: params.quality,
|
|
8793
|
+
n: params.n,
|
|
8794
|
+
image_keys: JSON.stringify(keys),
|
|
8795
|
+
revised_prompts: JSON.stringify(revisedPrompts),
|
|
8796
|
+
selected: false
|
|
8797
|
+
}).returning("*");
|
|
8798
|
+
res.status(200).json({
|
|
8799
|
+
generationId: row.id,
|
|
8800
|
+
images: keys.map((key2, i) => ({
|
|
8801
|
+
key: key2,
|
|
8802
|
+
presignedUrl: presignedUrls[i],
|
|
8803
|
+
revisedPrompt: revisedPrompts[i]
|
|
8804
|
+
}))
|
|
8805
|
+
});
|
|
8806
|
+
} catch (err) {
|
|
8807
|
+
if (abortController.signal.aborted) return;
|
|
8808
|
+
if (err instanceof ImageGenerationError) {
|
|
8809
|
+
const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
|
|
8810
|
+
res.status(code).json({ detail: err.message });
|
|
8811
|
+
return;
|
|
8812
|
+
}
|
|
8813
|
+
console.error("[EXULU] /images/generate failed", err);
|
|
8814
|
+
res.status(500).json({
|
|
8815
|
+
detail: err instanceof Error ? err.message : "Image generation failed."
|
|
8816
|
+
});
|
|
8817
|
+
}
|
|
8818
|
+
});
|
|
8819
|
+
app.post("/images/edit", async (req, res) => {
|
|
8820
|
+
if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
|
|
8821
|
+
const { sessionId, toolCallId, styleId, referenceImageKeys, maskKey } = req.body || {};
|
|
8822
|
+
if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
|
|
8823
|
+
res.status(400).json({ detail: "sessionId and toolCallId are required." });
|
|
8824
|
+
return;
|
|
8825
|
+
}
|
|
8826
|
+
if (!Array.isArray(referenceImageKeys) || referenceImageKeys.length === 0) {
|
|
8827
|
+
res.status(400).json({ detail: "referenceImageKeys must be a non-empty array." });
|
|
8828
|
+
return;
|
|
8829
|
+
}
|
|
8830
|
+
const authed = await loadAuthedSession(req, res, sessionId, "write");
|
|
8831
|
+
if (!authed) return;
|
|
8832
|
+
const params = validateGenerationParams(req.body, res);
|
|
8833
|
+
if (!params) return;
|
|
8834
|
+
if (!params.model.supports_edit) {
|
|
8835
|
+
res.status(400).json({
|
|
8836
|
+
detail: `Model ${params.model.model_name} does not support image editing.`
|
|
8837
|
+
});
|
|
8838
|
+
return;
|
|
8839
|
+
}
|
|
8840
|
+
const userPrefix = `user_${authed.user.id}/`;
|
|
8841
|
+
const sessionPrefix = `sessions/${sessionId}/`;
|
|
8842
|
+
const ownsKey = (k) => k.includes(userPrefix) || k.includes(sessionPrefix);
|
|
8843
|
+
if (!referenceImageKeys.every((k) => typeof k === "string" && ownsKey(k))) {
|
|
8844
|
+
res.status(403).json({ detail: "One or more reference image keys are not accessible." });
|
|
8845
|
+
return;
|
|
8846
|
+
}
|
|
8847
|
+
if (maskKey && (typeof maskKey !== "string" || !ownsKey(maskKey))) {
|
|
8848
|
+
res.status(403).json({ detail: "Mask image is not accessible." });
|
|
8849
|
+
return;
|
|
8850
|
+
}
|
|
8851
|
+
const style = await loadStyle(authed.db, styleId, authed.user, res);
|
|
8852
|
+
if (style === "error") return;
|
|
8853
|
+
const finalPrompt = style.markdown ? `${params.prompt}
|
|
8854
|
+
|
|
8855
|
+
${style.markdown}` : params.prompt;
|
|
8856
|
+
try {
|
|
8857
|
+
await Promise.race([
|
|
8858
|
+
waitForLiteLLMReady(),
|
|
8859
|
+
new Promise(
|
|
8860
|
+
(_, reject) => setTimeout(() => reject(new Error("LiteLLM not ready")), 5e3)
|
|
8861
|
+
)
|
|
8862
|
+
]);
|
|
8863
|
+
} catch {
|
|
8864
|
+
res.status(503).json({ detail: "Image service is not ready. Try again shortly." });
|
|
8865
|
+
return;
|
|
8866
|
+
}
|
|
8867
|
+
const fetchRef = async (fullKey) => {
|
|
8868
|
+
const slash = fullKey.indexOf("/");
|
|
8869
|
+
const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
|
|
8870
|
+
const buf = await getS3ObjectBytes(objectKey, config);
|
|
8871
|
+
const filename = fullKey.split("/").pop() ?? "image.png";
|
|
8872
|
+
const ext = filename.split(".").pop()?.toLowerCase() ?? "png";
|
|
8873
|
+
const mimetype = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : `image/${ext}`;
|
|
8874
|
+
return { buffer: buf, filename, mimetype };
|
|
8875
|
+
};
|
|
8876
|
+
const abortController = new AbortController();
|
|
8877
|
+
req.on("close", () => abortController.abort());
|
|
8878
|
+
try {
|
|
8879
|
+
const references = await Promise.all(referenceImageKeys.map(fetchRef));
|
|
8880
|
+
const mask = maskKey ? await fetchRef(maskKey) : void 0;
|
|
8881
|
+
const images = await editImage({
|
|
8882
|
+
model: params.model.model_name,
|
|
8883
|
+
prompt: finalPrompt,
|
|
8884
|
+
references,
|
|
8885
|
+
mask,
|
|
8886
|
+
n: params.n,
|
|
8887
|
+
size: params.size,
|
|
8888
|
+
quality: params.quality,
|
|
8889
|
+
signal: abortController.signal
|
|
8890
|
+
});
|
|
8891
|
+
const { keys, revisedPrompts, presignedUrls } = await uploadGeneratedImages(
|
|
8892
|
+
images,
|
|
8893
|
+
sessionId,
|
|
8894
|
+
toolCallId,
|
|
8895
|
+
authed.user.id
|
|
8896
|
+
);
|
|
8897
|
+
const [row] = await authed.db("image_generations").insert({
|
|
8898
|
+
session_id: sessionId,
|
|
8899
|
+
tool_call_id: toolCallId,
|
|
8900
|
+
user_id: authed.user.id,
|
|
8901
|
+
operation: "edit",
|
|
8902
|
+
model: params.model.model_name,
|
|
8903
|
+
prompt: params.prompt,
|
|
8904
|
+
applied_style_id: style.id,
|
|
8905
|
+
applied_style_markdown: style.markdown,
|
|
8906
|
+
size: params.size,
|
|
8907
|
+
quality: params.quality,
|
|
8908
|
+
n: params.n,
|
|
8909
|
+
reference_image_keys: JSON.stringify(referenceImageKeys),
|
|
8910
|
+
mask_image_key: maskKey,
|
|
8911
|
+
image_keys: JSON.stringify(keys),
|
|
8912
|
+
revised_prompts: JSON.stringify(revisedPrompts),
|
|
8913
|
+
selected: false
|
|
8914
|
+
}).returning("*");
|
|
8915
|
+
res.status(200).json({
|
|
8916
|
+
generationId: row.id,
|
|
8917
|
+
images: keys.map((key2, i) => ({
|
|
8918
|
+
key: key2,
|
|
8919
|
+
presignedUrl: presignedUrls[i],
|
|
8920
|
+
revisedPrompt: revisedPrompts[i]
|
|
8921
|
+
}))
|
|
8922
|
+
});
|
|
8923
|
+
} catch (err) {
|
|
8924
|
+
if (abortController.signal.aborted) return;
|
|
8925
|
+
if (err instanceof ImageGenerationError) {
|
|
8926
|
+
const code = err.upstreamStatus >= 500 ? 502 : err.upstreamStatus;
|
|
8927
|
+
res.status(code).json({ detail: err.message });
|
|
8928
|
+
return;
|
|
8929
|
+
}
|
|
8930
|
+
console.error("[EXULU] /images/edit failed", err);
|
|
8931
|
+
res.status(500).json({
|
|
8932
|
+
detail: err instanceof Error ? err.message : "Image edit failed."
|
|
8933
|
+
});
|
|
8934
|
+
}
|
|
8935
|
+
});
|
|
8936
|
+
app.post("/images/select", async (req, res) => {
|
|
8937
|
+
if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
|
|
8938
|
+
const { sessionId, toolCallId, selections } = req.body || {};
|
|
8939
|
+
if (typeof sessionId !== "string" || typeof toolCallId !== "string") {
|
|
8940
|
+
res.status(400).json({ detail: "sessionId and toolCallId are required." });
|
|
8941
|
+
return;
|
|
8942
|
+
}
|
|
8943
|
+
if (!Array.isArray(selections) || selections.length === 0) {
|
|
8944
|
+
res.status(400).json({ detail: "selections must be a non-empty array." });
|
|
8945
|
+
return;
|
|
8946
|
+
}
|
|
8947
|
+
const authed = await loadAuthedSession(req, res, sessionId, "write");
|
|
8948
|
+
if (!authed) return;
|
|
8949
|
+
const rows = await authed.db("image_generations").where({ session_id: sessionId, tool_call_id: toolCallId }).select("*");
|
|
8950
|
+
const rowsById = new Map(rows.map((r) => [r.id, r]));
|
|
8951
|
+
const selectedDetails = [];
|
|
8952
|
+
const rowsToMarkSelected = /* @__PURE__ */ new Set();
|
|
8953
|
+
for (const sel of selections) {
|
|
8954
|
+
if (typeof sel?.generationId !== "string" || typeof sel?.imageKey !== "string") {
|
|
8955
|
+
res.status(400).json({ detail: "Each selection needs generationId + imageKey." });
|
|
8956
|
+
return;
|
|
8957
|
+
}
|
|
8958
|
+
const row = rowsById.get(sel.generationId);
|
|
8959
|
+
if (!row) {
|
|
8960
|
+
res.status(404).json({ detail: `Generation ${sel.generationId} not found in this tool call.` });
|
|
8961
|
+
return;
|
|
8962
|
+
}
|
|
8963
|
+
const keys = Array.isArray(row.image_keys) ? row.image_keys : (() => {
|
|
8964
|
+
try {
|
|
8965
|
+
return JSON.parse(row.image_keys);
|
|
8966
|
+
} catch {
|
|
8967
|
+
return [];
|
|
8968
|
+
}
|
|
8969
|
+
})();
|
|
8970
|
+
if (!keys.includes(sel.imageKey)) {
|
|
8971
|
+
res.status(400).json({ detail: `imageKey not part of generation ${sel.generationId}.` });
|
|
8972
|
+
return;
|
|
8973
|
+
}
|
|
8974
|
+
const slash = sel.imageKey.indexOf("/");
|
|
8975
|
+
const bucket = slash > 0 ? sel.imageKey.slice(0, slash) : config.fileUploads.s3Bucket;
|
|
8976
|
+
const objectKey = slash > 0 ? sel.imageKey.slice(slash + 1) : sel.imageKey;
|
|
8977
|
+
const presignedUrl = await getPresignedUrl(bucket, objectKey, config);
|
|
8978
|
+
let styleName = null;
|
|
8979
|
+
if (row.applied_style_id) {
|
|
8980
|
+
const styleRow = await authed.db("platform_configurations").where({ id: row.applied_style_id }).first();
|
|
8981
|
+
const parsed = styleRow?.config_value && typeof styleRow.config_value === "string" ? (() => {
|
|
8982
|
+
try {
|
|
8983
|
+
return JSON.parse(styleRow.config_value);
|
|
8984
|
+
} catch {
|
|
8985
|
+
return null;
|
|
8986
|
+
}
|
|
8987
|
+
})() : styleRow?.config_value;
|
|
8988
|
+
styleName = parsed?.name ?? null;
|
|
8989
|
+
}
|
|
8990
|
+
selectedDetails.push({
|
|
8991
|
+
key: sel.imageKey,
|
|
8992
|
+
presignedUrl,
|
|
8993
|
+
prompt: row.prompt,
|
|
8994
|
+
model: row.model,
|
|
8995
|
+
styleName
|
|
8996
|
+
});
|
|
8997
|
+
rowsToMarkSelected.add(row.id);
|
|
8998
|
+
}
|
|
8999
|
+
await authed.db("image_generations").whereIn("id", [...rowsToMarkSelected]).update({ selected: true });
|
|
9000
|
+
const lines = selectedDetails.map(
|
|
9001
|
+
(d) => `- ${d.presignedUrl} (prompt: "${d.prompt}", model: ${d.model}${d.styleName ? `, style: ${d.styleName}` : ""})`
|
|
9002
|
+
);
|
|
9003
|
+
const messageText = "The user generated and selected the following image(s) in this chat:\n" + lines.join("\n");
|
|
9004
|
+
const messageId = randomUUID2();
|
|
9005
|
+
const uiMessage = {
|
|
9006
|
+
id: messageId,
|
|
9007
|
+
role: "system",
|
|
9008
|
+
parts: [{ type: "text", text: messageText }]
|
|
9009
|
+
};
|
|
9010
|
+
await authed.db("agent_messages").insert({
|
|
9011
|
+
content: JSON.stringify(uiMessage),
|
|
9012
|
+
message_id: messageId,
|
|
9013
|
+
session: sessionId,
|
|
9014
|
+
user: authed.user.id
|
|
9015
|
+
});
|
|
9016
|
+
res.status(200).json({ ok: true, systemMessage: uiMessage, selectedImages: selectedDetails });
|
|
9017
|
+
});
|
|
9018
|
+
app.get("/images/history", async (req, res) => {
|
|
9019
|
+
if (!imageRoutesEnabled) return respond503ImagesNotEnabled(res);
|
|
9020
|
+
const sessionId = typeof req.query.sessionId === "string" ? req.query.sessionId : "";
|
|
9021
|
+
const toolCallId = typeof req.query.toolCallId === "string" ? req.query.toolCallId : "";
|
|
9022
|
+
if (!sessionId || !toolCallId) {
|
|
9023
|
+
res.status(400).json({ detail: "sessionId and toolCallId query params are required." });
|
|
9024
|
+
return;
|
|
9025
|
+
}
|
|
9026
|
+
const authed = await loadAuthedSession(req, res, sessionId, "read");
|
|
9027
|
+
if (!authed) return;
|
|
9028
|
+
const rows = await authed.db("image_generations").where({ session_id: sessionId, tool_call_id: toolCallId }).orderBy("createdAt", "asc").select("*");
|
|
9029
|
+
const parseList = (v) => {
|
|
9030
|
+
if (!v) return [];
|
|
9031
|
+
if (Array.isArray(v)) return v;
|
|
9032
|
+
try {
|
|
9033
|
+
return JSON.parse(v);
|
|
9034
|
+
} catch {
|
|
9035
|
+
return [];
|
|
9036
|
+
}
|
|
9037
|
+
};
|
|
9038
|
+
const sign = async (fullKey) => {
|
|
9039
|
+
const slash = fullKey.indexOf("/");
|
|
9040
|
+
const bucket = slash > 0 ? fullKey.slice(0, slash) : config.fileUploads.s3Bucket;
|
|
9041
|
+
const objectKey = slash > 0 ? fullKey.slice(slash + 1) : fullKey;
|
|
9042
|
+
return getPresignedUrl(bucket, objectKey, config);
|
|
9043
|
+
};
|
|
9044
|
+
const history = await Promise.all(rows.map(async (r) => {
|
|
9045
|
+
const keys = parseList(r.image_keys);
|
|
9046
|
+
const refs = parseList(r.reference_image_keys);
|
|
9047
|
+
const revisedPrompts = parseList(r.revised_prompts);
|
|
9048
|
+
const [imageUrls, referenceUrls] = await Promise.all([
|
|
9049
|
+
Promise.all(keys.map(sign)),
|
|
9050
|
+
Promise.all(refs.map(sign))
|
|
9051
|
+
]);
|
|
9052
|
+
return {
|
|
9053
|
+
generationId: r.id,
|
|
9054
|
+
createdAt: r.createdAt,
|
|
9055
|
+
operation: r.operation,
|
|
9056
|
+
model: r.model,
|
|
9057
|
+
prompt: r.prompt,
|
|
9058
|
+
appliedStyleId: r.applied_style_id ?? null,
|
|
9059
|
+
appliedStyleMarkdown: r.applied_style_markdown ?? null,
|
|
9060
|
+
size: r.size ?? null,
|
|
9061
|
+
quality: r.quality ?? null,
|
|
9062
|
+
n: r.n ?? 1,
|
|
9063
|
+
selected: !!r.selected,
|
|
9064
|
+
error: r.error ?? null,
|
|
9065
|
+
maskImageKey: r.mask_image_key ?? null,
|
|
9066
|
+
images: keys.map((key2, i) => ({
|
|
9067
|
+
key: key2,
|
|
9068
|
+
presignedUrl: imageUrls[i],
|
|
9069
|
+
revisedPrompt: revisedPrompts[i] ?? null
|
|
9070
|
+
})),
|
|
9071
|
+
references: refs.map((key2, i) => ({ key: key2, presignedUrl: referenceUrls[i] }))
|
|
9072
|
+
};
|
|
9073
|
+
}));
|
|
9074
|
+
res.status(200).json({ history });
|
|
9075
|
+
});
|
|
9076
|
+
app.use("/litellm/:project", async (req, res) => {
|
|
9077
|
+
if (!isLiteLLMEnabled()) {
|
|
9078
|
+
res.status(503).json({
|
|
9079
|
+
detail: "LiteLLM is not enabled on this deployment. Set EXULU_USE_LITELLM=true."
|
|
9080
|
+
});
|
|
9081
|
+
return;
|
|
9082
|
+
}
|
|
9083
|
+
const masterKey = process.env.LITELLM_MASTER_KEY;
|
|
9084
|
+
if (!masterKey) {
|
|
9085
|
+
res.status(503).json({ detail: "LITELLM_MASTER_KEY is not configured." });
|
|
9086
|
+
return;
|
|
7888
9087
|
}
|
|
7889
9088
|
const authenticationResult = await requestValidators.authenticate(req);
|
|
7890
9089
|
if (!authenticationResult.user?.id) {
|
|
@@ -11004,216 +12203,382 @@ var emailTool = new ExuluTool({
|
|
|
11004
12203
|
}
|
|
11005
12204
|
});
|
|
11006
12205
|
|
|
11007
|
-
// src/
|
|
11008
|
-
|
|
11009
|
-
|
|
11010
|
-
|
|
11011
|
-
|
|
11012
|
-
|
|
12206
|
+
// src/templates/tools/transcribe.ts
|
|
12207
|
+
import { randomUUID as randomUUID5 } from "crypto";
|
|
12208
|
+
import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
|
|
12209
|
+
import { dirname, join as join2 } from "path";
|
|
12210
|
+
import { z as z10 } from "zod";
|
|
12211
|
+
var SANDBOX_ROOT = "/tmp/exulu-sessions";
|
|
12212
|
+
var parseSandboxPath = (input) => {
|
|
12213
|
+
const stripped = input.startsWith("file://") ? input.slice("file://".length) : input;
|
|
12214
|
+
if (!stripped.startsWith(`${SANDBOX_ROOT}/`)) return null;
|
|
12215
|
+
const tail = stripped.slice(SANDBOX_ROOT.length + 1);
|
|
12216
|
+
const slash = tail.indexOf("/");
|
|
12217
|
+
if (slash < 1) return null;
|
|
12218
|
+
const sessionId = tail.slice(0, slash);
|
|
12219
|
+
const relPath = tail.slice(slash + 1);
|
|
12220
|
+
if (!relPath) return null;
|
|
12221
|
+
return { sessionId, relPath };
|
|
11013
12222
|
};
|
|
11014
|
-
|
|
11015
|
-
|
|
11016
|
-
|
|
11017
|
-
|
|
11018
|
-
|
|
11019
|
-
|
|
11020
|
-
|
|
11021
|
-
|
|
11022
|
-
|
|
11023
|
-
|
|
11024
|
-
|
|
11025
|
-
|
|
11026
|
-
|
|
11027
|
-
|
|
11028
|
-
|
|
11029
|
-
|
|
11030
|
-
|
|
11031
|
-
|
|
11032
|
-
|
|
11033
|
-
|
|
11034
|
-
|
|
11035
|
-
|
|
11036
|
-
|
|
12223
|
+
var audioMimetypeFromExtension = (filename) => {
|
|
12224
|
+
const ext = filename.split(".").pop()?.toLowerCase();
|
|
12225
|
+
switch (ext) {
|
|
12226
|
+
case "mp3":
|
|
12227
|
+
return "audio/mpeg";
|
|
12228
|
+
case "m4a":
|
|
12229
|
+
case "mp4":
|
|
12230
|
+
return "audio/mp4";
|
|
12231
|
+
case "wav":
|
|
12232
|
+
return "audio/wav";
|
|
12233
|
+
case "ogg":
|
|
12234
|
+
case "oga":
|
|
12235
|
+
return "audio/ogg";
|
|
12236
|
+
case "flac":
|
|
12237
|
+
return "audio/flac";
|
|
12238
|
+
case "webm":
|
|
12239
|
+
return "audio/webm";
|
|
12240
|
+
case "aac":
|
|
12241
|
+
return "audio/aac";
|
|
12242
|
+
case "mpga":
|
|
12243
|
+
case "mpeg":
|
|
12244
|
+
return "audio/mpeg";
|
|
12245
|
+
default:
|
|
12246
|
+
throw new Error(
|
|
12247
|
+
`Unable to infer an audio mimetype from filename "${filename}". Supported extensions: mp3, m4a, mp4, wav, ogg, flac, webm, aac, mpga.`
|
|
12248
|
+
);
|
|
12249
|
+
}
|
|
12250
|
+
};
|
|
12251
|
+
var transcribeTool = new ExuluTool({
|
|
12252
|
+
id: "transcribe_audio",
|
|
12253
|
+
name: "Transcribe Audio",
|
|
12254
|
+
description: "Transcribe an audio file (mp3, wav, m4a, etc.) from a URL to text using the configured speech-to-text model. The transcript is stored as a .txt file on S3 and the URL is returned; use this for clips that may be too long to inline in the conversation.",
|
|
12255
|
+
inputSchema: z10.object({
|
|
12256
|
+
audio_url: z10.string().describe(
|
|
12257
|
+
"Location of the audio file to transcribe. Accepts a publicly fetchable URL (https URL or presigned S3 URL), or a sandbox path such as '/tmp/exulu-sessions/<sessionId>/<file>' or 'file:///tmp/exulu-sessions/<sessionId>/<file>' \u2014 sandbox paths are resolved to their persisted S3 copy."
|
|
12258
|
+
),
|
|
12259
|
+
language: z10.string().optional().describe(
|
|
12260
|
+
"ISO-639-1 language code of the audio, e.g. 'en' or 'de'. Omit to let the model auto-detect."
|
|
12261
|
+
)
|
|
12262
|
+
}),
|
|
12263
|
+
type: "function",
|
|
12264
|
+
config: [{
|
|
12265
|
+
name: "default_language",
|
|
12266
|
+
description: "ISO-639-1 language code of the audio, e.g. 'en' or 'de'. Omit to let the model auto-detect.",
|
|
12267
|
+
type: "string",
|
|
12268
|
+
default: void 0
|
|
12269
|
+
}],
|
|
12270
|
+
execute: async ({ audio_url, language, user, exuluConfig, sessionID }) => {
|
|
12271
|
+
if (!language && exuluConfig?.default_language) {
|
|
12272
|
+
language = exuluConfig?.default_language;
|
|
12273
|
+
} else {
|
|
12274
|
+
language = "en";
|
|
11037
12275
|
}
|
|
11038
|
-
|
|
11039
|
-
|
|
11040
|
-
|
|
12276
|
+
language = exuluConfig?.default_language;
|
|
12277
|
+
console.log("[EXULU] Exulu config", exuluConfig);
|
|
12278
|
+
if (!isLiteLLMEnabled()) {
|
|
12279
|
+
console.error("[EXULU] Speech-to-text is not enabled on this deployment (EXULU_USE_LITELLM is not 'true').");
|
|
12280
|
+
throw new Error(
|
|
12281
|
+
"Speech-to-text is not enabled on this deployment (EXULU_USE_LITELLM is not 'true')."
|
|
12282
|
+
);
|
|
11041
12283
|
}
|
|
11042
|
-
|
|
11043
|
-
|
|
11044
|
-
|
|
11045
|
-
const fallback = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
|
11046
|
-
return fallback;
|
|
11047
|
-
}
|
|
11048
|
-
function getSetupScriptPath(packageRoot) {
|
|
11049
|
-
return resolve(packageRoot, "ee/python/setup.sh");
|
|
11050
|
-
}
|
|
11051
|
-
function getVenvPath(packageRoot) {
|
|
11052
|
-
return resolve(packageRoot, "ee/python/.venv");
|
|
11053
|
-
}
|
|
11054
|
-
function isPythonEnvironmentSetup(packageRoot) {
|
|
11055
|
-
const root = packageRoot ?? getPackageRoot();
|
|
11056
|
-
const venvPath = getVenvPath(root);
|
|
11057
|
-
const pythonPath = join2(venvPath, "bin", "python");
|
|
11058
|
-
return existsSync2(venvPath) && existsSync2(pythonPath);
|
|
11059
|
-
}
|
|
11060
|
-
async function setupPythonEnvironment(options = {}) {
|
|
11061
|
-
const {
|
|
11062
|
-
packageRoot = getPackageRoot(),
|
|
11063
|
-
force = false,
|
|
11064
|
-
verbose = false,
|
|
11065
|
-
timeout = 6e5
|
|
11066
|
-
// 10 minutes
|
|
11067
|
-
} = options;
|
|
11068
|
-
if (!force && isPythonEnvironmentSetup(packageRoot)) {
|
|
11069
|
-
if (verbose) {
|
|
11070
|
-
console.log("\u2713 Python environment already set up");
|
|
12284
|
+
if (!process.env.TRANSCRIPTION_MODEL) {
|
|
12285
|
+
console.error("[EXULU] TRANSCRIPTION_MODEL env var is not set.");
|
|
12286
|
+
throw new Error("TRANSCRIPTION_MODEL env var is not set.");
|
|
11071
12287
|
}
|
|
11072
|
-
|
|
11073
|
-
|
|
11074
|
-
|
|
11075
|
-
|
|
11076
|
-
|
|
11077
|
-
}
|
|
11078
|
-
const setupScriptPath = getSetupScriptPath(packageRoot);
|
|
11079
|
-
if (!existsSync2(setupScriptPath)) {
|
|
11080
|
-
return {
|
|
11081
|
-
success: false,
|
|
11082
|
-
message: `Setup script not found at: ${setupScriptPath}`,
|
|
11083
|
-
alreadyExists: false
|
|
11084
|
-
};
|
|
11085
|
-
}
|
|
11086
|
-
try {
|
|
11087
|
-
if (verbose) {
|
|
11088
|
-
console.log("Setting up Python environment...");
|
|
12288
|
+
if (!exuluConfig?.fileUploads) {
|
|
12289
|
+
console.error("[EXULU] File uploads are not configured; the transcribe tool requires S3 to store transcripts.");
|
|
12290
|
+
throw new Error(
|
|
12291
|
+
"File uploads are not configured; the transcribe tool requires S3 to store transcripts."
|
|
12292
|
+
);
|
|
11089
12293
|
}
|
|
11090
|
-
const
|
|
11091
|
-
|
|
11092
|
-
|
|
11093
|
-
|
|
11094
|
-
|
|
11095
|
-
|
|
11096
|
-
|
|
11097
|
-
|
|
11098
|
-
|
|
11099
|
-
|
|
12294
|
+
const sandboxPath = parseSandboxPath(audio_url);
|
|
12295
|
+
let buffer;
|
|
12296
|
+
let mimetype;
|
|
12297
|
+
let originalname;
|
|
12298
|
+
if (sandboxPath) {
|
|
12299
|
+
if (!user?.id) {
|
|
12300
|
+
throw new Error(
|
|
12301
|
+
"Sandbox audio paths require an authenticated user; got no user on the tool call."
|
|
12302
|
+
);
|
|
12303
|
+
}
|
|
12304
|
+
if (sessionID && sandboxPath.sessionId !== sessionID) {
|
|
12305
|
+
throw new Error(
|
|
12306
|
+
`Refusing to transcribe an audio file from a different session's sandbox (path session=${sandboxPath.sessionId}, current session=${sessionID}).`
|
|
12307
|
+
);
|
|
12308
|
+
}
|
|
12309
|
+
const rawKey = `user_${user.id}/sessions/${sandboxPath.sessionId}/${sandboxPath.relPath}`;
|
|
12310
|
+
console.log("[EXULU] Transcribing audio from sandbox path", {
|
|
12311
|
+
rawKey
|
|
12312
|
+
});
|
|
12313
|
+
const matches = await listS3ObjectsByPrefix(rawKey, exuluConfig);
|
|
12314
|
+
const found = matches.find((m) => m.key.endsWith(rawKey));
|
|
12315
|
+
if (!found) {
|
|
12316
|
+
console.error("[EXULU] Sandbox audio file not found in S3 storage at", {
|
|
12317
|
+
rawKey,
|
|
12318
|
+
matches
|
|
12319
|
+
});
|
|
12320
|
+
throw new Error(
|
|
12321
|
+
`Sandbox audio file not found in S3 storage at "${rawKey}". The file may not have been persisted yet \u2014 try again after the sandbox flushes it.`
|
|
12322
|
+
);
|
|
12323
|
+
}
|
|
12324
|
+
buffer = await getS3ObjectBytes(found.key, exuluConfig);
|
|
12325
|
+
originalname = decodeURIComponent(
|
|
12326
|
+
sandboxPath.relPath.split("/").pop() || "audio"
|
|
12327
|
+
);
|
|
12328
|
+
mimetype = audioMimetypeFromExtension(originalname);
|
|
12329
|
+
} else {
|
|
12330
|
+
console.log("[EXULU] Fetching audio from URL", {
|
|
12331
|
+
audio_url
|
|
12332
|
+
});
|
|
12333
|
+
const upstream = await fetch(audio_url);
|
|
12334
|
+
if (!upstream.ok) {
|
|
12335
|
+
console.error("[EXULU] Failed to fetch audio from", {
|
|
12336
|
+
audio_url,
|
|
12337
|
+
upstream
|
|
12338
|
+
});
|
|
12339
|
+
throw new Error(
|
|
12340
|
+
`Failed to fetch audio from ${audio_url}: ${upstream.status} ${upstream.statusText}`
|
|
12341
|
+
);
|
|
12342
|
+
}
|
|
12343
|
+
mimetype = upstream.headers.get("content-type") || "audio/mpeg";
|
|
12344
|
+
if (!mimetype.startsWith("audio/")) {
|
|
12345
|
+
throw new Error(
|
|
12346
|
+
`URL did not return an audio file (content-type: ${mimetype}).`
|
|
12347
|
+
);
|
|
12348
|
+
}
|
|
12349
|
+
buffer = Buffer.from(await upstream.arrayBuffer());
|
|
12350
|
+
originalname = "audio";
|
|
12351
|
+
try {
|
|
12352
|
+
const pathname = new URL(audio_url).pathname;
|
|
12353
|
+
const last = pathname.split("/").pop();
|
|
12354
|
+
if (last) originalname = decodeURIComponent(last);
|
|
12355
|
+
} catch {
|
|
12356
|
+
}
|
|
12357
|
+
}
|
|
12358
|
+
const { text } = await transcribeAudio({
|
|
12359
|
+
file: { buffer, originalname, mimetype },
|
|
12360
|
+
language
|
|
12361
|
+
});
|
|
12362
|
+
const transcriptBuffer = Buffer.from(text, "utf-8");
|
|
12363
|
+
const transcriptFilename = `${randomUUID5()}.txt`;
|
|
12364
|
+
const transcriptKey = sessionID ? `sessions/${sessionID}/transcripts/${transcriptFilename}` : `transcripts/${transcriptFilename}`;
|
|
12365
|
+
console.log("[EXULU] Uploading transcript to S3", {
|
|
12366
|
+
transcriptFilename,
|
|
12367
|
+
transcriptKey
|
|
12368
|
+
});
|
|
12369
|
+
const url = await uploadFile(
|
|
12370
|
+
transcriptBuffer,
|
|
12371
|
+
transcriptKey,
|
|
12372
|
+
exuluConfig,
|
|
12373
|
+
{ contentType: "text/plain" },
|
|
12374
|
+
user?.id
|
|
12375
|
+
);
|
|
12376
|
+
console.log("[EXULU] Uploaded transcript to S3", {
|
|
12377
|
+
url
|
|
11100
12378
|
});
|
|
11101
|
-
|
|
11102
|
-
|
|
11103
|
-
|
|
11104
|
-
|
|
11105
|
-
|
|
12379
|
+
let sandboxLocalPath;
|
|
12380
|
+
if (sessionID) {
|
|
12381
|
+
sandboxLocalPath = join2(
|
|
12382
|
+
SANDBOX_ROOT,
|
|
12383
|
+
sessionID,
|
|
12384
|
+
"transcripts",
|
|
12385
|
+
transcriptFilename
|
|
12386
|
+
);
|
|
12387
|
+
console.log("[EXULU] Mirroring transcript into session sandbox", {
|
|
12388
|
+
sandboxLocalPath
|
|
12389
|
+
});
|
|
12390
|
+
try {
|
|
12391
|
+
await mkdir2(dirname(sandboxLocalPath), { recursive: true });
|
|
12392
|
+
await writeFile2(sandboxLocalPath, transcriptBuffer);
|
|
12393
|
+
} catch (err) {
|
|
12394
|
+
console.error(
|
|
12395
|
+
`[EXULU] Failed to write transcript to sandbox dir ${sandboxLocalPath}; S3 copy is unaffected.`,
|
|
12396
|
+
err
|
|
12397
|
+
);
|
|
12398
|
+
sandboxLocalPath = void 0;
|
|
12399
|
+
}
|
|
11106
12400
|
}
|
|
12401
|
+
console.log("[EXULU] Transcribed audio successfully", {
|
|
12402
|
+
text,
|
|
12403
|
+
url,
|
|
12404
|
+
sandboxLocalPath,
|
|
12405
|
+
length: text.length
|
|
12406
|
+
});
|
|
11107
12407
|
return {
|
|
11108
|
-
|
|
11109
|
-
message: "Python environment set up successfully",
|
|
11110
|
-
alreadyExists: false,
|
|
11111
|
-
pythonVersion,
|
|
11112
|
-
output
|
|
11113
|
-
};
|
|
11114
|
-
} catch (error) {
|
|
11115
|
-
const errorOutput = error.stdout + error.stderr;
|
|
11116
|
-
return {
|
|
11117
|
-
success: false,
|
|
11118
|
-
message: `Setup failed: ${error.message}`,
|
|
11119
|
-
alreadyExists: false,
|
|
11120
|
-
output: errorOutput
|
|
12408
|
+
result: sandboxLocalPath ? `Transcript stored at: ${url} (also available in the session sandbox at ${sandboxLocalPath}, ${text.length} characters).` : `${url}`
|
|
11121
12409
|
};
|
|
11122
12410
|
}
|
|
11123
|
-
}
|
|
11124
|
-
|
|
11125
|
-
|
|
11126
|
-
|
|
11127
|
-
|
|
11128
|
-
|
|
11129
|
-
|
|
11130
|
-
|
|
11131
|
-
|
|
11132
|
-
|
|
11133
|
-
|
|
11134
|
-
|
|
11135
|
-
Option 3 (Manual - for contributors):
|
|
11136
|
-
npm run python:setup
|
|
11137
|
-
|
|
11138
|
-
These commands will automatically create a Python virtual environment (.venv)
|
|
11139
|
-
in the @exulu/backend package and install all required dependencies.
|
|
11140
|
-
|
|
11141
|
-
Requirements:
|
|
11142
|
-
- Python 3.10 or higher must be installed
|
|
11143
|
-
- pip must be available
|
|
11144
|
-
- venv module must be available (for creating virtual environments)
|
|
11145
|
-
|
|
11146
|
-
If Python dependencies are not installed, install them first, then run one of the commands above:
|
|
11147
|
-
- macOS: brew install python@3.12
|
|
11148
|
-
- Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
|
|
11149
|
-
- Alpine Linux: apk add python3 py3-pip python3-dev
|
|
11150
|
-
- Windows: Download from https://www.python.org/downloads/
|
|
11151
|
-
|
|
11152
|
-
Note: In Docker containers, ensure you install all three components:
|
|
11153
|
-
Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
|
|
11154
|
-
Alpine: apk add python3 py3-pip python3-dev
|
|
11155
|
-
`.trim();
|
|
11156
|
-
}
|
|
11157
|
-
async function validatePythonEnvironment(packageRoot, checkPackages = true) {
|
|
11158
|
-
const root = packageRoot ?? getPackageRoot();
|
|
11159
|
-
const venvPath = getVenvPath(root);
|
|
11160
|
-
const pythonPath = join2(venvPath, "bin", "python");
|
|
11161
|
-
if (!existsSync2(venvPath)) {
|
|
11162
|
-
return {
|
|
11163
|
-
valid: false,
|
|
11164
|
-
message: getPythonSetupInstructions()
|
|
11165
|
-
};
|
|
12411
|
+
});
|
|
12412
|
+
|
|
12413
|
+
// src/templates/tools/image-generation.ts
|
|
12414
|
+
import { z as z11 } from "zod";
|
|
12415
|
+
var _cachedImageModels;
|
|
12416
|
+
var setCachedImageModels = (models2) => {
|
|
12417
|
+
_cachedImageModels = models2;
|
|
12418
|
+
};
|
|
12419
|
+
var buildDefaults = (models2) => {
|
|
12420
|
+
const m = models2[0];
|
|
12421
|
+
if (!m) {
|
|
12422
|
+
return { model: "", size: "1024x1024", quality: "auto", n: 1 };
|
|
11166
12423
|
}
|
|
11167
|
-
|
|
11168
|
-
|
|
11169
|
-
|
|
11170
|
-
|
|
11171
|
-
|
|
12424
|
+
return {
|
|
12425
|
+
model: m.model_name,
|
|
12426
|
+
size: m.sizes.includes("1024x1024") ? "1024x1024" : m.sizes[0],
|
|
12427
|
+
quality: m.qualities.includes("auto") ? "auto" : m.qualities[0],
|
|
12428
|
+
n: 1
|
|
12429
|
+
};
|
|
12430
|
+
};
|
|
12431
|
+
var loadAvailableStyles = async (user) => {
|
|
12432
|
+
if (!user?.id) return [];
|
|
12433
|
+
const { db } = await postgresClient();
|
|
12434
|
+
const rows = await db.from("platform_configurations").where("config_key", "like", "image_generation_style:%").select("*");
|
|
12435
|
+
const visible = [];
|
|
12436
|
+
for (const row of rows) {
|
|
12437
|
+
const rbac = await RBACResolver(
|
|
12438
|
+
db,
|
|
12439
|
+
"platform_configurations",
|
|
12440
|
+
row.id,
|
|
12441
|
+
row.rights_mode || "private"
|
|
12442
|
+
);
|
|
12443
|
+
const hasAccess = await checkRecordAccess(
|
|
12444
|
+
{ ...row, RBAC: rbac },
|
|
12445
|
+
"read",
|
|
12446
|
+
user
|
|
12447
|
+
);
|
|
12448
|
+
if (!hasAccess) continue;
|
|
12449
|
+
const value = typeof row.config_value === "string" ? safeJsonParse(row.config_value) : row.config_value;
|
|
12450
|
+
visible.push({
|
|
12451
|
+
id: row.id,
|
|
12452
|
+
name: value?.name ?? row.config_key.replace(/^image_generation_style:/, ""),
|
|
12453
|
+
description: row.description ?? null,
|
|
12454
|
+
owner: String(row.created_by) === String(user.id) ? "user" : "shared"
|
|
12455
|
+
});
|
|
11172
12456
|
}
|
|
12457
|
+
return visible;
|
|
12458
|
+
};
|
|
12459
|
+
var safeJsonParse = (s) => {
|
|
11173
12460
|
try {
|
|
11174
|
-
|
|
12461
|
+
return JSON.parse(s);
|
|
11175
12462
|
} catch {
|
|
11176
|
-
return
|
|
11177
|
-
valid: false,
|
|
11178
|
-
message: "Python executable is not working. Please run:\n await setupPythonEnvironment({ force: true })"
|
|
11179
|
-
};
|
|
12463
|
+
return null;
|
|
11180
12464
|
}
|
|
11181
|
-
|
|
11182
|
-
|
|
11183
|
-
|
|
11184
|
-
|
|
11185
|
-
|
|
11186
|
-
|
|
11187
|
-
|
|
11188
|
-
|
|
11189
|
-
|
|
11190
|
-
|
|
11191
|
-
|
|
11192
|
-
|
|
12465
|
+
};
|
|
12466
|
+
var createImageGenerationWidgetTool = (models2) => {
|
|
12467
|
+
setCachedImageModels(models2);
|
|
12468
|
+
return new ExuluTool({
|
|
12469
|
+
id: "image_generation",
|
|
12470
|
+
name: "image_generation",
|
|
12471
|
+
description: "Open an in-chat image generation widget pre-filled with your prompt. The user picks the model, size, quality and count, optionally attaches reference images for editing, applies a saved style, generates one or more candidates, and selects the final image(s) to share back into the conversation. Use this whenever the user asks to create or edit an image.",
|
|
12472
|
+
needsApproval: false,
|
|
12473
|
+
type: "function",
|
|
12474
|
+
config: [],
|
|
12475
|
+
inputSchema: z11.object({
|
|
12476
|
+
prompt: z11.string().describe(
|
|
12477
|
+
"Initial image prompt. The user can edit it before generating."
|
|
12478
|
+
)
|
|
12479
|
+
}),
|
|
12480
|
+
execute: async ({ prompt, user, sessionID }, options) => {
|
|
12481
|
+
if (!isLiteLLMEnabled()) {
|
|
12482
|
+
throw new Error(
|
|
12483
|
+
"Image generation is not enabled on this deployment (EXULU_USE_LITELLM is not 'true')."
|
|
12484
|
+
);
|
|
11193
12485
|
}
|
|
11194
|
-
|
|
11195
|
-
|
|
12486
|
+
if (!_cachedImageModels || _cachedImageModels.length === 0) {
|
|
12487
|
+
throw new Error(
|
|
12488
|
+
"No image-generation models are registered in config.litellm.yaml."
|
|
12489
|
+
);
|
|
12490
|
+
}
|
|
12491
|
+
const toolCallId = options?.toolCallId;
|
|
12492
|
+
const styles = await loadAvailableStyles(user);
|
|
11196
12493
|
return {
|
|
11197
|
-
|
|
11198
|
-
|
|
12494
|
+
result: JSON.stringify({
|
|
12495
|
+
type: "image_generation_widget",
|
|
12496
|
+
toolCallId,
|
|
12497
|
+
sessionId: sessionID,
|
|
12498
|
+
initialPrompt: prompt,
|
|
12499
|
+
models: _cachedImageModels.map((m) => ({
|
|
12500
|
+
name: m.model_name,
|
|
12501
|
+
sizes: m.sizes,
|
|
12502
|
+
qualities: m.qualities,
|
|
12503
|
+
supportsEdit: m.supports_edit,
|
|
12504
|
+
maxN: m.max_n
|
|
12505
|
+
})),
|
|
12506
|
+
styles,
|
|
12507
|
+
defaults: buildDefaults(_cachedImageModels)
|
|
12508
|
+
})
|
|
12509
|
+
};
|
|
12510
|
+
}
|
|
12511
|
+
});
|
|
12512
|
+
};
|
|
12513
|
+
|
|
12514
|
+
// src/exulu/app/index.ts
|
|
12515
|
+
import { resolve } from "path";
|
|
12516
|
+
|
|
12517
|
+
// src/validators/postgres-name.ts
|
|
12518
|
+
var isValidPostgresName = (id) => {
|
|
12519
|
+
const regex = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
12520
|
+
const isValid = regex.test(id);
|
|
12521
|
+
const length = id.length;
|
|
12522
|
+
return isValid && length <= 80 && length > 2;
|
|
12523
|
+
};
|
|
11199
12524
|
|
|
11200
|
-
|
|
11201
|
-
|
|
11202
|
-
|
|
12525
|
+
// src/templates/contexts/transcriptions.ts
|
|
12526
|
+
var transcriptionsContext = new ExuluContext({
|
|
12527
|
+
id: "transcriptions",
|
|
12528
|
+
name: "Transcriptions",
|
|
12529
|
+
description: "Diarized audio transcripts",
|
|
12530
|
+
fields: [
|
|
12531
|
+
{ name: "transcript_text", type: "longText", editable: true },
|
|
12532
|
+
{ name: "audio", type: "file" },
|
|
12533
|
+
{ name: "language", type: "text" },
|
|
12534
|
+
{ name: "duration_seconds", type: "number" },
|
|
12535
|
+
{ name: "speakers", type: "json" },
|
|
12536
|
+
{ name: "raw_segments", type: "json", editable: false }
|
|
12537
|
+
],
|
|
12538
|
+
sources: [],
|
|
12539
|
+
active: true,
|
|
12540
|
+
configuration: {
|
|
12541
|
+
calculateVectors: "onInsert",
|
|
12542
|
+
defaultRightsMode: "private"
|
|
12543
|
+
}
|
|
12544
|
+
});
|
|
11203
12545
|
|
|
11204
|
-
|
|
11205
|
-
|
|
12546
|
+
// src/templates/contexts/index.ts
|
|
12547
|
+
var builtInContexts = {
|
|
12548
|
+
transcriptions: transcriptionsContext
|
|
12549
|
+
};
|
|
11206
12550
|
|
|
11207
|
-
|
|
11208
|
-
|
|
11209
|
-
|
|
12551
|
+
// src/exulu/transcription/polling-loop.ts
|
|
12552
|
+
var POLL_INTERVAL_MS = 5e3;
|
|
12553
|
+
var MAX_PER_TICK = 50;
|
|
12554
|
+
var timer = null;
|
|
12555
|
+
var stopped = false;
|
|
12556
|
+
var tick = async () => {
|
|
12557
|
+
if (stopped) return;
|
|
12558
|
+
try {
|
|
12559
|
+
await transcriptionService.pollOnce(MAX_PER_TICK);
|
|
12560
|
+
} catch (err) {
|
|
12561
|
+
console.error(`[EXULU-TRANSCRIPTION] polling tick failed: ${err.message}`);
|
|
12562
|
+
} finally {
|
|
12563
|
+
if (!stopped) {
|
|
12564
|
+
timer = setTimeout(tick, POLL_INTERVAL_MS);
|
|
11210
12565
|
}
|
|
11211
12566
|
}
|
|
11212
|
-
|
|
11213
|
-
|
|
11214
|
-
|
|
12567
|
+
};
|
|
12568
|
+
var startTranscriptionPollingLoop = () => {
|
|
12569
|
+
if (timer) return;
|
|
12570
|
+
stopped = false;
|
|
12571
|
+
timer = setTimeout(tick, POLL_INTERVAL_MS);
|
|
12572
|
+
const stop = () => {
|
|
12573
|
+
stopped = true;
|
|
12574
|
+
if (timer) {
|
|
12575
|
+
clearTimeout(timer);
|
|
12576
|
+
timer = null;
|
|
12577
|
+
}
|
|
11215
12578
|
};
|
|
11216
|
-
|
|
12579
|
+
process.on("SIGINT", stop);
|
|
12580
|
+
process.on("SIGTERM", stop);
|
|
12581
|
+
};
|
|
11217
12582
|
|
|
11218
12583
|
// src/exulu/app/index.ts
|
|
11219
12584
|
var isDev = process.env.NODE_ENV !== "production";
|
|
@@ -11279,8 +12644,14 @@ var ExuluApp = class {
|
|
|
11279
12644
|
rerankers
|
|
11280
12645
|
}) => {
|
|
11281
12646
|
this._evals = redisServer.host?.length && redisServer.port?.length ? [...getDefaultEvals(), ...evals ?? []] : [];
|
|
12647
|
+
if (contexts && "transcriptions" in contexts) {
|
|
12648
|
+
console.warn(
|
|
12649
|
+
"[EXULU] User-defined 'transcriptions' context overridden by built-in. Rename your context to avoid the collision."
|
|
12650
|
+
);
|
|
12651
|
+
}
|
|
11282
12652
|
this._contexts = {
|
|
11283
|
-
...contexts
|
|
12653
|
+
...contexts,
|
|
12654
|
+
...builtInContexts
|
|
11284
12655
|
};
|
|
11285
12656
|
this._rerankers = [...rerankers ?? []];
|
|
11286
12657
|
this._agents = [...agents ?? []];
|
|
@@ -11307,12 +12678,30 @@ var ExuluApp = class {
|
|
|
11307
12678
|
...providers ?? []
|
|
11308
12679
|
];
|
|
11309
12680
|
this._config = config;
|
|
12681
|
+
const transcriptionTools = [];
|
|
12682
|
+
if (process.env.TRANSCRIPTION_MODEL && config?.fileUploads && config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
|
|
12683
|
+
transcriptionTools.push(transcribeTool);
|
|
12684
|
+
}
|
|
12685
|
+
const imageGenerationTools = [];
|
|
12686
|
+
const s3Configured = !!config?.fileUploads && !!config.fileUploads.s3region && !!config.fileUploads.s3key && !!config.fileUploads.s3secret && !!config.fileUploads.s3Bucket;
|
|
12687
|
+
if (isLiteLLMEnabled() && s3Configured) {
|
|
12688
|
+
const configPath = process.env.LITELLM_CONFIG_PATH ?? resolve(getPackageRoot(), "./config.litellm.yaml");
|
|
12689
|
+
const imageModels = parseImageGenerationModels(configPath);
|
|
12690
|
+
if (imageModels.length > 0) {
|
|
12691
|
+
console.log(
|
|
12692
|
+
`[EXULU] Registering image_generation widget tool with ${imageModels.length} model(s): ${imageModels.map((m) => m.model_name).join(", ")}`
|
|
12693
|
+
);
|
|
12694
|
+
imageGenerationTools.push(createImageGenerationWidgetTool(imageModels));
|
|
12695
|
+
}
|
|
12696
|
+
}
|
|
11310
12697
|
this._tools = [
|
|
11311
12698
|
...tools ?? [],
|
|
11312
12699
|
...todoTools,
|
|
11313
12700
|
...questionTools,
|
|
11314
12701
|
...perplexityTools,
|
|
11315
|
-
emailTool
|
|
12702
|
+
emailTool,
|
|
12703
|
+
...transcriptionTools,
|
|
12704
|
+
...imageGenerationTools
|
|
11316
12705
|
// Because agents are stored in the database, we add those as tools
|
|
11317
12706
|
// at request time, not during ExuluApp initialization. We add them
|
|
11318
12707
|
// in the grahql tools resolver.
|
|
@@ -11411,6 +12800,23 @@ var ExuluApp = class {
|
|
|
11411
12800
|
);
|
|
11412
12801
|
}
|
|
11413
12802
|
}
|
|
12803
|
+
if (process.env.TRANSCRIPTION_SERVER) {
|
|
12804
|
+
try {
|
|
12805
|
+
const health = await transcriptionClient.health();
|
|
12806
|
+
console.log(
|
|
12807
|
+
`[EXULU] Transcription: enabled (server=${process.env.TRANSCRIPTION_SERVER}, device=${health.device}, GPU=${health.gpu.available ? "enabled" : "disabled"}, diarization=${health.diarization ? "enabled" : "disabled"})`
|
|
12808
|
+
);
|
|
12809
|
+
startTranscriptionPollingLoop();
|
|
12810
|
+
} catch (err) {
|
|
12811
|
+
console.warn(
|
|
12812
|
+
`[EXULU] TRANSCRIPTION_SERVER set but unreachable: ${err.message}. Transcriptions will fail until the server is up.`
|
|
12813
|
+
);
|
|
12814
|
+
}
|
|
12815
|
+
} else {
|
|
12816
|
+
console.log(
|
|
12817
|
+
"[EXULU] Transcription: disabled (TRANSCRIPTION_SERVER not set). Start a whisper server with `npx @exulu/backend exulu-start-whisper`."
|
|
12818
|
+
);
|
|
12819
|
+
}
|
|
11414
12820
|
return this._expressApp;
|
|
11415
12821
|
}
|
|
11416
12822
|
};
|
|
@@ -13194,7 +14600,9 @@ var {
|
|
|
13194
14600
|
promptLibrarySchema: promptLibrarySchema2,
|
|
13195
14601
|
contextPresetsSchema: contextPresetsSchema2,
|
|
13196
14602
|
embedderSettingsSchema: embedderSettingsSchema2,
|
|
13197
|
-
promptFavoritesSchema: promptFavoritesSchema2
|
|
14603
|
+
promptFavoritesSchema: promptFavoritesSchema2,
|
|
14604
|
+
transcriptionJobsSchema: transcriptionJobsSchema2,
|
|
14605
|
+
imageGenerationsSchema
|
|
13198
14606
|
} = coreSchemas.get();
|
|
13199
14607
|
var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
|
|
13200
14608
|
for (const field of fields) {
|
|
@@ -13234,6 +14642,8 @@ var up = async function(knex) {
|
|
|
13234
14642
|
contextPresetsSchema2(),
|
|
13235
14643
|
embedderSettingsSchema2(),
|
|
13236
14644
|
promptFavoritesSchema2(),
|
|
14645
|
+
transcriptionJobsSchema2(),
|
|
14646
|
+
imageGenerationsSchema(),
|
|
13237
14647
|
rbacSchema2(),
|
|
13238
14648
|
agentsSchema2(),
|
|
13239
14649
|
feedbackSchema2(),
|
|
@@ -13508,7 +14918,7 @@ ${WARNING_BANNER}`);
|
|
|
13508
14918
|
console.warn(`${WARNING_BANNER}
|
|
13509
14919
|
`);
|
|
13510
14920
|
};
|
|
13511
|
-
var
|
|
14921
|
+
var log2 = (line) => console.log(`[EXULU-LITELLM] ${line}`);
|
|
13512
14922
|
var initLiteLLMDatabase = async (packageRoot) => {
|
|
13513
14923
|
const configPath = process.env.LITELLM_CONFIG_PATH ?? resolve2(packageRoot, "./config.litellm.yaml");
|
|
13514
14924
|
const safety = checkLiteLLMDatabaseSafety(configPath);
|
|
@@ -13542,7 +14952,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13542
14952
|
return;
|
|
13543
14953
|
}
|
|
13544
14954
|
const target = "litellmTarget" in safety ? safety.litellmTarget : void 0;
|
|
13545
|
-
|
|
14955
|
+
log2(
|
|
13546
14956
|
`LiteLLM database mode detected (${target?.host}:${target?.port}/${target?.database}).`
|
|
13547
14957
|
);
|
|
13548
14958
|
const ensureDatabaseExists = async () => {
|
|
@@ -13570,7 +14980,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13570
14980
|
return false;
|
|
13571
14981
|
}
|
|
13572
14982
|
url.pathname = "/postgres";
|
|
13573
|
-
|
|
14983
|
+
log2(`Target database "${targetDbName}" does not exist; creating it\u2026`);
|
|
13574
14984
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(targetDbName)) {
|
|
13575
14985
|
warn([
|
|
13576
14986
|
`Refusing to auto-create database "${targetDbName}" \u2014 name`,
|
|
@@ -13583,7 +14993,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13583
14993
|
try {
|
|
13584
14994
|
await admin.connect();
|
|
13585
14995
|
await admin.query(`CREATE DATABASE "${targetDbName}"`);
|
|
13586
|
-
|
|
14996
|
+
log2(`\u2713 Created database "${targetDbName}".`);
|
|
13587
14997
|
return true;
|
|
13588
14998
|
} catch (createErr) {
|
|
13589
14999
|
warn([
|
|
@@ -13605,7 +15015,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13605
15015
|
}
|
|
13606
15016
|
};
|
|
13607
15017
|
if (!await ensureDatabaseExists()) return;
|
|
13608
|
-
|
|
15018
|
+
log2("Checking that the target database is safe to push into\u2026");
|
|
13609
15019
|
const client2 = new Client({ connectionString: litellmUrl });
|
|
13610
15020
|
let foreignTables = [];
|
|
13611
15021
|
try {
|
|
@@ -13670,7 +15080,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13670
15080
|
]);
|
|
13671
15081
|
return;
|
|
13672
15082
|
}
|
|
13673
|
-
|
|
15083
|
+
log2("Running `prisma db push` against LiteLLM's schema\u2026");
|
|
13674
15084
|
const result = spawnSync(prismaCli, ["db", "push", "--skip-generate"], {
|
|
13675
15085
|
cwd: litellmProxyDir,
|
|
13676
15086
|
env: {
|
|
@@ -13700,7 +15110,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
|
|
|
13700
15110
|
]);
|
|
13701
15111
|
return;
|
|
13702
15112
|
}
|
|
13703
|
-
|
|
15113
|
+
log2("\u2713 LiteLLM database ready.");
|
|
13704
15114
|
};
|
|
13705
15115
|
|
|
13706
15116
|
// src/postgres/init-litellm-db.ts
|
|
@@ -14237,23 +15647,23 @@ var MarkdownChunker = class {
|
|
|
14237
15647
|
import * as fs4 from "fs";
|
|
14238
15648
|
import * as path from "path";
|
|
14239
15649
|
import { generateText as generateText5, Output as Output2 } from "ai";
|
|
14240
|
-
import { z as
|
|
15650
|
+
import { z as z12 } from "zod";
|
|
14241
15651
|
import pLimit from "p-limit";
|
|
14242
|
-
import { randomUUID as
|
|
15652
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
14243
15653
|
import * as mammoth from "mammoth";
|
|
14244
15654
|
import TurndownService from "turndown";
|
|
14245
15655
|
import WordExtractor from "word-extractor";
|
|
14246
15656
|
import { parseOfficeAsync as parseOfficeAsync2 } from "officeparser";
|
|
14247
15657
|
|
|
14248
15658
|
// src/utils/python-executor.ts
|
|
14249
|
-
import { exec as
|
|
14250
|
-
import { promisify as
|
|
15659
|
+
import { exec as exec2 } from "child_process";
|
|
15660
|
+
import { promisify as promisify2 } from "util";
|
|
14251
15661
|
import { resolve as resolve3, join as join3, dirname as dirname2 } from "path";
|
|
14252
15662
|
import { existsSync as existsSync5, readFileSync as readFileSync3 } from "fs";
|
|
14253
|
-
import { fileURLToPath
|
|
14254
|
-
var
|
|
15663
|
+
import { fileURLToPath } from "url";
|
|
15664
|
+
var execAsync2 = promisify2(exec2);
|
|
14255
15665
|
function getPackageRoot2() {
|
|
14256
|
-
const currentFile =
|
|
15666
|
+
const currentFile = fileURLToPath(import.meta.url);
|
|
14257
15667
|
let currentDir = dirname2(currentFile);
|
|
14258
15668
|
let attempts = 0;
|
|
14259
15669
|
const maxAttempts = 10;
|
|
@@ -14275,7 +15685,7 @@ function getPackageRoot2() {
|
|
|
14275
15685
|
currentDir = parentDir;
|
|
14276
15686
|
attempts++;
|
|
14277
15687
|
}
|
|
14278
|
-
return resolve3(dirname2(
|
|
15688
|
+
return resolve3(dirname2(fileURLToPath(import.meta.url)), "../..");
|
|
14279
15689
|
}
|
|
14280
15690
|
var PythonEnvironmentError = class extends Error {
|
|
14281
15691
|
constructor(message) {
|
|
@@ -14295,11 +15705,11 @@ var PythonExecutionError = class extends Error {
|
|
|
14295
15705
|
this.exitCode = exitCode;
|
|
14296
15706
|
}
|
|
14297
15707
|
};
|
|
14298
|
-
function
|
|
15708
|
+
function getVenvPath(packageRoot) {
|
|
14299
15709
|
return resolve3(packageRoot, "ee/python/.venv");
|
|
14300
15710
|
}
|
|
14301
15711
|
function getPythonExecutable(packageRoot) {
|
|
14302
|
-
const venvPath =
|
|
15712
|
+
const venvPath = getVenvPath(packageRoot);
|
|
14303
15713
|
return join3(venvPath, "bin", "python");
|
|
14304
15714
|
}
|
|
14305
15715
|
async function validatePythonEnvironmentForExecution(packageRoot) {
|
|
@@ -14337,7 +15747,7 @@ async function executePythonScript(config) {
|
|
|
14337
15747
|
});
|
|
14338
15748
|
const command = `${pythonExecutable} "${resolvedScriptPath}" ${quotedArgs.join(" ")}`;
|
|
14339
15749
|
try {
|
|
14340
|
-
const { stdout, stderr } = await
|
|
15750
|
+
const { stdout, stderr } = await execAsync2(command, {
|
|
14341
15751
|
cwd,
|
|
14342
15752
|
timeout,
|
|
14343
15753
|
env: {
|
|
@@ -14589,15 +15999,15 @@ If the page contains a flow-chart, schematic, technical drawing or control board
|
|
|
14589
15999
|
const result = await generateText5({
|
|
14590
16000
|
model,
|
|
14591
16001
|
output: Output2.object({
|
|
14592
|
-
schema:
|
|
14593
|
-
needs_correction:
|
|
14594
|
-
corrected_text:
|
|
14595
|
-
current_page_table:
|
|
14596
|
-
headers:
|
|
14597
|
-
is_continuation:
|
|
16002
|
+
schema: z12.object({
|
|
16003
|
+
needs_correction: z12.boolean(),
|
|
16004
|
+
corrected_text: z12.string().nullable(),
|
|
16005
|
+
current_page_table: z12.object({
|
|
16006
|
+
headers: z12.array(z12.string()),
|
|
16007
|
+
is_continuation: z12.boolean()
|
|
14598
16008
|
}).nullable(),
|
|
14599
|
-
confidence:
|
|
14600
|
-
reasoning:
|
|
16009
|
+
confidence: z12.enum(["high", "medium", "low"]),
|
|
16010
|
+
reasoning: z12.string()
|
|
14601
16011
|
})
|
|
14602
16012
|
}),
|
|
14603
16013
|
messages: [
|
|
@@ -14997,7 +16407,7 @@ var loadFile = async (file, name, tempDir) => {
|
|
|
14997
16407
|
if (!fileType) {
|
|
14998
16408
|
throw new Error("[EXULU] File name does not include extension, extension is required for document processing.");
|
|
14999
16409
|
}
|
|
15000
|
-
const UUID =
|
|
16410
|
+
const UUID = randomUUID6();
|
|
15001
16411
|
let buffer;
|
|
15002
16412
|
if (Buffer.isBuffer(file)) {
|
|
15003
16413
|
filePath = path.join(tempDir, `${UUID}.${fileType}`);
|
|
@@ -15027,7 +16437,7 @@ async function documentProcessor({
|
|
|
15027
16437
|
if (!license["advanced-document-processing"]) {
|
|
15028
16438
|
throw new Error("Advanced document processing is an enterprise feature, please add a valid Exulu enterprise license key to use it.");
|
|
15029
16439
|
}
|
|
15030
|
-
const uuid =
|
|
16440
|
+
const uuid = randomUUID6();
|
|
15031
16441
|
const tempDir = path.join(process.cwd(), "temp", uuid);
|
|
15032
16442
|
const localFilesAndFoldersToDelete = [tempDir];
|
|
15033
16443
|
console.log(`[EXULU] Temporary directory for processing document ${name}: ${tempDir}`);
|