@yrpri/api 9.0.91 → 9.0.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/active-citizen/engine/allOurIdeas/iconGenerator.js +29 -42
- package/active-citizen/llms/baseChatBot.js +9 -4
- package/active-citizen/llms/collectionImageGenerator.js +27 -30
- package/active-citizen/llms/imageGeneration/collectionImageGenerator.js +103 -0
- package/active-citizen/llms/imageGeneration/dalleImageGenerator.js +83 -0
- package/active-citizen/llms/imageGeneration/fluxImageGenerator.js +49 -0
- package/active-citizen/llms/imageGeneration/iImageGenerator.js +1 -0
- package/active-citizen/llms/imageGeneration/imageProcessorService.js +64 -0
- package/active-citizen/llms/imageGeneration/imagenImageGenerator.js +107 -0
- package/active-citizen/llms/imageGeneration/s3Service.js +110 -0
- package/active-citizen/models/ac_translation_cache.cjs +2 -0
- package/active-citizen/workers/generativeAi.js +1 -1
- package/agents/assistants/baseAssistant.js +38 -25
- package/agents/assistants/baseAssistantWithVoice.js +33 -6
- package/agents/assistants/voiceAssistant.js +31 -3
- package/agents/controllers/agentSubscriptionController.js +21 -16
- package/agents/controllers/assistantsController.js +29 -10
- package/agents/managers/newAiModelSetup.js +3 -0
- package/agents/managers/subscriptionManager.js +2 -2
- package/app.js +4 -130
- package/models/image.cjs +1 -1
- package/models/index.cjs +20 -12
- package/models/video.cjs +1 -1
- package/package.json +28 -27
- package/utils/manifest_generator.cjs +0 -1
- package/utils/sitemap_generator.cjs +6 -0
- package/webSockets.js +346 -0
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { v4 as uuidv4 } from "uuid";
|
|
4
|
+
import { PredictionServiceClient } from "@google-cloud/aiplatform";
|
|
5
|
+
import { helpers } from "@google-cloud/aiplatform";
|
|
6
|
+
export class ImagenImageGenerator {
|
|
7
|
+
constructor(s3Service) {
|
|
8
|
+
this.s3Service = s3Service;
|
|
9
|
+
this.maxRetryCount = 3;
|
|
10
|
+
// Pull these from your environment or config
|
|
11
|
+
this.projectId = process.env.GOOGLE_CLOUD_PROJECT_ID || "";
|
|
12
|
+
this.location = process.env.GOOGLE_CLOUD_IMAGEN_LOCATION || "us-east1";
|
|
13
|
+
// Matches the official endpoint for Imagen v3.0:
|
|
14
|
+
this.endpoint = `projects/${this.projectId}/locations/${this.location}/publishers/google/models/imagen-3.0-generate-002`;
|
|
15
|
+
this.s3Bucket = process.env.S3_BUCKET || "";
|
|
16
|
+
if (!this.projectId) {
|
|
17
|
+
console.warn("Warning: GOOGLE_CLOUD_PROJECT_ID is not set. Vertex AI calls may fail.");
|
|
18
|
+
}
|
|
19
|
+
if (!this.s3Bucket) {
|
|
20
|
+
console.warn("Warning: S3_BUCKET is not set. Image upload may fail.");
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Generates an image from a text prompt using Vertex AI Imagen
|
|
25
|
+
* and returns a public S3 URL to the uploaded image.
|
|
26
|
+
*/
|
|
27
|
+
async generateImageUrl(prompt, type = "logo") {
|
|
28
|
+
let retryCount = 0;
|
|
29
|
+
let finalUrl;
|
|
30
|
+
// Configure the API endpoint for Vertex AI
|
|
31
|
+
const clientOptions = {
|
|
32
|
+
apiEndpoint: `${this.location}-aiplatform.googleapis.com`,
|
|
33
|
+
};
|
|
34
|
+
const predictionServiceClient = new PredictionServiceClient(clientOptions);
|
|
35
|
+
while (retryCount < this.maxRetryCount && !finalUrl) {
|
|
36
|
+
try {
|
|
37
|
+
// 1) Prepare the prompt and parameters for Imagen
|
|
38
|
+
const promptText = {
|
|
39
|
+
prompt: prompt, // The text prompt describing what you want to see
|
|
40
|
+
};
|
|
41
|
+
const instanceValue = helpers.toValue(promptText);
|
|
42
|
+
const instances = [instanceValue];
|
|
43
|
+
const parameter = {
|
|
44
|
+
sampleCount: 1,
|
|
45
|
+
// You can tweak these settings to your needs:
|
|
46
|
+
// seed: 100,
|
|
47
|
+
// addWatermark: false,
|
|
48
|
+
aspectRatio: "16:9",
|
|
49
|
+
safetyFilterLevel: "block_some",
|
|
50
|
+
personGeneration: "allow_adult",
|
|
51
|
+
};
|
|
52
|
+
const parameters = helpers.toValue(parameter);
|
|
53
|
+
// 2) Build the predict request
|
|
54
|
+
const request = {
|
|
55
|
+
endpoint: this.endpoint,
|
|
56
|
+
instances,
|
|
57
|
+
parameters,
|
|
58
|
+
};
|
|
59
|
+
// 3) Make the API call
|
|
60
|
+
//@ts-ignore
|
|
61
|
+
const [response] = await predictionServiceClient.predict(request);
|
|
62
|
+
const predictions = response.predictions;
|
|
63
|
+
if (!predictions || predictions.length === 0) {
|
|
64
|
+
console.warn("No image was generated. Check the request parameters and prompt.");
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
// 4) Extract base64 data from the first prediction
|
|
68
|
+
const prediction = predictions[0];
|
|
69
|
+
const b64Data = prediction.structValue?.fields?.bytesBase64Encoded?.stringValue;
|
|
70
|
+
if (!b64Data) {
|
|
71
|
+
throw new Error("Prediction did not contain base64 image data.");
|
|
72
|
+
}
|
|
73
|
+
// 5) Decode and write to a temporary file
|
|
74
|
+
const buff = Buffer.from(b64Data, "base64");
|
|
75
|
+
const tmpFilePath = path.join("/tmp", `${uuidv4()}.png`);
|
|
76
|
+
fs.writeFileSync(tmpFilePath, buff);
|
|
77
|
+
// 6) Upload the image to S3
|
|
78
|
+
const s3Key = `imagenAi/${uuidv4()}.png`;
|
|
79
|
+
await this.s3Service.uploadImageToS3(this.s3Bucket, tmpFilePath, s3Key);
|
|
80
|
+
// 7) Construct a public URL (optionally using Cloudflare)
|
|
81
|
+
if (process.env.CLOUDFLARE_IMAGE_PROXY_DOMAIN) {
|
|
82
|
+
finalUrl = `https://${process.env.CLOUDFLARE_IMAGE_PROXY_DOMAIN}/${s3Key}`;
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
finalUrl = `https://${this.s3Bucket}.s3.amazonaws.com/${s3Key}`;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
console.warn("Error generating image with Vertex AI Imagen. Will retry...");
|
|
91
|
+
console.warn(error?.message || error);
|
|
92
|
+
}
|
|
93
|
+
// Retry logic
|
|
94
|
+
if (!finalUrl) {
|
|
95
|
+
retryCount++;
|
|
96
|
+
const sleepingFor = 5000 + retryCount * 10000;
|
|
97
|
+
console.debug(`Sleeping for ${sleepingFor} ms before retry #${retryCount}...`);
|
|
98
|
+
await new Promise((resolve) => setTimeout(resolve, sleepingFor));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (!finalUrl) {
|
|
102
|
+
console.error(`Failed to generate Imagen after ${retryCount} retries.`);
|
|
103
|
+
return undefined;
|
|
104
|
+
}
|
|
105
|
+
return finalUrl;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import AWS from "aws-sdk";
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import axios from "axios";
|
|
4
|
+
export class S3Service {
|
|
5
|
+
constructor(cloudflareApiKey, cloudflareZoneId) {
|
|
6
|
+
this.cloudflareApiKey = cloudflareApiKey;
|
|
7
|
+
this.cloudflareZoneId = cloudflareZoneId;
|
|
8
|
+
}
|
|
9
|
+
async uploadImageToS3(bucket, filePath, key) {
|
|
10
|
+
const s3 = new AWS.S3();
|
|
11
|
+
const fileContent = fs.readFileSync(filePath);
|
|
12
|
+
const params = {
|
|
13
|
+
Bucket: bucket,
|
|
14
|
+
Key: key,
|
|
15
|
+
Body: fileContent,
|
|
16
|
+
ACL: "public-read",
|
|
17
|
+
ContentType: "image/png",
|
|
18
|
+
ContentDisposition: "inline",
|
|
19
|
+
};
|
|
20
|
+
return new Promise((resolve, reject) => {
|
|
21
|
+
s3.upload(params, (err, data) => {
|
|
22
|
+
if (err) {
|
|
23
|
+
reject(err);
|
|
24
|
+
}
|
|
25
|
+
fs.unlinkSync(filePath);
|
|
26
|
+
resolve(data);
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
async deleteS3Url(imageUrl) {
|
|
31
|
+
const { bucket, key } = this.parseImageUrl(imageUrl);
|
|
32
|
+
if (!bucket || !key) {
|
|
33
|
+
throw new Error("Could not parse bucket or key from URL");
|
|
34
|
+
}
|
|
35
|
+
const s3 = new AWS.S3();
|
|
36
|
+
const params = {
|
|
37
|
+
Bucket: bucket,
|
|
38
|
+
Key: key,
|
|
39
|
+
ACL: "private",
|
|
40
|
+
};
|
|
41
|
+
console.log(`Disabling/Deleting Key from S3: ${JSON.stringify(params)}`);
|
|
42
|
+
return new Promise((resolve, reject) => {
|
|
43
|
+
s3.putObjectAcl(params, (err, data) => {
|
|
44
|
+
if (err) {
|
|
45
|
+
console.error(`Error deleting image from S3: ${err}`);
|
|
46
|
+
reject(err);
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
console.log(`Deleted image from S3: ${imageUrl}`, data);
|
|
50
|
+
if (this.cloudflareApiKey && this.cloudflareZoneId) {
|
|
51
|
+
console.log("Purging Cloudflare cache for image:", imageUrl);
|
|
52
|
+
axios
|
|
53
|
+
.post(`https://api.cloudflare.com/client/v4/zones/${this.cloudflareZoneId}/purge_cache`, { files: [imageUrl] }, {
|
|
54
|
+
headers: {
|
|
55
|
+
Authorization: `Bearer ${this.cloudflareApiKey}`,
|
|
56
|
+
"Content-Type": "application/json",
|
|
57
|
+
},
|
|
58
|
+
})
|
|
59
|
+
.then((response) => {
|
|
60
|
+
console.log("Cloudflare cache purged:", response.data);
|
|
61
|
+
resolve(data);
|
|
62
|
+
})
|
|
63
|
+
.catch((error) => {
|
|
64
|
+
if (error.response) {
|
|
65
|
+
console.error("Error purging Cloudflare cache:", error.response.data);
|
|
66
|
+
console.error("Status code:", error.response.status);
|
|
67
|
+
console.error("Headers:", error.response.headers);
|
|
68
|
+
}
|
|
69
|
+
else if (error.request) {
|
|
70
|
+
console.error("No response received:", error.request);
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
console.error("Error setting up request:", error.message);
|
|
74
|
+
}
|
|
75
|
+
resolve(data);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
resolve(data);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
parseImageUrl(imageUrl) {
|
|
86
|
+
let bucket, key;
|
|
87
|
+
const cfImageProxyDomain = process.env.CLOUDFLARE_IMAGE_PROXY_DOMAIN;
|
|
88
|
+
const s3Bucket = process.env.S3_BUCKET;
|
|
89
|
+
if (cfImageProxyDomain && imageUrl.includes(cfImageProxyDomain)) {
|
|
90
|
+
const urlPath = new URL(imageUrl).pathname;
|
|
91
|
+
const [, ...pathParts] = urlPath.split("/");
|
|
92
|
+
bucket = s3Bucket;
|
|
93
|
+
key = pathParts.join("/");
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
const match = imageUrl.match(/https:\/\/(.+?)\.s3\.amazonaws\.com\/(.+)/);
|
|
97
|
+
if (match) {
|
|
98
|
+
bucket = match[1];
|
|
99
|
+
key = match[2];
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return { bucket, key };
|
|
103
|
+
}
|
|
104
|
+
async deleteMediaFormatsUrls(formats) {
|
|
105
|
+
for (const url of formats) {
|
|
106
|
+
await this.deleteS3Url(url);
|
|
107
|
+
console.log(`Deleted image from S3: ${url}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -432,6 +432,7 @@ module.exports = (sequelize, DataTypes) => {
|
|
|
432
432
|
let languageInfo = {};
|
|
433
433
|
for (let i = 0; i < textsToTranslate.length; i += chunkSize) {
|
|
434
434
|
const chunk = textsToTranslate.slice(i, i + chunkSize);
|
|
435
|
+
console.log("Calling Google Translate...");
|
|
435
436
|
const [translatedChunk, info] = await translateAPI.translate(chunk, targetLanguage);
|
|
436
437
|
translatedStrings.push(...translatedChunk);
|
|
437
438
|
if (i === 0) {
|
|
@@ -474,6 +475,7 @@ module.exports = (sequelize, DataTypes) => {
|
|
|
474
475
|
callback("No translation API");
|
|
475
476
|
return;
|
|
476
477
|
}
|
|
478
|
+
console.log("Calling Google Translate...");
|
|
477
479
|
translateAPI
|
|
478
480
|
.translate(contentToTranslate, targetLanguage)
|
|
479
481
|
.then((results) => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AoiIconGenerator } from "../engine/allOurIdeas/iconGenerator.js";
|
|
2
2
|
import models from "../../models/index.cjs";
|
|
3
|
-
import { CollectionImageGenerator } from "../llms/collectionImageGenerator.js";
|
|
3
|
+
import { CollectionImageGenerator } from "../llms/imageGeneration/collectionImageGenerator.js";
|
|
4
4
|
const dbModels = models;
|
|
5
5
|
const Image = dbModels.Image;
|
|
6
6
|
const AcBackgroundJob = dbModels.AcBackgroundJob;
|
|
@@ -52,37 +52,44 @@ Never engage in off topic conversations, always politely steer the conversation
|
|
|
52
52
|
throw new Error("Domain ID is required");
|
|
53
53
|
}
|
|
54
54
|
this.redis = redis;
|
|
55
|
-
this.wsClientId = wsClientId;
|
|
56
|
-
this.wsClientSocket = wsClients.get(this.wsClientId);
|
|
57
|
-
this.wsClients = wsClients;
|
|
58
55
|
this.openaiClient = new OpenAI({
|
|
59
56
|
apiKey: process.env.OPENAI_API_KEY,
|
|
60
57
|
});
|
|
61
58
|
this.eventEmitter = new EventEmitter();
|
|
62
59
|
this.setupClientSystemMessageListener();
|
|
60
|
+
this.clientSystemMessageListener = this.handleClientSystemMessage.bind(this);
|
|
63
61
|
this.on("update-ai-model-session", this.updateAiModelSession.bind(this));
|
|
64
62
|
}
|
|
63
|
+
destroy() {
|
|
64
|
+
// remove the WebSocket “message” listener
|
|
65
|
+
this.removeClientSystemMessageListener();
|
|
66
|
+
// remove all other event listeners on the assistant’s own EventEmitter
|
|
67
|
+
this.eventEmitter.removeAllListeners();
|
|
68
|
+
// clear references
|
|
69
|
+
this.wsClientSocket = undefined;
|
|
70
|
+
}
|
|
65
71
|
removeClientSystemMessageListener() {
|
|
66
|
-
this.wsClientSocket.
|
|
72
|
+
this.wsClientSocket.removeListener("message", this.clientSystemMessageListener);
|
|
67
73
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
default:
|
|
79
|
-
//console.log('Unhandled message type:', message.type);
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
catch (error) {
|
|
83
|
-
console.error("Error processing message:", error);
|
|
74
|
+
handleClientSystemMessage(data) {
|
|
75
|
+
try {
|
|
76
|
+
const message = JSON.parse(data.toString());
|
|
77
|
+
switch (message.type) {
|
|
78
|
+
case "client_system_message":
|
|
79
|
+
console.log("WebSockets: Processing client_system_message:", message);
|
|
80
|
+
this.processClientSystemMessage(message);
|
|
81
|
+
break;
|
|
82
|
+
default:
|
|
83
|
+
//console.log('Unhandled message type:', message.type);
|
|
84
84
|
}
|
|
85
|
-
}
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
console.error("Error processing message:", error);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
setupClientSystemMessageListener() {
|
|
91
|
+
console.log("WebSockets: setupClientSystemMessageListener called for wsClientId:", this.wsClientId);
|
|
92
|
+
this.wsClientSocket.on("message", this.handleClientSystemMessage.bind(this));
|
|
86
93
|
const listenerCountAfter = this.wsClientSocket.listenerCount("message");
|
|
87
94
|
console.log('Number of "message" listeners after adding:', listenerCountAfter);
|
|
88
95
|
}
|
|
@@ -623,10 +630,16 @@ Never engage in off topic conversations, always politely steer the conversation
|
|
|
623
630
|
convertToOpenAIMessages(chatLog) {
|
|
624
631
|
return chatLog
|
|
625
632
|
.filter((message) => message.sender !== "system")
|
|
626
|
-
.map((message) =>
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
633
|
+
.map((message) => {
|
|
634
|
+
if (message.message != null) {
|
|
635
|
+
return { role: message.sender, content: message.message };
|
|
636
|
+
}
|
|
637
|
+
else {
|
|
638
|
+
console.debug("Message content is null, message: " + JSON.stringify(message));
|
|
639
|
+
return null;
|
|
640
|
+
}
|
|
641
|
+
})
|
|
642
|
+
.filter((message) => message !== null);
|
|
630
643
|
}
|
|
631
644
|
/**
|
|
632
645
|
* Handle mode switching
|
|
@@ -8,6 +8,25 @@ export class YpBaseAssistantWithVoice extends YpBaseAssistant {
|
|
|
8
8
|
}
|
|
9
9
|
this.voiceEnabled = voiceEnabled;
|
|
10
10
|
}
|
|
11
|
+
destroy() {
|
|
12
|
+
// 2) Remove the WebSocket listener we set up in createVoiceBot()
|
|
13
|
+
const ws = this.wsClients.get(this.wsClientId);
|
|
14
|
+
if (ws && this.mainBotWsHandler) {
|
|
15
|
+
ws.removeListener("message", this.mainBotWsHandler);
|
|
16
|
+
this.mainBotWsHandler = undefined;
|
|
17
|
+
}
|
|
18
|
+
// 3) Remove the forward event handler on the voiceBot socket
|
|
19
|
+
if (this.voiceBot?.wsClientSocket && this.forwardEventHandler) {
|
|
20
|
+
this.voiceBot.wsClientSocket.removeListener("message", this.forwardEventHandler);
|
|
21
|
+
this.forwardEventHandler = undefined;
|
|
22
|
+
}
|
|
23
|
+
if (this.voiceBot) {
|
|
24
|
+
this.voiceBot.destroy();
|
|
25
|
+
this.voiceBot = undefined;
|
|
26
|
+
}
|
|
27
|
+
// 4) Finally call super.destroy()
|
|
28
|
+
super.destroy();
|
|
29
|
+
}
|
|
11
30
|
async initialize() {
|
|
12
31
|
await this.initializeModes();
|
|
13
32
|
if (this.voiceEnabled) {
|
|
@@ -25,7 +44,7 @@ export class YpBaseAssistantWithVoice extends YpBaseAssistant {
|
|
|
25
44
|
});
|
|
26
45
|
const ws = this.wsClients.get(this.wsClientId);
|
|
27
46
|
if (ws) {
|
|
28
|
-
|
|
47
|
+
this.mainBotWsHandler = async (data) => {
|
|
29
48
|
try {
|
|
30
49
|
const message = JSON.parse(data.toString());
|
|
31
50
|
switch (message.type) {
|
|
@@ -42,19 +61,26 @@ export class YpBaseAssistantWithVoice extends YpBaseAssistant {
|
|
|
42
61
|
catch (error) {
|
|
43
62
|
console.error("Error processing message:", error);
|
|
44
63
|
}
|
|
45
|
-
}
|
|
64
|
+
};
|
|
65
|
+
ws.on("message", this.mainBotWsHandler);
|
|
46
66
|
}
|
|
47
67
|
else {
|
|
48
68
|
console.error("No WebSocket found for client: ", this.wsClientId);
|
|
49
69
|
}
|
|
50
70
|
}
|
|
51
71
|
destroyVoiceBot() {
|
|
52
|
-
this.voiceBot?.
|
|
53
|
-
|
|
72
|
+
if (this.voiceBot?.wsClientSocket && this.forwardEventHandler) {
|
|
73
|
+
this.voiceBot.wsClientSocket.removeListener("message", this.forwardEventHandler);
|
|
74
|
+
this.forwardEventHandler = undefined;
|
|
75
|
+
}
|
|
76
|
+
if (this.voiceBot) {
|
|
77
|
+
this.voiceBot.destroy();
|
|
78
|
+
this.voiceBot = undefined;
|
|
79
|
+
}
|
|
54
80
|
}
|
|
55
81
|
setupVoiceEventForwarder() {
|
|
56
82
|
// Forward all voice bot events to client
|
|
57
|
-
this.
|
|
83
|
+
this.forwardEventHandler = (data) => {
|
|
58
84
|
try {
|
|
59
85
|
const event = JSON.parse(data.toString());
|
|
60
86
|
// Forward all voice-related events to client
|
|
@@ -78,7 +104,8 @@ export class YpBaseAssistantWithVoice extends YpBaseAssistant {
|
|
|
78
104
|
catch (error) {
|
|
79
105
|
console.error("Error forwarding voice event:", error);
|
|
80
106
|
}
|
|
81
|
-
}
|
|
107
|
+
};
|
|
108
|
+
this.voiceBot?.wsClientSocket?.on("message", this.forwardEventHandler);
|
|
82
109
|
}
|
|
83
110
|
async setVoiceMode(enabled) {
|
|
84
111
|
this.voiceEnabled = enabled;
|
|
@@ -19,7 +19,8 @@ export class YpBaseChatBotWithVoice extends YpBaseChatBot {
|
|
|
19
19
|
};
|
|
20
20
|
// Default voice configuration
|
|
21
21
|
this.voiceConfig = {
|
|
22
|
-
model: process.env.OPENAI_VOICE_MODEL_NAME ||
|
|
22
|
+
model: process.env.OPENAI_VOICE_MODEL_NAME ||
|
|
23
|
+
"gpt-4o-realtime-preview-2024-12-17",
|
|
23
24
|
voice: "echo",
|
|
24
25
|
modalities: ["text", "audio"],
|
|
25
26
|
};
|
|
@@ -155,7 +156,8 @@ export class YpBaseChatBotWithVoice extends YpBaseChatBot {
|
|
|
155
156
|
}
|
|
156
157
|
}
|
|
157
158
|
setupVoiceMessageHandlers(ws, disableWhenAgentIsSpeaking) {
|
|
158
|
-
|
|
159
|
+
// Instead of an inline callback, keep a reference
|
|
160
|
+
const handler = async (data) => {
|
|
159
161
|
try {
|
|
160
162
|
const event = JSON.parse(data.toString());
|
|
161
163
|
if (disableWhenAgentIsSpeaking && this.directAgentVoiceConnection) {
|
|
@@ -221,7 +223,33 @@ export class YpBaseChatBotWithVoice extends YpBaseChatBot {
|
|
|
221
223
|
catch (error) {
|
|
222
224
|
console.error("Error handling voice message:", error);
|
|
223
225
|
}
|
|
224
|
-
}
|
|
226
|
+
};
|
|
227
|
+
// Attach it
|
|
228
|
+
ws.on("message", handler);
|
|
229
|
+
// Store the reference so we can remove it in destroy()
|
|
230
|
+
if (disableWhenAgentIsSpeaking) {
|
|
231
|
+
this.voiceMainMessageHandler = handler;
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
this.voiceDirectMessageHandler = handler;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
destroy() {
|
|
238
|
+
// 1) Remove voice message handlers from assistantVoiceConnection
|
|
239
|
+
if (this.assistantVoiceConnection?.ws && this.voiceMainMessageHandler) {
|
|
240
|
+
this.assistantVoiceConnection.ws.removeListener("message", this.voiceMainMessageHandler);
|
|
241
|
+
this.voiceMainMessageHandler = undefined;
|
|
242
|
+
}
|
|
243
|
+
// 2) Remove voice message handlers from directAgentVoiceConnection
|
|
244
|
+
if (this.directAgentVoiceConnection?.ws && this.voiceDirectMessageHandler) {
|
|
245
|
+
this.directAgentVoiceConnection.ws.removeListener("message", this.voiceDirectMessageHandler);
|
|
246
|
+
this.voiceDirectMessageHandler = undefined;
|
|
247
|
+
}
|
|
248
|
+
// 3) Close any open connections
|
|
249
|
+
this.destroyAssistantVoiceConnection();
|
|
250
|
+
this.destroyDirectAgentVoiceConnection();
|
|
251
|
+
// 4) Call super destroy so it can remove *its* WebSocket listeners, etc.
|
|
252
|
+
super.destroy();
|
|
225
253
|
}
|
|
226
254
|
async handleResponseDone(event) {
|
|
227
255
|
if (event.item?.content && event.item.content.length > 0) {
|
|
@@ -11,23 +11,28 @@ export class AgentSubscriptionController {
|
|
|
11
11
|
this.router = express.Router();
|
|
12
12
|
this.getAgentConfigurationAnswers = async (req, res) => {
|
|
13
13
|
try {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
14
|
+
if (req.user) {
|
|
15
|
+
const subscriptionId = parseInt(req.params.subscriptionId);
|
|
16
|
+
// Make sure the user can only fetch their own subscription
|
|
17
|
+
const subscription = await YpSubscription.findOne({
|
|
18
|
+
where: {
|
|
19
|
+
id: subscriptionId,
|
|
20
|
+
user_id: req.user?.id
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
if (!subscription) {
|
|
24
|
+
return res.status(404).json({ error: "Subscription not found" });
|
|
25
|
+
}
|
|
26
|
+
// Extract the requiredQuestionsAnswered from subscription.configuration
|
|
27
|
+
const answers = subscription.configuration?.requiredQuestionsAnswered || [];
|
|
28
|
+
return res.status(200).json({
|
|
29
|
+
success: true,
|
|
30
|
+
data: answers,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
return res.status(401).json({ error: "Unauthorized" });
|
|
24
35
|
}
|
|
25
|
-
// Extract the requiredQuestionsAnswered from subscription.configuration
|
|
26
|
-
const answers = subscription.configuration?.requiredQuestionsAnswered || [];
|
|
27
|
-
return res.status(200).json({
|
|
28
|
-
success: true,
|
|
29
|
-
data: answers,
|
|
30
|
-
});
|
|
31
36
|
}
|
|
32
37
|
catch (error) {
|
|
33
38
|
console.error("Error retrieving subscription agent configuration:", error);
|
|
@@ -113,6 +113,9 @@ export class AssistantController {
|
|
|
113
113
|
};
|
|
114
114
|
this.getAgentConfigurationAnswers = async (req, res) => {
|
|
115
115
|
try {
|
|
116
|
+
if (!req.user) {
|
|
117
|
+
return res.status(401).json({ error: "Unauthorized" });
|
|
118
|
+
}
|
|
116
119
|
const subscriptionId = parseInt(req.params.subscriptionId);
|
|
117
120
|
// Make sure the user can only fetch their own subscription
|
|
118
121
|
const subscription = await YpSubscription.findOne({
|
|
@@ -139,6 +142,9 @@ export class AssistantController {
|
|
|
139
142
|
this.getUpdatedWorkflow = async (req, res) => {
|
|
140
143
|
const { runId } = req.params;
|
|
141
144
|
const userId = req.user?.id;
|
|
145
|
+
if (!userId) {
|
|
146
|
+
return res.status(401).json({ error: "Unauthorized" });
|
|
147
|
+
}
|
|
142
148
|
try {
|
|
143
149
|
const agentRun = await YpAgentProductRun.findOne({
|
|
144
150
|
where: {
|
|
@@ -359,12 +365,18 @@ export class AssistantController {
|
|
|
359
365
|
const { wsClientId, currentMode } = req.body;
|
|
360
366
|
const memoryId = this.getMemoryUserId(req);
|
|
361
367
|
console.log(`Starting chat session for client: ${wsClientId}`);
|
|
362
|
-
let
|
|
363
|
-
if (
|
|
364
|
-
|
|
368
|
+
let oldVoiceAssistant = this.voiceAssistantInstances.get("voiceAssistant");
|
|
369
|
+
if (oldVoiceAssistant) {
|
|
370
|
+
oldVoiceAssistant.destroy();
|
|
371
|
+
this.voiceAssistantInstances.delete("voiceAssistant");
|
|
372
|
+
}
|
|
373
|
+
let oldChatAssistant = this.chatAssistantInstances.get("mainAssistant");
|
|
374
|
+
if (oldChatAssistant) {
|
|
375
|
+
oldChatAssistant.destroy();
|
|
376
|
+
this.chatAssistantInstances.delete("mainAssistant");
|
|
365
377
|
}
|
|
366
|
-
assistant = new YpAgentAssistant(wsClientId, this.wsClients, req.redisClient, true, parseInt(req.params.domainId), memoryId);
|
|
367
|
-
this.voiceAssistantInstances.set(
|
|
378
|
+
const assistant = new YpAgentAssistant(wsClientId, this.wsClients, req.redisClient, true, parseInt(req.params.domainId), memoryId);
|
|
379
|
+
this.voiceAssistantInstances.set("voiceAssistant", assistant);
|
|
368
380
|
await assistant.initialize();
|
|
369
381
|
res.status(200).json({
|
|
370
382
|
message: "Voice session initialized",
|
|
@@ -372,7 +384,7 @@ export class AssistantController {
|
|
|
372
384
|
});
|
|
373
385
|
}
|
|
374
386
|
catch (error) {
|
|
375
|
-
console.error("Error starting
|
|
387
|
+
console.error("Error starting voice session:", error);
|
|
376
388
|
res.status(500).json({ error: "Internal server error" });
|
|
377
389
|
}
|
|
378
390
|
}
|
|
@@ -381,11 +393,18 @@ export class AssistantController {
|
|
|
381
393
|
const { wsClientId, chatLog, currentMode } = req.body;
|
|
382
394
|
const memoryId = this.getMemoryUserId(req);
|
|
383
395
|
console.log(`Starting chat session for client: ${wsClientId} with currentMode: ${currentMode}`);
|
|
384
|
-
|
|
385
|
-
if (
|
|
386
|
-
|
|
387
|
-
this.
|
|
396
|
+
const oldVoiceAssistant = this.voiceAssistantInstances.get("voiceAssistant");
|
|
397
|
+
if (oldVoiceAssistant) {
|
|
398
|
+
oldVoiceAssistant.destroy();
|
|
399
|
+
this.voiceAssistantInstances.delete("voiceAssistant");
|
|
400
|
+
}
|
|
401
|
+
const oldAssistant = this.chatAssistantInstances.get("mainAssistant");
|
|
402
|
+
if (oldAssistant) {
|
|
403
|
+
oldAssistant.destroy();
|
|
404
|
+
this.chatAssistantInstances.delete("mainAssistant");
|
|
388
405
|
}
|
|
406
|
+
const assistant = new YpAgentAssistant(wsClientId, this.wsClients, req.redisClient, false, parseInt(req.params.domainId), memoryId);
|
|
407
|
+
this.chatAssistantInstances.set("mainAssistant", assistant);
|
|
389
408
|
assistant.conversation(chatLog);
|
|
390
409
|
res.status(200).json({
|
|
391
410
|
message: "Chat session initialized",
|
|
@@ -39,6 +39,9 @@ export class NewAiModelSetup {
|
|
|
39
39
|
await psModels[modelName].associate(psSequelize.models);
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
|
+
if (process.env.FORCE_DB_SYNC || process.env.NODE_ENV === "development") {
|
|
43
|
+
await psSequelize.sync();
|
|
44
|
+
}
|
|
42
45
|
console.log("All models initialized successfully.");
|
|
43
46
|
}
|
|
44
47
|
catch (error) {
|
|
@@ -321,7 +321,7 @@ export class SubscriptionManager {
|
|
|
321
321
|
return { freeSubscription: true };
|
|
322
322
|
}
|
|
323
323
|
const stripe = new Stripe(process.env.STRIPE_SECRET_KEY, {
|
|
324
|
-
apiVersion: "
|
|
324
|
+
apiVersion: "2025-01-27.acacia",
|
|
325
325
|
});
|
|
326
326
|
// Create a PaymentIntent with Stripe
|
|
327
327
|
const paymentIntent = await stripe.paymentIntents.create({
|
|
@@ -359,7 +359,7 @@ export class SubscriptionManager {
|
|
|
359
359
|
async handleSuccessfulPayment(paymentIntentId) {
|
|
360
360
|
try {
|
|
361
361
|
const stripe = new Stripe(process.env.STRIPE_SECRET_KEY, {
|
|
362
|
-
apiVersion: "
|
|
362
|
+
apiVersion: "2025-01-27.acacia",
|
|
363
363
|
});
|
|
364
364
|
const paymentIntent = await stripe.paymentIntents.retrieve(paymentIntentId);
|
|
365
365
|
if (paymentIntent.status !== "succeeded") {
|