@yrpri/api 9.0.103 → 9.0.104

package/dist/active-citizen/llms/imageGeneration/s3Service.js

@@ -1,110 +0,0 @@
-import AWS from "aws-sdk";
-import fs from "fs";
-import axios from "axios";
-export class S3Service {
-    constructor(cloudflareApiKey, cloudflareZoneId) {
-        this.cloudflareApiKey = cloudflareApiKey;
-        this.cloudflareZoneId = cloudflareZoneId;
-    }
-    async uploadImageToS3(bucket, filePath, key) {
-        const s3 = new AWS.S3();
-        const fileContent = fs.readFileSync(filePath);
-        const params = {
-            Bucket: bucket,
-            Key: key,
-            Body: fileContent,
-            ACL: "public-read",
-            ContentType: "image/png",
-            ContentDisposition: "inline",
-        };
-        return new Promise((resolve, reject) => {
-            s3.upload(params, (err, data) => {
-                if (err) {
-                    reject(err);
-                }
-                fs.unlinkSync(filePath);
-                resolve(data);
-            });
-        });
-    }
-    async deleteS3Url(imageUrl) {
-        const { bucket, key } = this.parseImageUrl(imageUrl);
-        if (!bucket || !key) {
-            throw new Error("Could not parse bucket or key from URL");
-        }
-        const s3 = new AWS.S3();
-        const params = {
-            Bucket: bucket,
-            Key: key,
-            ACL: "private",
-        };
-        console.log(`Disabling/Deleting Key from S3: ${JSON.stringify(params)}`);
-        return new Promise((resolve, reject) => {
-            s3.putObjectAcl(params, (err, data) => {
-                if (err) {
-                    console.error(`Error deleting image from S3: ${err}`);
-                    reject(err);
-                }
-                else {
-                    console.log(`Deleted image from S3: ${imageUrl}`, data);
-                    if (this.cloudflareApiKey && this.cloudflareZoneId) {
-                        console.log("Purging Cloudflare cache for image:", imageUrl);
-                        axios
-                            .post(`https://api.cloudflare.com/client/v4/zones/${this.cloudflareZoneId}/purge_cache`, { files: [imageUrl] }, {
-                            headers: {
-                                Authorization: `Bearer ${this.cloudflareApiKey}`,
-                                "Content-Type": "application/json",
-                            },
-                        })
-                            .then((response) => {
-                            console.log("Cloudflare cache purged:", response.data);
-                            resolve(data);
-                        })
-                            .catch((error) => {
-                            if (error.response) {
-                                console.error("Error purging Cloudflare cache:", error.response.data);
-                                console.error("Status code:", error.response.status);
-                                console.error("Headers:", error.response.headers);
-                            }
-                            else if (error.request) {
-                                console.error("No response received:", error.request);
-                            }
-                            else {
-                                console.error("Error setting up request:", error.message);
-                            }
-                            resolve(data);
-                        });
-                    }
-                    else {
-                        resolve(data);
-                    }
-                }
-            });
-        });
-    }
-    parseImageUrl(imageUrl) {
-        let bucket, key;
-        const cfImageProxyDomain = process.env.CLOUDFLARE_IMAGE_PROXY_DOMAIN;
-        const s3Bucket = process.env.S3_BUCKET;
-        if (cfImageProxyDomain && imageUrl.includes(cfImageProxyDomain)) {
-            const urlPath = new URL(imageUrl).pathname;
-            const [, ...pathParts] = urlPath.split("/");
-            bucket = s3Bucket;
-            key = pathParts.join("/");
-        }
-        else {
-            const match = imageUrl.match(/https:\/\/(.+?)\.s3\.amazonaws\.com\/(.+)/);
-            if (match) {
-                bucket = match[1];
-                key = match[2];
-            }
-        }
-        return { bucket, key };
-    }
-    async deleteMediaFormatsUrls(formats) {
-        for (const url of formats) {
-            await this.deleteS3Url(url);
-            console.log(`Deleted image from S3: ${url}`);
-        }
-    }
-}

package/dist/active-citizen/llms/llmTranslation.js

@@ -1,472 +0,0 @@
-import { jsonrepair } from "jsonrepair";
-import { OpenAI } from "openai";
-import { YpLanguages } from "../../utils/ypLanguages.js";
-import * as cheerio from "cheerio";
-export class YpLlmTranslation {
-    constructor() {
-        this.modelName = "gpt-4o";
-        this.maxTokens = 4000;
-        this.temperature = 0.0;
-        this.openaiClient = new OpenAI({
-            apiKey: process.env.OPENAI_API_KEY,
-        });
-    }
-    extractHtmlStrings(html) {
-        const $ = cheerio.load(html);
-        const strings = [];
-        // Function to recursively extract text from all elements
-        function recursivelyExtractText(elements) {
-            elements.each((index, element) => {
-                if (element.tagName.toLowerCase() === "script" ||
-                    element.tagName.toLowerCase().indexOf("icon") > -1) {
-                    return;
-                }
-                // Check if the element itself contains a direct text node
-                $(element)
-                    .contents()
-                    .filter((idx, content) => {
-                    // Ensure that content is a text node and not empty
-                    return (content.type === "text" && $(content).text().trim().length > 0);
-                })
-                    .each((idx, content) => {
-                    const text = $(content).text().trim();
-                    if (text) {
-                        strings.push(text);
-                    }
-                });
-                // Recursively extract text from child elements
-                recursivelyExtractText($(element).children());
-            });
-        }
-        // Start the recursive text extraction from the body or root of the document
-        recursivelyExtractText($("body").length ? $("body") : $.root());
-        // Attributes with user-facing text
-        $("input[placeholder], input[value]").each((index, element) => {
-            const placeholder = $(element).attr("placeholder");
-            if (placeholder && placeholder.trim().length > 0) {
-                strings.push(placeholder.trim());
-            }
-            const value = $(element).attr("value");
-            if (value &&
-                $(element).attr("type") !== "text" &&
-                value.trim().length > 0) {
-                strings.push(value.trim());
-            }
-        });
-        $('[label]').each((index, element) => {
-            const label = $(element).attr('label')?.trim();
-            if (label && label.length > 0) {
-                strings.push(label);
-            }
-        });
-        // Return unique non-empty strings
-        return [...new Set(strings)];
-    }
-    replaceHtmlStrings(html, originalStrings, translatedStrings) {
-        const $ = cheerio.load(html);
-        // Function to replace attribute values safely
-        function replaceAttributeValues(container) {
-            $(container)
-                .find("input, textarea, select, [label]")
-                .each(function () {
-                const element = $(this);
-                // Replace placeholder attribute for input, textarea, and select
-                const placeholder = element.attr("placeholder");
-                if (placeholder) {
-                    const index = originalStrings.indexOf(placeholder.trim());
-                    if (index > -1 && translatedStrings[index]) {
-                        element.attr("placeholder", translatedStrings[index]);
-                    }
-                }
-                // Replace value attribute for inputs that are not of type text, email, etc.
-                if (element.is('input[type="button"], input[type="submit"]')) {
-                    const value = element.val();
-                    if (typeof value === "string") {
-                        const index = originalStrings.indexOf(value.trim());
-                        if (index > -1 && translatedStrings[index]) {
-                            element.val(translatedStrings[index]);
-                        }
-                    }
-                }
-                // Replace label attribute for any element with a label
-                if (element.is('[label]')) {
-                    const label = element.attr('label');
-                    if (label) {
-                        const index = originalStrings.indexOf(label.trim());
-                        if (index > -1 && translatedStrings[index]) {
-                            element.attr('label', translatedStrings[index]);
-                        }
-                    }
-                }
-            });
-        }
-        // Function to safely replace text without disrupting child elements
-        function safelyReplaceText(container) {
-            $(container)
-                .contents()
-                .each(function () {
-                // If the node is a text node
-                if (this.type === "text") {
-                    let text = $(this).text();
-                    originalStrings.forEach((str, index) => {
-                        // Replace text if it exactly matches the original string (considering whitespace)
-                        if (text.trim() === str.trim() && translatedStrings[index]) {
-                            text = text.replace(str, translatedStrings[index]);
-                            $(this).replaceWith(text);
-                        }
-                    });
-                }
-                else if (this.type === "tag") {
-                    // Recursively handle nested elements
-                    safelyReplaceText(this);
-                }
-            });
-        }
-        // Replace attribute values before replacing text
-        replaceAttributeValues($.root());
-        // Initiate replacement from the root element for text nodes
-        safelyReplaceText($.root());
-        return $.html();
-    }
-    // System messages
-    renderSchemaSystemMessage(jsonInSchema, jsonOutSchema, lengthInfo) {
-        return `You are a helpful answer translation assistant that knows all the world languages.
-      INPUTS:
-      The user will tell us the Language to translate to.
-
-      The user will let you know what the question is but you do not need to translate that one.
-
-      You will get JSON input with the string to be translated: ${jsonInSchema}
-
-      OUTPUT:
-      You will output JSON format with the translation. ${jsonOutSchema}
-
-      INSTRUCTIONS:
-      The translated text MUST NEVER be more than ${lengthInfo}, otherwise it wont fit the UI.
-      Please count the words and never go over the limit. Leave some things out off the translation it's going to be too long.
-      Translate the tone of the original language also.
-      Always output only JSON.`;
-    }
-    renderSchemaTryAgainSystemMessage(jsonInSchema, jsonOutSchema, lengthInfo, currentToLong) {
-        return `You are a helpful answer translation assistant that knows all the world languages.
-      INPUTS:
-      The user will tell us the Language to translate to.
-
-      The user will let you know what the question is but you do not need to translate that one.
-
-      You will get JSON input with the string to be translated: ${jsonInSchema}
-
-      OUTPUT:
-      You will output JSON format with the translation. ${jsonOutSchema}
-
-      INSTRUCTIONS:
-      The translated text MUST NEVER be more than ${lengthInfo}, otherwise it wont fit the UI.
-      Please count the words and never go over the limit. Leave some things out off the translation it's going to be too long.
-      Translate the tone of the original language also.
-      Always output only JSON.
-
-      IMPORTANT INSTRUCTIONS:
-      You have already attempted to translate the string user is submitting and it is too long, see here:
-      ${currentToLong}
-      It MUST be shorter than this. Please try again and make it shorter.
-      `;
-    }
-    renderOneTranslationSystemMessage() {
-        return `You are a helpful answer translation assistant that knows all the world languages.
-      INPUTS:
-      The user will tell us the Language to translate to.
-      The user will give you a string to be translated.
-
-      OUTPUT:
-      You will output only the translated string.
-
-      INSTRUCTIONS:
-      Keep it similar length as the original text.
-      Translate the tone of the original language also.
-      NEVER output anything else than the translated string.`;
-    }
-    renderListTranslationSystemMessage() {
-        return `You are a helpful translation assistant that knows all the world languages.
-
-      INPUTS:
-      The user will tell us the Language to translate to.
-
-      You will get JSON with an array of strings to translate:
-      [
-        "string",
-        ...
-      ]
-
-      OUTPUT:
-      You will output JSON string array in the same order as the input array.
-      [
-        "string",
-        ...
-      ]
-
-
-      INSTRUCTIONS:
-      Do not translate brand names including ours: All Our Ideas and Your Priorities except it is a well known brand in the target language.
-      Always output only a JSON string array.`;
-    }
-    // User messages
-    renderOneTranslationUserMessage(language, stringToTranslate) {
-        return `Language to translate to: ${language}
-      String to translate:
-      ${stringToTranslate}
-      Your translated string:`;
-    }
-    renderListTranslationUserMessage(language, textsToTranslate) {
-        return `Language to translate to: ${language}
-      Texts to translate in JSON string array:
-      ${JSON.stringify(textsToTranslate, null, 2)}
-
-      Your ${language} translations as JSON string array:`;
-    }
-    renderAnswersUserMessage(language, question, answer) {
-        return `Language to translate to: ${language}
-
-    Question:
-    ${question}
-
-    Answers to translate in JSON Input:
-    ${JSON.stringify(answer, null, 2)}
-
-    Your ${language} JSON output:`;
-    }
-    renderQuestionUserMessage(language, question, questionData) {
-        return `Language to translate to: ${language}
-
-    Question to translate in JSON format:
-    ${JSON.stringify(questionData, null, 2)}
-
-    Your ${language} JSON output:`;
-    }
-    async getModerationFlag(content) {
-        const moderationResponse = await this.openaiClient.moderations.create({
-            input: content,
-        });
-        console.log("Moderation response:", moderationResponse);
-        const flagged = moderationResponse.results[0].flagged;
-        return flagged;
-    }
-    async getHtmlTranslation(languageIsoCode, htmlToTranslate) {
-        try {
-            const originalStrings = this.extractHtmlStrings(htmlToTranslate);
-            if (originalStrings.length === 0) {
-                console.warn("No HTML strings to translate");
-                return htmlToTranslate;
-            }
-            const batchSize = 10;
-            const translatedStrings = [];
-            for (let i = 0; i < originalStrings.length; i += batchSize) {
-                const batch = originalStrings.slice(i, i + batchSize);
-                const translatedBatch = await this.getListTranslation(languageIsoCode, batch);
-                if (translatedBatch) {
-                    translatedStrings.push(...translatedBatch);
-                }
-                else {
-                    console.error("Failed to translate batch:", batch);
-                    return undefined;
-                }
-            }
-            // Replace original strings in HTML with their translations
-            const translatedHtml = this.replaceHtmlStrings(htmlToTranslate, originalStrings, translatedStrings);
-            return translatedHtml;
-        }
-        catch (error) {
-            console.error("Error in getHtmlTranslation:", error);
-            return undefined;
-        }
-    }
-    async getOneTranslation(languageIsoCode, stringToTranslate) {
-        try {
-            console.log(`getOneTranslation: ${stringToTranslate} ${languageIsoCode}`);
-            const languageName = YpLanguages.getEnglishName(languageIsoCode) || languageIsoCode;
-            return (await this.callSimpleLlm(languageName, stringToTranslate, false, this.renderOneTranslationSystemMessage, this.renderOneTranslationUserMessage));
-        }
-        catch (error) {
-            console.error("Error in getAnswerIdeas:", error);
-            return undefined;
-        }
-    }
-    async getListTranslation(languageIsoCode, stringsToTranslate) {
-        try {
-            console.log(`getOneTranslation: ${languageIsoCode}`);
-            const languageName = YpLanguages.getEnglishName(languageIsoCode) || languageIsoCode;
-            if (await this.getModerationFlag(stringsToTranslate.join(" "))) {
-                console.error("Flagged:", stringsToTranslate);
-                return null;
-            }
-            else {
-                return (await this.callSimpleLlm(languageName, stringsToTranslate, true, this.renderListTranslationSystemMessage, this.renderListTranslationUserMessage));
-            }
-        }
-        catch (error) {
-            console.error("Error in getAnswerIdeas:", error);
-            return undefined;
-        }
-    }
-    async getChoiceTranslation(languageIsoCode, answerContent, maxCharactersInTranslation = 140) {
-        try {
-            console.log(`async getChoiceTranslation: ${answerContent}`);
-            const languageName = YpLanguages.getEnglishName(languageIsoCode) || languageIsoCode;
-            if (await this.getModerationFlag(answerContent)) {
-                console.error("Flagged:", answerContent);
-                return null;
-            }
-            else {
-                const inAnswer = {
-                    answerToTranslate: answerContent,
-                };
-                const jsonInSchema = `{ answerToTranslate: string}`;
-                const jsonOutSchema = `{ translatedContent: string}`;
-                const lengthInfo = `26 words long or 140 characters`;
-                return await this.callSchemaLlm(jsonInSchema, jsonOutSchema, lengthInfo, languageName, "", inAnswer, maxCharactersInTranslation, this.renderSchemaSystemMessage, this.renderAnswersUserMessage);
-            }
-        }
-        catch (error) {
-            console.error("Error in getAnswerIdeas:", error);
-            return undefined;
-        }
-    }
-    async getQuestionTranslation(languageIsoCode, question, maxCharactersInTranslation = 300) {
-        try {
-            console.log(`getQuestionTranslation: ${question} ${languageIsoCode}`);
-            const languageName = YpLanguages.getEnglishName(languageIsoCode) || languageIsoCode;
-            if (await this.getModerationFlag(question)) {
-                console.error("Flagged:", question);
-                return null;
-            }
-            else {
-                const inQuestion = {
-                    questionToTranslate: question,
-                };
-                const jsonInSchema = `{ questionToTranslate: string}`;
-                const jsonOutSchema = `{ translatedContent: string}`;
-                const lengthInfo = `40 words long or 250 characters`;
-                return await this.callSchemaLlm(jsonInSchema, jsonOutSchema, lengthInfo, languageName, question, inQuestion, maxCharactersInTranslation, this.renderSchemaSystemMessage, this.renderQuestionUserMessage);
-            }
-        }
-        catch (error) {
-            console.error("Error in getAnswerIdeas:", error);
-            return undefined;
-        }
-    }
-    async callSimpleLlm(languageName, toTranslate, parseJson, systemRenderer, userRenderer) {
-        const messages = [
-            {
-                role: "system",
-                content: systemRenderer(),
-            },
-            {
-                role: "user",
-                content: userRenderer(languageName, toTranslate),
-            },
-        ];
-        const maxRetries = 3;
-        let retries = 0;
-        let running = true;
-        while (running) {
-            try {
-                console.log(`Messages ${retries}:`, messages);
-                const results = await this.openaiClient.chat.completions.create({
-                    model: this.modelName,
-                    messages,
-                    max_tokens: this.maxTokens,
-                    temperature: this.temperature,
-                });
-                console.log("Results:", results);
-                let llmOutput = results.choices[0].message.content;
-                console.log("Return text:", llmOutput);
-                if (parseJson) {
-                    if (llmOutput) {
-                        llmOutput = llmOutput.replace(/```json/g, "");
-                        llmOutput = llmOutput.replace(/```/g, "");
-                        return JSON.parse(jsonrepair(llmOutput));
-                    }
-                    {
-                        console.error("No content in response");
-                        return undefined;
-                    }
-                }
-                else {
-                    return llmOutput;
-                }
-            }
-            catch (error) {
-                console.error("Error in getChoiceTranslation:", error);
-                retries++;
-                if (retries > maxRetries) {
-                    running = false;
-                    return undefined;
-                }
-            }
-        }
-    }
-    async callSchemaLlm(jsonInSchema, jsonOutSchema, lengthInfo, languageName, question, toTranslate, maxCharactersInTranslation, systemRenderer, userRenderer) {
-        console.log("Call schema LLM:", jsonInSchema, jsonOutSchema, lengthInfo, languageName, question, toTranslate, maxCharactersInTranslation);
-        const messages = [
-            {
-                role: "system",
-                content: this.renderSchemaSystemMessage(jsonInSchema, jsonOutSchema, lengthInfo),
-            },
-            {
-                role: "user",
-                content: userRenderer(languageName, question, toTranslate),
-            },
-        ];
-        const maxRetries = 5;
-        let retries = 0;
-        let running = true;
-        while (running) {
-            try {
-                console.log(`Messages ${retries}:`, messages);
-                const results = await this.openaiClient.chat.completions.create({
-                    model: this.modelName,
-                    messages,
-                    max_tokens: this.maxTokens,
-                    temperature: this.temperature,
-                });
-                console.log("Results:", results);
-                let textJson = results.choices[0].message.content;
-                console.log("Text JSON:", textJson);
-                if (textJson) {
-                    textJson = textJson.replace(/```json/g, "");
-                    textJson = textJson.replace(/```/g, "");
-                    const translationData = JSON.parse(jsonrepair(textJson));
-                    if (translationData && translationData.translatedContent) {
-                        if (maxCharactersInTranslation &&
-                            translationData.translatedContent.length >
-                                maxCharactersInTranslation) {
-                            console.log("Translation too long retrying:", translationData.translatedContent);
-                            messages[0].content = this.renderSchemaTryAgainSystemMessage(jsonInSchema, jsonOutSchema, lengthInfo, translationData.translatedContent);
-                            throw new Error("Translation too long");
-                        }
-                        running = false;
-                        console.log("Return text " + translationData.translatedContent);
-                        return translationData.translatedContent;
-                    }
-                    else {
-                        this.temperature = Math.random() * 0.99;
-                        console.log("No content in response. Temperature set to: " + this.temperature);
-                        throw new Error("No content in response");
-                    }
-                }
-                else {
-                    this.temperature = Math.random() * 0.99;
-                    console.log("No content in response. Temperature set to:" + this.temperature);
-                    throw new Error("No content in response");
-                }
-            }
-            catch (error) {
-                console.error("Error in callSchemaLlm:", error);
-                retries++;
-                if (retries > maxRetries) {
-                    running = false;
-                    return undefined;
-                }
-            }
-        }
-    }
-}