@learnpack/learnpack 5.0.262 → 5.0.265

package/lib/commands/serve.js

@@ -330,9 +330,18 @@ class ServeCommand extends SessionCommand_1.default {
         app.post("/webhooks/:courseSlug/initial-readme-processor", async (req, res) => {
             const { courseSlug } = req.params;
             const body = req.body;
-            // Save the file as courses/courseSlug/README.md
-            const filePath = `courses/${courseSlug}/README${(0, creatorUtilities_1.getReadmeExtension)(body.parsed.language_code)}`;
-            await uploadFileToBucket(bucket, body.parsed.content, filePath);
+            console.log("RECEIVING INITIAL README WEBHOOK", body);
+            const langCode = body.parsed.language_code || body.parsed.output_language_code || "";
+            const content = body.parsed.content || body.parsed.translation || "";
+            if (!content || !langCode) {
+                console.log("No content or language code to save", body);
+                return res.status(400).json({
+                    status: "ERROR",
+                    message: "No content to save",
+                });
+            }
+            const filePath = `courses/${courseSlug}/README${(0, creatorUtilities_1.getReadmeExtension)(langCode)}`;
+            await uploadFileToBucket(bucket, content, filePath);
             res.json({ status: "SUCCESS" });
         });
         app.post("/webhooks/:courseSlug/images/:imageId", async (req, res) => {
@@ -438,6 +447,15 @@ class ServeCommand extends SessionCommand_1.default {
             await uploadFileToBucket(bucket, JSON.stringify(newSyllabus), `courses/${courseSlug}/.learn/initialSyllabus.json`);
             res.json({ status: "SUCCESS" });
         });
+        // The following endpoint is used to store an incoming translation where it supposed to be
+        app.post("/webhooks/:courseSlug/:exSlug/save-translation", async (req, res) => {
+            const { courseSlug, exSlug } = req.params;
+            const body = req.body;
+            console.log("RECEIVING TRANSLATION WEBHOOK", body);
+            const readmePath = `courses/${courseSlug}/exercises/${exSlug}/README${(0, creatorUtilities_1.getReadmeExtension)(body.parsed.output_language_code)}`;
+            await uploadFileToBucket(bucket, body.parsed.translation, readmePath);
+            res.json({ status: "SUCCESS" });
+        });
         app.get("/check-preview-image/:slug", async (req, res) => {
             const { slug } = req.params;
             const file = bucket.file(`courses/${slug}/preview.png`);
@@ -623,65 +641,79 @@ class ServeCommand extends SessionCommand_1.default {
                 return res.status(400).json({ error: "RigoToken not found" });
             }
             const languagesToTranslate = languages.split(",");
-            const course = await bucket
-                .file(`courses/${courseSlug}/learn.json`)
-                .download();
-            const courseJson = JSON.parse(course.toString());
-            const languageCodes = new Set();
+            const languageCodesRes = await (0, rigoActions_1.getLanguageCodes)(rigoToken, {
+                raw_languages: languagesToTranslate.join(","),
+            });
+            const languageCodes = languageCodesRes.parsed.language_codes;
             try {
                 await Promise.all(exerciseSlugs.map(async (slug) => {
                     const readmePath = `courses/${courseSlug}/exercises/${slug}/README${(0, creatorUtilities_1.getReadmeExtension)(currentLanguage)}`;
                     const readme = await bucket.file(readmePath).download();
-                    await Promise.all(languagesToTranslate.map(async (language) => {
-                        const response = await (0, rigoActions_1.translateExercise)(rigoToken, {
+                    await Promise.all(languageCodes.map(async (language) => {
+                        // verify if the translation already exists
+                        const translationPath = `courses/${courseSlug}/exercises/${slug}/README${(0, creatorUtilities_1.getReadmeExtension)(language)}`;
+                        const [exists] = await bucket.file(translationPath).exists();
+                        if (exists) {
+                            console.log(`Translation in ${language} already exists for exercise ${slug}`);
+                            return;
+                        }
+                        await (0, rigoActions_1.translateExercise)(rigoToken, {
                             text_to_translate: readme.toString(),
                             output_language: language,
-                            exercise_slug: slug,
-                        });
-                        const translatedReadme = await bucket.file(`courses/${courseSlug}/exercises/${slug}/README${(0, creatorUtilities_1.getReadmeExtension)(response.parsed.output_language_code)}`);
-                        await translatedReadme.save(response.parsed.translation);
-                        languageCodes.add(response.parsed.output_language_code);
+                        }, `${process.env.HOST}/webhooks/${courseSlug}/${slug}/save-translation`);
                     }));
                 }));
-                const currentLanguages = Object.keys(courseJson.title);
-                for (const languageCode of currentLanguages) {
-                    if (languageCodes.has(languageCode)) {
-                        languageCodes.delete(languageCode);
+                const course = await bucket
+                    .file(`courses/${courseSlug}/learn.json`)
+                    .download();
+                const courseJson = JSON.parse(course.toString());
+                const neededLanguages = new Set();
+                let currentLanguages = Object.keys(courseJson.title);
+                for (const languageCode of languageCodes) {
+                    if (!currentLanguages.includes(languageCode)) {
+                        neededLanguages.add(languageCode);
                     }
                 }
-                if ([...languageCodes].length > 0) {
-                    const translatedCourseMetadata = await Promise.all([...languageCodes].map(async (languageCode) => {
-                        const result = await (0, rigoActions_1.translateCourseMetadata)(rigoToken, {
-                            title: courseJson.title[currentLanguage],
-                            description: courseJson.description[currentLanguage],
-                            destination_lang_code: languageCode,
-                        });
-                        return {
-                            languageCode,
-                            title: result.parsed.title,
-                            description: result.parsed.description,
-                        };
-                    }));
-                    for (const metadata of translatedCourseMetadata) {
-                        courseJson.title[metadata.languageCode] = metadata.title;
-                        courseJson.description[metadata.languageCode] =
-                            metadata.description;
-                    }
+                const neededLanguagesList = [...neededLanguages]
+                    .map((lang) => lang.toLowerCase())
+                    .sort();
+                if (neededLanguagesList.length > 0) {
+                    const result = await (0, rigoActions_1.translateCourseMetadata)(rigoToken, {
+                        title: JSON.stringify(courseJson.title),
+                        description: JSON.stringify(courseJson.description),
+                        new_languages: neededLanguagesList.join(","),
+                    });
+                    const translateTitle = JSON.parse(result.parsed.title);
+                    const translateDescription = JSON.parse(result.parsed.description);
+                    courseJson.title = translateTitle;
+                    courseJson.description = translateDescription;
                     await uploadFileToBucket(bucket, JSON.stringify(courseJson), `courses/${courseSlug}/learn.json`);
-                    const previewReadme = await bucket.file(`courses/${courseSlug}/README${(0, creatorUtilities_1.getReadmeExtension)(currentLanguage)}`);
-                    const [previewReadmeContent] = await previewReadme.download();
-                    const previewReadmeContentString = previewReadmeContent.toString();
-                    await Promise.all([...languageCodes].map(async (languageCode) => {
-                        const translatedPreviewReadme = await (0, rigoActions_1.translateExercise)(rigoToken, {
-                            text_to_translate: previewReadmeContentString,
-                            output_language: languageCode,
-                            exercise_slug: "preview-readme",
-                        });
-                        await bucket
-                            .file(`courses/${courseSlug}/README${(0, creatorUtilities_1.getReadmeExtension)(languageCode)}`)
-                            .save(translatedPreviewReadme.parsed.translation);
-                    }));
+                    currentLanguages = Object.keys(courseJson.title);
+                }
+                // Check that all the READMEs exists
+                const missingReadmeTranslations = [];
+                let firstAvailable = "";
+                for (const languageCode of currentLanguages) {
+                    // eslint-disable-next-line no-await-in-loop
+                    const previewReadme = await bucket.file(`courses/${courseSlug}/README${(0, creatorUtilities_1.getReadmeExtension)(languageCode)}`);
+                    // eslint-disable-next-line no-await-in-loop
+                    const [exists] = await previewReadme.exists();
+                    if (!exists) {
+                        missingReadmeTranslations.push(languageCode);
+                    }
+                    else {
+                        // eslint-disable-next-line no-await-in-loop
+                        const [previewReadmeContent] = await previewReadme.download();
+                        const previewReadmeContentString = previewReadmeContent.toString();
+                        firstAvailable = previewReadmeContentString;
+                    }
                 }
+                await Promise.all(missingReadmeTranslations.map(async (languageCode) => {
+                    await (0, rigoActions_1.translateExercise)(rigoToken, {
+                        text_to_translate: firstAvailable,
+                        output_language: languageCode,
+                    }, `${process.env.HOST}/webhooks/${courseSlug}/initial-readme-processor`);
+                }));
                 return res.status(200).json({ message: "Translated exercises" });
             }
             catch (error) {

package/lib/commands/translate.js

@@ -78,8 +78,7 @@ class BuildCommand extends SessionCommand_1.default {
                 const response = await (0, rigoActions_1.translateExercise)(rigoToken, {
                     text_to_translate: readme,
                     output_language: language,
-                    exercise_slug: exercise,
-                });
+                }, `${process.env.HOST}/webhooks/translate-exercise`);
                 await (0, creatorUtilities_1.saveTranslatedReadme)(exercise, response.parsed.output_language_code, response.parsed.translation);
                 console_1.default.success(`Translated ${exercise} to ${language} successfully`);
             }));

package/lib/managers/server/routes.js

@@ -305,8 +305,7 @@ async function default_1(app, configObject, configManager) {
                         const response = await (0, rigoActions_1.translateExercise)(rigoToken, {
                             text_to_translate: readme.body,
                             output_language: language,
-                            exercise_slug: slug,
-                        });
+                        }, `${process.env.HOST}/webhooks/translate-exercise`);
                         await (0, creatorUtilities_1.saveTranslatedReadme)(slug, response.parsed.output_language_code, response.parsed.translation);
                         (0, sidebarGenerator_1.addExerciseToSidebar)(slug, response.parsed.output_language_code, response.parsed.translated_slug, configDirPath || "");
                         console_1.default.success(`Translated ${slug} to ${language} successfully`);

package/lib/utils/rigoActions.d.ts

@@ -19,9 +19,8 @@ export declare function downloadImage(imageUrl: string, savePath: string): Promi
 type TTranslateInputs = {
     text_to_translate: string;
     output_language: string;
-    exercise_slug: string;
 };
-export declare const translateExercise: (token: string, inputs: TTranslateInputs) => Promise<any>;
+export declare const translateExercise: (token: string, inputs: TTranslateInputs, webhookUrl: string) => Promise<any>;
 type TGenerateCourseIntroductionInputs = {
     course_title: string;
     lessons_context: string;
@@ -63,7 +62,7 @@ export declare const createStructuredPreviewReadme: (token: string, inputs: TCre
 export declare const translateCourseMetadata: (token: string, inputs: {
     title: string;
     description: string;
-    destination_lang_code: string;
+    new_languages: string;
 }) => Promise<any>;
 export declare function createPreviewReadme(tutorialDir: string, packageInfo: PackageInfo, rigoToken: string, readmeContents: string[]): Promise<void>;
 type TReduceReadmeInputs = {
@@ -88,4 +87,8 @@ export declare const isPackageAuthor: (token: string, packageSlug: string) => Pr
     isAuthor: boolean;
     status: number;
 }>;
+type TGetLanguageCodesInputs = {
+    raw_languages: string;
+};
+export declare const getLanguageCodes: (token: string, inputs: TGetLanguageCodesInputs) => Promise<any>;
 export {};

package/lib/utils/rigoActions.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isPackageAuthor = exports.fillSidebarJSON = exports.generateCourseShortName = exports.isValidRigoToken = exports.translateCourseMetadata = exports.createStructuredPreviewReadme = exports.readmeCreator = exports.createCodingReadme = exports.createCodeFile = exports.interactiveCreation = exports.generateCourseIntroduction = exports.translateExercise = exports.generateImage = exports.hasCreatorPermission = exports.createReadme = void 0;
+exports.getLanguageCodes = exports.isPackageAuthor = exports.fillSidebarJSON = exports.generateCourseShortName = exports.isValidRigoToken = exports.translateCourseMetadata = exports.createStructuredPreviewReadme = exports.readmeCreator = exports.createCodingReadme = exports.createCodeFile = exports.interactiveCreation = exports.generateCourseIntroduction = exports.translateExercise = exports.generateImage = exports.hasCreatorPermission = exports.createReadme = void 0;
 exports.downloadImage = downloadImage;
 exports.createPreviewReadme = createPreviewReadme;
 exports.makeReadmeReadable = makeReadmeReadable;
@@ -79,11 +79,12 @@ async function downloadImage(imageUrl, savePath) {
     const buffer = Buffer.from(response.data, "binary");
     await (0, promises_1.writeFile)(savePath, buffer);
 }
-const translateExercise = async (token, inputs) => {
-    const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/159/`, {
+const translateExercise = async (token, inputs, webhookUrl) => {
+    const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/translate-asset-markdown/`, {
         inputs: inputs,
         include_purpose_objective: false,
-        execute_async: false,
+        execute_async: true,
+        webhook_url: webhookUrl,
     }, {
         headers: {
             "Content-Type": "application/json",
@@ -272,3 +273,13 @@ const isPackageAuthor = async (token, packageSlug) => {
     }
 };
 exports.isPackageAuthor = isPackageAuthor;
+const getLanguageCodes = async (token, inputs) => {
+    const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/get-language-codes/`, { inputs, include_purpose_objective: false, execute_async: false }, {
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: "Token " + token,
+        },
+    });
+    return response.data;
+};
+exports.getLanguageCodes = getLanguageCodes;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@learnpack/learnpack",
   "description": "Seamlessly build, sell and/or take interactive & auto-graded tutorials, start learning now or build a new tutorial to your audience.",
-  "version": "5.0.262",
+  "version": "5.0.265",
   "author": "Alejandro Sanchez @alesanchezr",
   "contributors": [
     {

package/src/commands/serve.ts

@@ -31,6 +31,7 @@ import {
   isPackageAuthor,
   createStructuredPreviewReadme,
   translateCourseMetadata,
+  getLanguageCodes,
 } from "../utils/rigoActions"
 import * as dotenv from "dotenv"
 import {
@@ -519,11 +520,25 @@ export default class ServeCommand extends SessionCommand {
         const { courseSlug } = req.params
         const body = req.body
 
-        // Save the file as courses/courseSlug/README.md
+        console.log("RECEIVING INITIAL README WEBHOOK", body)
+
+        const langCode =
+          body.parsed.language_code || body.parsed.output_language_code || ""
+
+        const content = body.parsed.content || body.parsed.translation || ""
+
+        if (!content || !langCode) {
+          console.log("No content or language code to save", body)
+          return res.status(400).json({
+            status: "ERROR",
+            message: "No content to save",
+          })
+        }
+
         const filePath = `courses/${courseSlug}/README${getReadmeExtension(
-          body.parsed.language_code
+          langCode
         )}`
-        await uploadFileToBucket(bucket, body.parsed.content, filePath)
+        await uploadFileToBucket(bucket, content, filePath)
 
         res.json({ status: "SUCCESS" })
       }
@@ -707,6 +722,25 @@ export default class ServeCommand extends SessionCommand {
       }
     )
 
+    // The following endpoint is used to store an incoming translation where it supposed to be
+    app.post(
+      "/webhooks/:courseSlug/:exSlug/save-translation",
+      async (req, res) => {
+        const { courseSlug, exSlug } = req.params
+        const body = req.body
+
+        console.log("RECEIVING TRANSLATION WEBHOOK", body)
+
+        const readmePath = `courses/${courseSlug}/exercises/${exSlug}/README${getReadmeExtension(
+          body.parsed.output_language_code
+        )}`
+
+        await uploadFileToBucket(bucket, body.parsed.translation, readmePath)
+
+        res.json({ status: "SUCCESS" })
+      }
+    )
+
     app.get("/check-preview-image/:slug", async (req, res) => {
       const { slug } = req.params
       const file = bucket.file(`courses/${slug}/preview.png`)
@@ -948,13 +982,10 @@ export default class ServeCommand extends SessionCommand {
       }
 
       const languagesToTranslate: string[] = languages.split(",")
-
-      const course = await bucket
-        .file(`courses/${courseSlug}/learn.json`)
-        .download()
-
-      const courseJson = JSON.parse(course.toString())
-      const languageCodes = new Set()
+      const languageCodesRes = await getLanguageCodes(rigoToken, {
+        raw_languages: languagesToTranslate.join(","),
+      })
+      const languageCodes = languageCodesRes.parsed.language_codes
 
       try {
         await Promise.all(
@@ -962,59 +993,68 @@ export default class ServeCommand extends SessionCommand {
             const readmePath = `courses/${courseSlug}/exercises/${slug}/README${getReadmeExtension(
               currentLanguage
             )}`
-
             const readme = await bucket.file(readmePath).download()
 
             await Promise.all(
-              languagesToTranslate.map(async (language: string) => {
-                const response = await translateExercise(rigoToken, {
-                  text_to_translate: readme.toString(),
-                  output_language: language,
-                  exercise_slug: slug,
-                })
-
-                const translatedReadme = await bucket.file(
-                  `courses/${courseSlug}/exercises/${slug}/README${getReadmeExtension(
-                    response.parsed.output_language_code
-                  )}`
+              languageCodes.map(async (language: string) => {
+                // verify if the translation already exists
+                const translationPath = `courses/${courseSlug}/exercises/${slug}/README${getReadmeExtension(
+                  language
+                )}`
+                 
+                const [exists] = await bucket.file(translationPath).exists()
+                if (exists) {
+                  console.log(
+                    `Translation in ${language} already exists for exercise ${slug}`
+                  )
+                  return
+                }
+                 
+                await translateExercise(
+                  rigoToken,
+                  {
+                    text_to_translate: readme.toString(),
+                    output_language: language,
+                  },
+                  `${process.env.HOST}/webhooks/${courseSlug}/${slug}/save-translation`
                 )
-                await translatedReadme.save(response.parsed.translation)
-
-                languageCodes.add(response.parsed.output_language_code)
               })
             )
           })
         )
 
-        const currentLanguages = Object.keys(courseJson.title)
-        for (const languageCode of currentLanguages) {
-          if (languageCodes.has(languageCode)) {
-            languageCodes.delete(languageCode)
+        const course = await bucket
+          .file(`courses/${courseSlug}/learn.json`)
+          .download()
+
+        const courseJson = JSON.parse(course.toString())
+
+        const neededLanguages = new Set<string>()
+
+        let currentLanguages = Object.keys(courseJson.title)
+
+        for (const languageCode of languageCodes) {
+          if (!currentLanguages.includes(languageCode)) {
+            neededLanguages.add(languageCode)
           }
         }
 
-        if ([...languageCodes].length > 0) {
-          const translatedCourseMetadata = await Promise.all(
-            [...languageCodes].map(async languageCode => {
-              const result = await translateCourseMetadata(rigoToken, {
-                title: courseJson.title[currentLanguage],
-                description: courseJson.description[currentLanguage],
-                destination_lang_code: languageCode as string,
-              })
+        const neededLanguagesList: string[] = [...neededLanguages]
+          .map((lang: string) => lang.toLowerCase())
+          .sort()
 
-              return {
-                languageCode,
-                title: result.parsed.title,
-                description: result.parsed.description,
-              }
-            })
-          )
+        if (neededLanguagesList.length > 0) {
+          const result = await translateCourseMetadata(rigoToken, {
+            title: JSON.stringify(courseJson.title),
+            description: JSON.stringify(courseJson.description),
+            new_languages: neededLanguagesList.join(","),
+          })
 
-          for (const metadata of translatedCourseMetadata) {
-            courseJson.title[metadata.languageCode as string] = metadata.title
-            courseJson.description[metadata.languageCode as string] =
-              metadata.description
-          }
+          const translateTitle = JSON.parse(result.parsed.title)
+          const translateDescription = JSON.parse(result.parsed.description)
+
+          courseJson.title = translateTitle
+          courseJson.description = translateDescription
 
           await uploadFileToBucket(
             bucket,
@@ -1022,34 +1062,45 @@ export default class ServeCommand extends SessionCommand {
             `courses/${courseSlug}/learn.json`
           )
 
+          currentLanguages = Object.keys(courseJson.title)
+        }
+
+        // Check that all the READMEs exists
+
+        const missingReadmeTranslations = []
+        let firstAvailable = ""
+        for (const languageCode of currentLanguages) {
+          // eslint-disable-next-line no-await-in-loop
           const previewReadme = await bucket.file(
-            `courses/${courseSlug}/README${getReadmeExtension(currentLanguage)}`
+            `courses/${courseSlug}/README${getReadmeExtension(languageCode)}`
           )
-          const [previewReadmeContent] = await previewReadme.download()
-          const previewReadmeContentString = previewReadmeContent.toString()
-
-          await Promise.all(
-            [...languageCodes].map(async languageCode => {
-              const translatedPreviewReadme = await translateExercise(
-                rigoToken,
-                {
-                  text_to_translate: previewReadmeContentString,
-                  output_language: languageCode as string,
-                  exercise_slug: "preview-readme",
-                }
-              )
 
-              await bucket
-                .file(
-                  `courses/${courseSlug}/README${getReadmeExtension(
-                    languageCode as string
-                  )}`
-                )
-                .save(translatedPreviewReadme.parsed.translation)
-            })
-          )
+          // eslint-disable-next-line no-await-in-loop
+          const [exists] = await previewReadme.exists()
+          if (!exists) {
+            missingReadmeTranslations.push(languageCode)
+          } else {
+            // eslint-disable-next-line no-await-in-loop
+            const [previewReadmeContent] = await previewReadme.download()
+            const previewReadmeContentString = previewReadmeContent.toString()
+            firstAvailable = previewReadmeContentString
+          }
         }
 
+        await Promise.all(
+          missingReadmeTranslations.map(async languageCode => {
+             
+            await translateExercise(
+              rigoToken,
+              {
+                text_to_translate: firstAvailable,
+                output_language: languageCode as string,
+              },
+              `${process.env.HOST}/webhooks/${courseSlug}/initial-readme-processor`
+            )
+          })
+        )
+
         return res.status(200).json({ message: "Translated exercises" })
       } catch (error) {
         console.log(error, "ERROR")

package/src/commands/translate.ts

@@ -96,11 +96,14 @@ export default class BuildCommand extends SessionCommand {
             .split(",")
             .map(async (language: string) => {
               const readme = await getReadmeForExercise(exercise)
-              const response = await translateExercise(rigoToken, {
-                text_to_translate: readme,
-                output_language: language,
-                exercise_slug: exercise,
-              })
+              const response = await translateExercise(
+                rigoToken,
+                {
+                  text_to_translate: readme,
+                  output_language: language,
+                },
+                `${process.env.HOST}/webhooks/translate-exercise`
+              )
 
               await saveTranslatedReadme(
                 exercise,

package/src/managers/server/routes.ts

@@ -204,8 +204,7 @@ export default async function (
   // symbolic link to maintain path compatiblity
   const fetchStaticAsset = withHandler((req, res) => {
     const filePath = `${config?.dirPath}/assets/${req.params.filePath}`
-    if (!fs.existsSync(filePath)) 
-throw new Error("File not found: " + filePath)
+    if (!fs.existsSync(filePath)) throw new Error("File not found: " + filePath)
     const content = fs.readFileSync(filePath)
     res.write(content)
     res.end()
@@ -436,11 +435,14 @@ throw new Error("File not found: " + filePath)
 
               await Promise.all(
                 languagesToTranslate.map(async (language: string) => {
-                  const response = await translateExercise(rigoToken, {
-                    text_to_translate: readme.body,
-                    output_language: language,
-                    exercise_slug: slug,
-                  })
+                  const response = await translateExercise(
+                    rigoToken,
+                    {
+                      text_to_translate: readme.body,
+                      output_language: language,
+                    },
+                    `${process.env.HOST}/webhooks/translate-exercise`
+                  )
 
                   await saveTranslatedReadme(
                     slug,