@learnpack/learnpack 5.0.272 → 5.0.275

package/src/commands/serve.ts

@@ -44,7 +44,7 @@ import {
 // import { handleAssetCreation } from "./publish"
 import axios from "axios"
 import * as FormData from "form-data"
-import api, { RIGOBOT_HOST } from "../utils/api"
+import api, { RIGOBOT_HOST, RIGOBOT_REALTIME_HOST } from "../utils/api"
 import { createUploadMiddleware, minutesToISO8601Duration } from "../utils/misc"
 import { buildConfig, ConfigResponse } from "../utils/configBuilder"
 import { checkReadability, slugify } from "../utils/creatorUtilities"
@@ -125,7 +125,6 @@ export const processImage = async (
       callbackUrl: webhookUrl,
     })
 
-    // console.log("✅ Image", imagePath, "generated successfully!")
     return true
   } catch {
     return false
@@ -157,11 +156,9 @@ const uploadInitialReadme = async (
   :rigo
   \`\`\`
       `
-  const readmeFilename = `README.${
-    packageContext.language && packageContext.language !== "en" ?
-      `${packageContext.language}.` :
-      ""
-  }md`
+  const readmeFilename = `README${getReadmeExtension(
+    packageContext.language || "en"
+  )}`
 
   await uploadFileToBucket(
     bucket,
@@ -229,16 +226,14 @@ const createMultiLangAsset = async (
 }
 
 async function startExerciseGeneration(
-  bucket: Bucket,
   rigoToken: string,
   steps: Lesson[],
   packageContext: FormState,
   exercise: Lesson,
-  tutorialDir: string,
   courseSlug: string,
   purposeSlug: string,
   lastLesson = ""
-): Promise<void> {
+): Promise<number> {
   const exSlug = slugify(exercise.id + "-" + exercise.title)
   console.log("Starting generation of", exSlug)
 
@@ -246,7 +241,7 @@ async function startExerciseGeneration(
 
   console.log("WEBHOOK URL", webhookUrl)
 
-  await readmeCreator(
+  const res = await readmeCreator(
     rigoToken,
     {
       title: `${exercise.id} - ${exercise.title}`,
@@ -262,6 +257,9 @@ async function startExerciseGeneration(
     purposeSlug,
     webhookUrl
   )
+
+  console.log("README CREATOR RES", res)
+  return res.id
 }
 
 async function createInitialReadme(
@@ -648,13 +646,11 @@ export default class ServeCommand extends SessionCommand {
           }
         }
 
-        await startExerciseGeneration(
-          bucket,
+        const completionId = await startExerciseGeneration(
           rigoToken,
           syllabusJson.lessons,
           syllabusJson.courseInfo,
           exercise,
-          `courses/${courseSlug}`,
           courseSlug,
           syllabusJson.courseInfo.purpose,
           previousReadme +
@@ -663,6 +659,14 @@ export default class ServeCommand extends SessionCommand {
         )
 
         syllabusJson.lessons[parseInt(position)].status = "GENERATING"
+        syllabusJson.lessons[parseInt(position)].translations = {
+          [syllabusJson.courseInfo.language || "en"]: {
+            completionId,
+            startedAt: Date.now(),
+            completedAt: 0,
+          },
+        }
+        console.log("Lesson", syllabusJson.lessons[parseInt(position)])
         if (
           syllabusJson.feedback &&
           typeof syllabusJson.feedback === "string"
@@ -682,6 +686,27 @@ export default class ServeCommand extends SessionCommand {
       }
     )
 
+    app.post("/actions/generate-image/:courseSlug", async (req, res) => {
+      const rigoToken = req.header("x-rigo-token")
+      const { courseSlug } = req.params
+      const { image } = req.body
+
+      if (!image) {
+        return res.status(400).json({
+          error: "Image is required",
+        })
+      }
+
+      if (!rigoToken) {
+        return res.status(400).json({
+          error: "Rigo token is required. x-rigo-token header is missing",
+        })
+      }
+
+      await processImage(image.url, image.alt, rigoToken, courseSlug)
+      res.json({ status: "QUEUED" })
+    })
+
     app.post(
       "/webhooks/:courseSlug/exercise-processor/:lessonID/:rigoToken",
       async (req, res) => {
@@ -773,7 +798,7 @@ export default class ServeCommand extends SessionCommand {
           })
         }
 
-        let nextStarted = false
+        let nextCompletionId: number | null = null
         if (
           nextExercise &&
           (exerciseIndex === 0 || !(exerciseIndex % 3 === 0))
@@ -783,18 +808,15 @@ export default class ServeCommand extends SessionCommand {
             feedback = `\n\nThe user added the following feedback with relation to the previous generations: ${syllabusJson.feedback}`
           }
 
-          startExerciseGeneration(
-            bucket,
+          nextCompletionId = await startExerciseGeneration(
             rigoToken,
             syllabusJson.lessons,
             syllabusJson.courseInfo,
             nextExercise,
-            `courses/${courseSlug}`,
             courseSlug,
             syllabusJson.courseInfo.purpose,
             readme.parsed.content + "\n\n" + feedback
           )
-          nextStarted = true
         } else {
           console.log(
             "Stopping generation process at",
@@ -808,32 +830,60 @@ export default class ServeCommand extends SessionCommand {
           readability.newMarkdown
         )
 
-        if (imagesArray.length > 0) {
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: exSlug,
-            status: "pending",
-            log: `🔄 Generating images for ${exercise.title}`,
-          })
-          for (const image of imagesArray) {
-            // eslint-disable-next-line no-await-in-loop
-            await processImage(image.url, image.alt, rigoToken, courseSlug)
-          }
-        }
-
         const newSyllabus = {
           ...syllabusJson,
           lessons: syllabusJson.lessons.map((lesson, index) => {
             if (index === exerciseIndex) {
-              return { ...lesson, generated: true, status: "DONE" }
+              const currentTranslations = lesson.translations || {}
+              let currentTranslation =
+                currentTranslations[syllabusJson.courseInfo.language || "en"]
+              if (currentTranslation) {
+                currentTranslation.completedAt = Date.now()
+              } else {
+                currentTranslation = {
+                  completionId: readme.id,
+                  startedAt: Date.now(),
+                  completedAt: Date.now(),
+                }
+              }
+
+              currentTranslations[syllabusJson.courseInfo.language || "en"] =
+                currentTranslation
+              return {
+                ...lesson,
+                generated: true,
+                status: "DONE",
+                translations: {
+                  [syllabusJson.courseInfo.language || "en"]: {
+                    completionId: nextCompletionId,
+                    completedAt: Date.now(),
+                  },
+                },
+              }
             }
 
-            if (nextExercise && nextExercise.id === lesson.id && nextStarted) {
-              return { ...lesson, generated: false, status: "GENERATING" }
+            if (
+              nextExercise &&
+              nextExercise.id === lesson.id &&
+              nextCompletionId
+            ) {
+              return {
+                ...lesson,
+                generated: false,
+                status: "GENERATING",
+                translations: {
+                  [syllabusJson.courseInfo.language || "en"]: {
+                    completionId: nextCompletionId,
+                    startedAt: Date.now(),
+                  },
+                },
+              }
             }
 
             return { ...lesson }
           }),
         }
+        console.log("New syllabus", newSyllabus)
         await uploadFileToBucket(
           bucket,
           JSON.stringify(newSyllabus),
@@ -1444,17 +1494,24 @@ export default class ServeCommand extends SessionCommand {
 
       const lastResult = "---"
 
-      await startExerciseGeneration(
-        bucket,
+      const completionId = await startExerciseGeneration(
         rigoToken,
         syllabus.lessons,
         syllabus.courseInfo,
         firstLesson,
-        tutorialDir,
         courseSlug,
         syllabus.courseInfo.purpose,
         lastResult
       )
+      if (firstLesson) {
+        firstLesson.translations = {
+          [syllabus.courseInfo.language || "en"]: {
+            completionId,
+            startedAt: Date.now(),
+            completedAt: 0,
+          },
+        }
+      }
 
       await createInitialReadme(
         JSON.stringify(syllabus.courseInfo),
@@ -1484,54 +1541,52 @@ export default class ServeCommand extends SessionCommand {
       }
     })
 
-    app.post("/actions/continue-course/:courseSlug", async (req, res) => {
-      console.log("POST /actions/continue-course/:courseSlug")
-      const { courseSlug } = req.params
-
-      const { feedback }: { feedback: string } = req.body
-
-      const rigoToken = req.header("x-rigo-token")
-      const bcToken = req.header("x-breathecode-token")
-      if (!rigoToken || !bcToken) {
-        return res.status(400).json({ error: "Missing tokens" })
-      }
-
-      const syllabus = await bucket.file(
-        `courses/${courseSlug}/.learn/initialSyllabus.json`
-      )
-      const [content] = await syllabus.download()
-      const syllabusJson: Syllabus = JSON.parse(content.toString())
-      const notGeneratedLessons = syllabusJson.lessons.filter(
-        lesson => !lesson.generated
-      )
-
-      const lastGeneratedLesson = findLast(
-        syllabusJson.lessons,
-        lesson => lesson.generated ?? false
-      )
-
-      console.log("ABout to generate", notGeneratedLessons.length, "lessons")
-
-      const firstLessonToGenerate = notGeneratedLessons[0]
-
-      await startExerciseGeneration(
-        bucket,
-        rigoToken,
-        syllabusJson.lessons,
-        syllabusJson.courseInfo,
-        firstLessonToGenerate,
-        `courses/${courseSlug}`,
-        courseSlug,
-        syllabusJson.courseInfo.purpose,
-        JSON.stringify(lastGeneratedLesson) +
-          `\n\nThe user provided this feedback in relation to the course: ${feedback}`
-      )
-
-      return res.json({
-        message: "Course continued",
-        slug: courseSlug,
-      })
-    })
+    // app.post("/actions/continue-course/:courseSlug", async (req, res) => {
+    //   console.log("POST /actions/continue-course/:courseSlug")
+    //   const { courseSlug } = req.params
+
+    //   const { feedback }: { feedback: string } = req.body
+
+    //   const rigoToken = req.header("x-rigo-token")
+    //   const bcToken = req.header("x-breathecode-token")
+    //   if (!rigoToken || !bcToken) {
+    //     return res.status(400).json({ error: "Missing tokens" })
+    //   }
+
+    //   const syllabus = await bucket.file(
+    //     `courses/${courseSlug}/.learn/initialSyllabus.json`
+    //   )
+    //   const [content] = await syllabus.download()
+    //   const syllabusJson: Syllabus = JSON.parse(content.toString())
+    //   const notGeneratedLessons = syllabusJson.lessons.filter(
+    //     lesson => !lesson.generated
+    //   )
+
+    //   const lastGeneratedLesson = findLast(
+    //     syllabusJson.lessons,
+    //     lesson => lesson.generated ?? false
+    //   )
+
+    //   console.log("ABout to generate", notGeneratedLessons.length, "lessons")
+
+    //   const firstLessonToGenerate = notGeneratedLessons[0]
+
+    //   const completionId = await startExerciseGeneration(
+    //     rigoToken,
+    //     syllabusJson.lessons,
+    //     syllabusJson.courseInfo,
+    //     firstLessonToGenerate,
+    //     courseSlug,
+    //     syllabusJson.courseInfo.purpose,
+    //     JSON.stringify(lastGeneratedLesson) +
+    //       `\n\nThe user provided this feedback in relation to the course: ${feedback}`
+    //   )
+
+    //   return res.json({
+    //     message: "Course continued",
+    //     slug: courseSlug,
+    //   })
+    // })
 
     app.get(
       "/courses/:courseSlug/exercises/:exerciseSlug/",
@@ -1753,9 +1808,9 @@ export default class ServeCommand extends SessionCommand {
           return res.send(content.toString("utf-8"))
         } catch (error) {
           console.error("❌ Error fetching file:", error)
-          return res
-            .status(500)
-            .json({ error: (error as Error).message || "Unable to fetch file" })
+          return res.status(500).json({
+            error: (error as Error).message || "Unable to fetch file",
+          })
         }
       }
     )
@@ -1771,26 +1826,36 @@ export default class ServeCommand extends SessionCommand {
 
         if (ytMatch) {
           const videoId = ytMatch[1]
-          // fetch metadata
-          const items = await YoutubeTranscript.fetchTranscript(videoId)
-          const transcript = items.map(i => i.text).join(" ")
-
-          const { data: meta } = await axios.get(
-            "https://www.youtube.com/oembed",
-            {
-              params: { url: decoded, format: "json" },
-            }
+          const resFromRigo = await axios.get(
+            `${RIGOBOT_REALTIME_HOST}/actions/youtube-transcript/${videoId}`
           )
+          console.log("RES FROM RIGO", resFromRigo.data)
+          const transcript = resFromRigo.data.transcript
+
+          // let meta: any = null
+          // try {
+          //   const { data: meta } = await axios.get(
+          //     "https://www.youtube.com/oembed",
+          //     {
+          //       params: { url: decoded, format: "json" },
+          //     }
+          //   )
+          //   console.log("META", meta)
+          // } catch (error) {
+          //   console.error("ERROR FETCHING META", error)
+          //   meta = null
+          // }
 
           return res.json({
             url: decoded,
-            title: meta.title,
-            author: meta.author_name,
-            thumbnail: meta.thumbnail_url,
+            title: resFromRigo.data.title || null,
+            author: resFromRigo.data.author || null,
             transcript,
           })
         }
 
+        console.log("NOT A YOUTUBE LINK", decoded)
+
         const response = await axios.get(decoded, { responseType: "text" })
         const html = response.data as string
         const title = getTitleFromHTML(html)