@learnpack/learnpack 5.0.290 → 5.0.292

package/src/commands/serve.ts

@@ -34,6 +34,8 @@ import {
   createStructuredPreviewReadme,
   translateCourseMetadata,
   getLanguageCodes,
+  initialContentGenerator,
+  addInteractivity,
 } from "../utils/rigoActions"
 import * as dotenv from "dotenv"
 
@@ -119,6 +121,38 @@ const PARAMS = {
   max_title_length: 50,
 }
 
+async function fetchComponentsYml(): Promise<string> {
+  try {
+    const [assessmentResponse, explanatoryResponse] = await Promise.all([
+      axios.get(
+        "https://raw.githubusercontent.com/learnpack/ide/refs/heads/master/docs/assessment_components.yml"
+      ),
+      axios.get(
+        "https://raw.githubusercontent.com/learnpack/ide/refs/heads/master/docs/explanatory_components.yml"
+      ),
+    ])
+
+    const combinedContent = `
+# ASSESSMENT COMPONENTS
+These components are designed for evaluation and knowledge assessment:
+
+${assessmentResponse.data}
+
+---
+
+# EXPLANATORY COMPONENTS  
+These components are designed for explanation and learning support:
+
+${explanatoryResponse.data}
+`
+
+    return combinedContent
+  } catch (error) {
+    console.error("Failed to fetch components YAML files:", error)
+    return ""
+  }
+}
+
 export const processImage = async (
   url: string,
   description: string,
@@ -193,6 +227,21 @@ const cleanFormState = (formState: FormState) => {
   return rest
 }
 
+const cleanFormStateForSyllabus = (formState: FormState) => {
+  return {
+    ...formState,
+    description: formState.description,
+    technologies: formState.technologies,
+    purposse: undefined,
+    duration: undefined,
+    hasContentIndex: undefined,
+    variables: undefined,
+    currentStep: undefined,
+    language: undefined,
+    isCompleted: undefined,
+  }
+}
+
 const createMultiLangAsset = async (
   bucket: Bucket,
   rigoToken: string,
@@ -211,9 +260,17 @@ const createMultiLangAsset = async (
     const indexReadme = await bucket.file(
       `courses/${courseSlug}/README${getReadmeExtension(lang)}`
     )
-    // eslint-disable-next-line no-await-in-loop
-    const [indexReadmeContent] = await indexReadme.download()
-    const indexReadmeString = indexReadmeContent.toString()
+    let indexReadmeString = ""
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      const [indexReadmeContent] = await indexReadme.download()
+      indexReadmeString = indexReadmeContent.toString()
+    } catch (error) {
+      console.error("Error downloading index readme", error)
+      // TODO: Trigger generation of the index readme
+      indexReadmeString = ""
+    }
+
     const b64IndexReadme = Buffer.from(indexReadmeString).toString("base64")
 
     // eslint-disable-next-line no-await-in-loop
@@ -249,8 +306,6 @@ async function startExerciseGeneration(
 
   const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/exercise-processor/${exercise.id}/${rigoToken}`
 
-  console.log("WEBHOOK URL", webhookUrl)
-
   const res = await readmeCreator(
     rigoToken.trim(),
     {
@@ -272,6 +327,118 @@ async function startExerciseGeneration(
   return res.id
 }
 
+const lessonCleaner = (lesson: Lesson) => {
+  return {
+    ...lesson,
+    description: lesson.description,
+    duration: undefined,
+    generated: undefined,
+    status: undefined,
+    translations: undefined,
+    uid: undefined,
+  }
+}
+
+async function startInitialContentGeneration(
+  rigoToken: string,
+  steps: Lesson[],
+  packageContext: FormState,
+  exercise: Lesson,
+  courseSlug: string,
+  purposeSlug: string,
+  lastLesson = ""
+): Promise<number> {
+  const exSlug = slugify(exercise.id + "-" + exercise.title)
+  console.log("Starting initial content generation for", exSlug)
+
+  const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/initial-content-processor/${exercise.id}/${rigoToken}`
+
+  // Emit notification that initial content generation is starting
+  emitToCourse(courseSlug, "course-creation", {
+    lesson: exSlug,
+    status: "generating",
+    log: `🔄 Starting initial content generation for lesson: ${exercise.title}`,
+  })
+
+  const fullSyllabus = {
+    steps: [steps.map(lessonCleaner)],
+    courseInfo: cleanFormStateForSyllabus(packageContext),
+    purpose: purposeSlug,
+  }
+
+  // Add random 6-digit number to avoid cache issues
+  // eslint-disable-next-line no-mixed-operators
+  const randomCacheEvict = Math.floor(100_000 + Math.random() * 900_000)
+
+  const res = await initialContentGenerator(
+    rigoToken.trim(),
+    {
+      // prev_lesson: lastLesson,
+      output_language: packageContext.language || "en",
+      current_syllabus: JSON.stringify(fullSyllabus),
+      lesson_description:
+        JSON.stringify(lessonCleaner(exercise)) + `-${randomCacheEvict}`,
+    },
+    webhookUrl
+  )
+
+  console.log("INITIAL CONTENT GENERATOR RES", res)
+  return res.id
+}
+
+async function startInteractivityGeneration(
+  rigoToken: string,
+  steps: Lesson[],
+  packageContext: FormState,
+  exercise: Lesson,
+  courseSlug: string,
+  purposeSlug: string,
+  bucket: Bucket,
+  lastLesson = ""
+): Promise<number> {
+  const exSlug = slugify(exercise.id + "-" + exercise.title)
+  console.log("Starting interactivity generation for", exSlug)
+
+  const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/interactivity-processor/${exercise.id}/${rigoToken}`
+
+  // Emit notification that interactivity generation is starting
+  emitToCourse(courseSlug, "course-creation", {
+    lesson: exSlug,
+    status: "generating",
+    log: `🔄 Starting interactivity generation for lesson: ${exercise.title}`,
+  })
+
+  const componentsYml = await fetchComponentsYml()
+
+  // Get current syllabus to include used_components
+  const currentSyllabus = await getSyllabus(courseSlug, bucket)
+
+  const fullSyllabus = {
+    steps: steps.map(lessonCleaner),
+    courseInfo: cleanFormStateForSyllabus(packageContext),
+    used_components: currentSyllabus.used_components || {},
+  }
+
+  // Add random 6-digit number to avoid cache issues
+  // eslint-disable-next-line no-mixed-operators
+  const randomCacheEvict = Math.floor(100_000 + Math.random() * 900_000)
+
+  const res = await addInteractivity(
+    rigoToken.trim(),
+    {
+      components: componentsYml,
+      prev_lesson: lastLesson,
+      initial_lesson: exercise.initialContent + `-${randomCacheEvict}`,
+      output_language: packageContext.language || "en",
+      current_syllabus: JSON.stringify(fullSyllabus),
+    },
+    webhookUrl
+  )
+
+  console.log("INTERACTIVITY GENERATOR RES", res)
+  return res.id
+}
+
 async function createInitialReadme(
   tutorialInfo: string,
   tutorialSlug: string,
@@ -293,6 +460,219 @@ async function createInitialReadme(
   }
 }
 
+async function getSyllabus(
+  courseSlug: string,
+  bucket: Bucket
+): Promise<Syllabus> {
+  const syllabus = await bucket.file(
+    `courses/${courseSlug}/.learn/initialSyllabus.json`
+  )
+  const [content] = await syllabus.download()
+  return JSON.parse(content.toString())
+}
+
+async function saveSyllabus(
+  courseSlug: string,
+  syllabus: Syllabus,
+  bucket: Bucket
+): Promise<void> {
+  await uploadFileToBucket(
+    bucket,
+    JSON.stringify(syllabus),
+    `courses/${courseSlug}/.learn/initialSyllabus.json`
+  )
+}
+
+async function updateUsedComponents(
+  courseSlug: string,
+  usedComponents: string[],
+  bucket: Bucket
+): Promise<void> {
+  const syllabus = await getSyllabus(courseSlug, bucket)
+
+  // Initialize used_components if undefined
+  if (!syllabus.used_components) {
+    syllabus.used_components = {}
+  }
+
+  // Update component counts
+  for (const componentName of usedComponents) {
+    if (syllabus.used_components[componentName]) {
+      syllabus.used_components[componentName] += 1
+    } else {
+      syllabus.used_components[componentName] = 1
+    }
+  }
+
+  console.log("Updated component usage:", syllabus.used_components)
+  await saveSyllabus(courseSlug, syllabus, bucket)
+}
+
+async function updateLessonWithInitialContent(
+  courseSlug: string,
+  lessonID: string,
+  initialResponse: any,
+  bucket: Bucket
+): Promise<void> {
+  const syllabus = await getSyllabus(courseSlug, bucket)
+  const lessonIndex = syllabus.lessons.findIndex(
+    lesson => lesson.id === lessonID
+  )
+
+  if (lessonIndex === -1) {
+    console.error(`Lesson ${lessonID} not found in syllabus`)
+    return
+  }
+
+  const lesson = syllabus.lessons[lessonIndex]
+
+  // Update initial content
+  lesson.initialContent = initialResponse.lesson_content
+
+  await saveSyllabus(courseSlug, syllabus, bucket)
+}
+
+async function updateLessonStatusToError(
+  courseSlug: string,
+  lessonID: string,
+  bucket: Bucket
+): Promise<void> {
+  try {
+    const syllabus = await getSyllabus(courseSlug, bucket)
+    const lessonIndex = syllabus.lessons.findIndex(
+      lesson => lesson.id === lessonID
+    )
+
+    if (lessonIndex === -1) {
+      console.error(
+        `Lesson ${lessonID} not found in syllabus when updating status to error`
+      )
+      return
+    }
+
+    const lesson = syllabus.lessons[lessonIndex]
+
+    // Update lesson status to ERROR
+    lesson.status = "ERROR"
+
+    // Update translations to mark as failed
+    const currentTranslations = lesson.translations || {}
+    const language = syllabus.courseInfo.language || "en"
+
+    if (currentTranslations[language]) {
+      currentTranslations[language].completedAt = Date.now()
+    } else {
+      currentTranslations[language] = {
+        completionId: 0,
+        startedAt: Date.now(),
+        completedAt: Date.now(),
+      }
+    }
+
+    lesson.translations = currentTranslations
+
+    await saveSyllabus(courseSlug, syllabus, bucket)
+    console.log(`Updated lesson ${lessonID} status to ERROR in syllabus`)
+  } catch (error) {
+    console.error(`Error updating lesson ${lessonID} status to ERROR:`, error)
+  }
+}
+
+async function continueWithNextLesson(
+  courseSlug: string,
+  currentExerciseIndex: number,
+  rigoToken: string,
+  finalContent: string,
+  bucket: Bucket
+): Promise<void> {
+  const syllabus = await getSyllabus(courseSlug, bucket)
+  const nextExercise = syllabus.lessons[currentExerciseIndex + 1] || null
+
+  let nextCompletionId: number | null = null
+  if (
+    nextExercise &&
+    (currentExerciseIndex === 0 ||
+      !(currentExerciseIndex % 3 === 0) ||
+      syllabus.generationMode === "continue-with-all")
+  ) {
+    let feedback = ""
+    if (syllabus.feedback) {
+      feedback = `\n\nThe user added the following feedback with relation to the previous generations: ${syllabus.feedback}`
+    }
+
+    nextCompletionId = await startInitialContentGeneration(
+      rigoToken,
+      syllabus.lessons,
+      syllabus.courseInfo,
+      nextExercise,
+      courseSlug,
+      syllabus.courseInfo.purpose,
+      finalContent + "\n\n" + feedback
+    )
+  } else {
+    console.log(
+      "Stopping generation process at",
+      currentExerciseIndex,
+      syllabus.lessons[currentExerciseIndex].title,
+      "because it's a multiple of 3"
+    )
+  }
+
+  // Update syllabus with next lesson status
+  const newSyllabus = {
+    ...syllabus,
+    lessons: syllabus.lessons.map((lesson, index) => {
+      if (index === currentExerciseIndex) {
+        const currentTranslations = lesson.translations || {}
+        let currentTranslation =
+          currentTranslations[syllabus.courseInfo.language || "en"]
+        if (currentTranslation) {
+          currentTranslation.completedAt = Date.now()
+        } else {
+          currentTranslation = {
+            completionId: nextCompletionId || 0,
+            startedAt: Date.now(),
+            completedAt: Date.now(),
+          }
+        }
+
+        currentTranslations[syllabus.courseInfo.language || "en"] =
+          currentTranslation
+        return {
+          ...lesson,
+          generated: true,
+          status: "DONE" as const,
+          translations: {
+            [syllabus.courseInfo.language || "en"]: {
+              completionId: nextCompletionId || 0,
+              startedAt: currentTranslation.startedAt,
+              completedAt: Date.now(),
+            },
+          },
+        }
+      }
+
+      if (nextExercise && nextExercise.id === lesson.id && nextCompletionId) {
+        return {
+          ...lesson,
+          generated: false,
+          status: "GENERATING" as const,
+          translations: {
+            [syllabus.courseInfo.language || "en"]: {
+              completionId: nextCompletionId,
+              startedAt: Date.now(),
+            },
+          },
+        }
+      }
+
+      return { ...lesson }
+    }),
+  }
+
+  await saveSyllabus(courseSlug, newSyllabus, bucket)
+}
+
 const fixPreviewUrl = (slug: string, previewUrl: string) => {
   if (!previewUrl) {
     return null
@@ -627,17 +1007,13 @@ export default class ServeCommand extends SessionCommand {
           })
         }
 
-        const syllabus = await bucket.file(
-          `courses/${courseSlug}/.learn/initialSyllabus.json`
-        )
-        const [content] = await syllabus.download()
-        const syllabusJson: Syllabus = JSON.parse(content.toString())
+        const syllabus = await getSyllabus(courseSlug, bucket)
 
-        const exercise = syllabusJson.lessons[parseInt(position)]
+        const exercise = syllabus.lessons[parseInt(position)]
 
         // previous exercise
         let previousReadme = "---"
-        const previousExercise = syllabusJson.lessons[parseInt(position) - 1]
+        const previousExercise = syllabus.lessons[parseInt(position) - 1]
         if (previousExercise) {
           // Get the readme of the previous exercise
           const exSlug = slugify(
@@ -656,43 +1032,37 @@ export default class ServeCommand extends SessionCommand {
           }
         }
 
-        const completionId = await startExerciseGeneration(
+        // Use new two-phase generation workflow
+        const completionId = await startInitialContentGeneration(
           rigoToken,
-          syllabusJson.lessons,
-          syllabusJson.courseInfo,
+          syllabus.lessons,
+          syllabus.courseInfo,
           exercise,
           courseSlug,
-          syllabusJson.courseInfo.purpose,
+          syllabus.courseInfo.purpose,
           previousReadme +
             "\n\nThe user provided the following feedback related to the content of the course so far: \n\n" +
             feedback
         )
 
-        syllabusJson.lessons[parseInt(position)].status = "GENERATING"
-        syllabusJson.lessons[parseInt(position)].translations = {
-          [syllabusJson.courseInfo.language || "en"]: {
+        syllabus.lessons[parseInt(position)].status = "GENERATING"
+        syllabus.lessons[parseInt(position)].translations = {
+          [syllabus.courseInfo.language || "en"]: {
             completionId,
             startedAt: Date.now(),
             completedAt: 0,
           },
         }
-        console.log("Lesson", syllabusJson.lessons[parseInt(position)])
-        if (
-          syllabusJson.feedback &&
-          typeof syllabusJson.feedback === "string"
-        ) {
-          syllabusJson.feedback += "\n\n" + feedback
+        console.log("Lesson", syllabus.lessons[parseInt(position)])
+        if (syllabus.feedback && typeof syllabus.feedback === "string") {
+          syllabus.feedback += "\n\n" + feedback
         } else {
-          syllabusJson.feedback = feedback
+          syllabus.feedback = feedback
         }
 
-        syllabusJson.generationMode = mode
+        syllabus.generationMode = mode
 
-        await uploadFileToBucket(
-          bucket,
-          JSON.stringify(syllabusJson),
-          `courses/${courseSlug}/.learn/initialSyllabus.json`
-        )
+        await saveSyllabus(courseSlug, syllabus, bucket)
 
         res.json({ status: "SUCCESS" })
       }
@@ -927,6 +1297,400 @@ export default class ServeCommand extends SessionCommand {
       }
     )
 
+    // Phase 1: Initial content generation webhook
+    app.post(
+      "/webhooks/:courseSlug/initial-content-processor/:lessonID/:rigoToken",
+      async (req, res) => {
+        const { courseSlug, lessonID, rigoToken } = req.params
+        const response = req.body
+
+        console.log("RECEIVING INITIAL CONTENT WEBHOOK", response)
+
+        // Handle errors
+        if (response.status === "ERROR") {
+          await updateLessonStatusToError(courseSlug, lessonID, bucket)
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: lessonID,
+            status: "error",
+            log: `❌ Error generating initial content for lesson ${lessonID}`,
+          })
+
+          // Retry initial content generation
+          try {
+            const syllabus = await getSyllabus(courseSlug, bucket)
+            const lessonIndex = syllabus.lessons.findIndex(
+              lesson => lesson.id === lessonID
+            )
+            const exercise = syllabus.lessons[lessonIndex]
+
+            if (exercise) {
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: lessonID,
+                status: "generating",
+                log: `🔄 Retrying initial content generation for lesson ${lessonID}`,
+              })
+
+              const retryCompletionId = await startInitialContentGeneration(
+                rigoToken,
+                syllabus.lessons,
+                syllabus.courseInfo,
+                exercise,
+                courseSlug,
+                syllabus.courseInfo.purpose,
+                ""
+              )
+
+              // Update lesson status to show it's retrying
+              exercise.status = "GENERATING"
+              exercise.translations = {
+                [syllabus.courseInfo.language || "en"]: {
+                  completionId: retryCompletionId,
+                  startedAt: Date.now(),
+                  completedAt: 0,
+                },
+              }
+              await saveSyllabus(courseSlug, syllabus, bucket)
+            }
+          } catch (retryError) {
+            console.error(
+              "Error retrying initial content generation:",
+              retryError
+            )
+            emitToCourse(courseSlug, "course-creation", {
+              lesson: lessonID,
+              status: "error",
+              log: `❌ Failed to retry initial content generation for lesson ${lessonID}`,
+            })
+          }
+
+          return res.json({ status: "ERROR" })
+        }
+
+        try {
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: lessonID,
+            status: "generating",
+            log: `✅ Initial content generated for lesson ${lessonID}`,
+          })
+
+          // Update lesson with initial content
+          await updateLessonWithInitialContent(
+            courseSlug,
+            lessonID,
+            response.parsed,
+            bucket
+          )
+
+          // Start Phase 2: Add interactivity
+          const syllabus = await getSyllabus(courseSlug, bucket)
+          const lessonIndex = syllabus.lessons.findIndex(
+            lesson => lesson.id === lessonID
+          )
+          const exercise = syllabus.lessons[lessonIndex]
+
+          let lastLesson = ""
+
+          const prevLessonIndex = lessonIndex - 1
+          if (prevLessonIndex >= 0) {
+            try {
+              const prevLesson = syllabus.lessons[prevLessonIndex]
+              // search the lesson content in the bucket
+              const prevLessonSlug = slugify(
+                prevLesson.id + "-" + prevLesson.title
+              )
+              const file = bucket.file(
+                `courses/${courseSlug}/exercises/${prevLessonSlug}/README${getReadmeExtension(
+                  syllabus.courseInfo.language || "en"
+                )}`
+              )
+              const [content] = await file.download()
+              lastLesson = content.toString()
+            } catch (error) {
+              console.error("Error searching previous lesson content:", error)
+            }
+          }
+
+          if (exercise) {
+            const completionId = await startInteractivityGeneration(
+              rigoToken,
+              syllabus.lessons,
+              syllabus.courseInfo,
+              exercise,
+              courseSlug,
+              syllabus.courseInfo.purpose,
+              bucket,
+              lastLesson
+            )
+
+            // Update lesson status to show it's in Phase 2
+            exercise.status = "GENERATING"
+            exercise.translations = {
+              [syllabus.courseInfo.language || "en"]: {
+                completionId,
+                startedAt: Date.now(),
+                completedAt: 0,
+              },
+            }
+            await saveSyllabus(courseSlug, syllabus, bucket)
+
+            emitToCourse(courseSlug, "course-creation", {
+              lesson: lessonID,
+              status: "generating",
+              log: `🔄 Starting interactivity phase for lesson ${exercise.title}`,
+            })
+          }
+
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: lessonID,
+            status: "initial-content-complete",
+            log: `✅ Initial content generated for lesson ${lessonID}, starting interactivity phase`,
+          })
+
+          res.json({ status: "SUCCESS" })
+        } catch (error) {
+          console.error("Error processing initial content webhook:", error)
+          await updateLessonStatusToError(courseSlug, lessonID, bucket)
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: lessonID,
+            status: "error",
+            log: `❌ Error processing initial content for lesson ${lessonID}`,
+          })
+          res
+            .status(500)
+            .json({ status: "ERROR", error: (error as Error).message })
+        }
+      }
+    )
+
+    // Phase 2: Interactivity generation webhook (replaces exercise-processor logic)
+    app.post(
+      "/webhooks/:courseSlug/interactivity-processor/:lessonID/:rigoToken",
+      async (req, res) => {
+        const { courseSlug, lessonID, rigoToken } = req.params
+        const response = req.body
+
+        console.log("RECEIVING INTERACTIVITY WEBHOOK", response)
+
+        // Handle errors
+        if (response.status === "ERROR") {
+          await updateLessonStatusToError(courseSlug, lessonID, bucket)
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: lessonID,
+            status: "error",
+            log: `❌ Error adding interactivity to lesson ${lessonID}`,
+          })
+
+          // Retry interactivity generation
+          try {
+            const syllabus = await getSyllabus(courseSlug, bucket)
+            const lessonIndex = syllabus.lessons.findIndex(
+              lesson => lesson.id === lessonID
+            )
+            const exercise = syllabus.lessons[lessonIndex]
+
+            if (exercise && exercise.initialContent) {
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: lessonID,
+                status: "generating",
+                log: `🔄 Retrying interactivity generation for lesson ${lessonID}`,
+              })
+
+              // Get previous lesson content for context
+              let lastLesson = ""
+              const prevLessonIndex = lessonIndex - 1
+              if (prevLessonIndex >= 0) {
+                try {
+                  const prevLesson = syllabus.lessons[prevLessonIndex]
+                  const prevLessonSlug = slugify(
+                    prevLesson.id + "-" + prevLesson.title
+                  )
+                  const file = bucket.file(
+                    `courses/${courseSlug}/exercises/${prevLessonSlug}/README${getReadmeExtension(
+                      syllabus.courseInfo.language || "en"
+                    )}`
+                  )
+                  const [content] = await file.download()
+                  lastLesson = content.toString()
+                } catch (error) {
+                  console.error(
+                    "Error getting previous lesson content for retry:",
+                    error
+                  )
+                }
+              }
+
+              const retryCompletionId = await startInteractivityGeneration(
+                rigoToken,
+                syllabus.lessons,
+                syllabus.courseInfo,
+                exercise,
+                courseSlug,
+                syllabus.courseInfo.purpose,
+                bucket,
+                lastLesson
+              )
+
+              // Update lesson status to show it's retrying
+              exercise.status = "GENERATING"
+              exercise.translations = {
+                [syllabus.courseInfo.language || "en"]: {
+                  completionId: retryCompletionId,
+                  startedAt: Date.now(),
+                  completedAt: 0,
+                },
+              }
+              await saveSyllabus(courseSlug, syllabus, bucket)
+            }
+          } catch (retryError) {
+            console.error(
+              "Error retrying interactivity generation:",
+              retryError
+            )
+            emitToCourse(courseSlug, "course-creation", {
+              lesson: lessonID,
+              status: "error",
+              log: `❌ Failed to retry interactivity generation for lesson ${lessonID}`,
+            })
+          }
+
+          return res.json({ status: "ERROR" })
+        }
+
+        try {
+          const syllabus = await getSyllabus(courseSlug, bucket)
+          const exerciseIndex = syllabus.lessons.findIndex(
+            lesson => lesson.id === lessonID
+          )
+
+          if (exerciseIndex === -1) {
+            console.error("Exercise not found receiving webhook:", lessonID)
+            return res.json({ status: "ERROR", error: "Exercise not found" })
+          }
+
+          const exercise = syllabus.lessons[exerciseIndex]
+          const exSlug = slugify(exercise.id + "-" + exercise.title)
+
+          // Process readability of final content
+          const readability = checkReadability(
+            response.parsed.final_content,
+            PARAMS.max_words,
+            3
+          )
+
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: exSlug,
+            status: "generating",
+            log: `🔄 The lesson ${exercise.title} has a readability score of ${readability.fkglResult.fkgl}`,
+          })
+
+          // Upload README with final content
+          const exercisesDir = `courses/${courseSlug}/exercises`
+          const targetDir = `${exercisesDir}/${exSlug}`
+          const readmeFilename = `README${getReadmeExtension(
+            response.parsed.output_language ||
+              syllabus.courseInfo.language ||
+              "en"
+          )}`
+
+          await uploadFileToBucket(
+            bucket,
+            readability.newMarkdown,
+            `${targetDir}/${readmeFilename}`
+          )
+
+          // Handle code files if it's a coding exercise
+          if (
+            exercise.type.toLowerCase() === "code" &&
+            response.parsed.codefile_content
+          ) {
+            emitToCourse(courseSlug, "course-creation", {
+              lesson: exSlug,
+              status: "generating",
+              log: `🔄 Creating code file for ${exercise.title}`,
+            })
+
+            await uploadFileToBucket(
+              bucket,
+              response.parsed.codefile_content,
+              `${targetDir}/${response.parsed.codefile_name
+                .toLowerCase()
+                .trim()}`
+            )
+
+            emitToCourse(courseSlug, "course-creation", {
+              lesson: exSlug,
+              status: "generating",
+              log: `✅ Code file created for ${exercise.title}`,
+            })
+
+            if (response.parsed.solution_content) {
+              const codeFileName = response.parsed.codefile_name
+                .toLowerCase()
+                .trim()
+              const solutionFileName = "solution.hide." + codeFileName
+              await uploadFileToBucket(
+                bucket,
+                response.parsed.solution_content,
+                `${targetDir}/${solutionFileName}`
+              )
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: exSlug,
+                status: "generating",
+                log: `✅ Solution file created for ${exercise.title}`,
+              })
+            }
+          }
+
+          // Update used components if provided by the AI
+          if (
+            response.parsed.used_components &&
+            Array.isArray(response.parsed.used_components)
+          ) {
+            await updateUsedComponents(
+              courseSlug,
+              response.parsed.used_components,
+              bucket
+            )
+            emitToCourse(courseSlug, "course-creation", {
+              lesson: exSlug,
+              status: "generating",
+              log: `📊 Updated component usage tracking: ${response.parsed.used_components.join(
+                ", "
+              )}`,
+            })
+          }
+
+          // Continue with next lesson
+          await continueWithNextLesson(
+            courseSlug,
+            exerciseIndex,
+            rigoToken,
+            response.parsed.final_content,
+            bucket
+          )
+
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: exSlug,
+            status: "done",
+            log: `✅ The lesson ${exercise.id} - ${exercise.title} has been generated successfully with interactivity!`,
+          })
+
+          res.json({ status: "SUCCESS" })
+        } catch (error) {
+          console.error("Error processing interactivity webhook:", error)
+          await updateLessonStatusToError(courseSlug, lessonID, bucket)
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: lessonID,
+            status: "error",
+            log: `❌ Error processing interactivity for lesson ${lessonID}`,
+          })
+          res
+            .status(500)
+            .json({ status: "ERROR", error: (error as Error).message })
+        }
+      }
+    )
+
     // The following endpoint is used to store an incoming translation where it supposed to be
     app.post(
       "/webhooks/:courseSlug/:exSlug/save-translation",
@@ -1016,8 +1780,11 @@ export default class ServeCommand extends SessionCommand {
       }
 
       try {
-        console.log("GET CONFIG, COURSE SLUG", courseSlug)
         const { config, exercises } = await buildConfig(bucket, courseSlug)
+
+        console.log("CONFIG", config)
+        console.log("EXERCISES", exercises)
+
         res.set("X-Creator-Web", "true")
         res.set("Access-Control-Expose-Headers", "X-Creator-Web")
 
@@ -1526,7 +2293,8 @@ export default class ServeCommand extends SessionCommand {
 
       const lastResult = "---"
 
-      const completionId = await startExerciseGeneration(
+      // Use new two-phase generation workflow
+      const completionId = await startInitialContentGeneration(
         rigoToken,
         syllabus.lessons,
         syllabus.courseInfo,
@@ -1804,32 +2572,21 @@ export default class ServeCommand extends SessionCommand {
 
     app.get("/actions/fetch/:link", async (req, res) => {
       const { link } = req.params
+      console.log("[DEBUG] FETCHING LINK", link)
       try {
         // 1) Decode the URL
         const decoded = Buffer.from(link, "base64url").toString("utf-8")
         const ytMatch = decoded.match(YT_REGEX)
-
+        console.log("[DEBUG] YT MATCH", ytMatch)
         if (ytMatch) {
           const videoId = ytMatch[1]
           const resFromRigo = await axios.get(
             `${RIGOBOT_REALTIME_HOST}/actions/youtube-transcript/${videoId}`
           )
 
-          const transcript = resFromRigo.data.transcript
+          console.log("[DEBUG] RES FROM RIGO", resFromRigo.data)
 
-          // let meta: any = null
-          // try {
-          //   const { data: meta } = await axios.get(
-          //     "https://www.youtube.com/oembed",
-          //     {
-          //       params: { url: decoded, format: "json" },
-          //     }
-          //   )
-          //   console.log("META", meta)
-          // } catch (error) {
-          //   console.error("ERROR FETCHING META", error)
-          //   meta = null
-          // }
+          const transcript = resFromRigo.data.transcript
 
           return res.json({
             url: decoded,
@@ -1839,14 +2596,13 @@ export default class ServeCommand extends SessionCommand {
           })
         }
 
-        console.log("NOT A YOUTUBE LINK", decoded)
-
         const response = await axios.get(decoded, { responseType: "text" })
         const html = response.data as string
         const title = getTitleFromHTML(html)
         console.log("TITLE", title)
 
         const text = convert(html)
+        console.log("[DEBUG] TEXT", text)
         return res.json({
           url: decoded,
           text,
@@ -2001,68 +2757,71 @@ export default class ServeCommand extends SessionCommand {
         let filename: string
 
         switch (format) {
-        case "scorm": {
-          outputPath = await exportToScorm({
-            courseSlug: course_slug,
-            format: "scorm",
-            bucket,
-            outDir: path.join(__dirname, "../output/directory"),
-          })
-          filename = `${course_slug}-scorm.zip`
-        
-        break
-        }
-
-        case "zip": {
-          outputPath = await exportToZip({
-            courseSlug: course_slug,
-            format: "zip",
-            bucket,
-            outDir: path.join(__dirname, "../output/directory"),
-          })
-          filename = `${course_slug}.zip`
-        
-        break
-        }
-
-        case "epub": {
-          console.log("EPUB export", metadata)
-          // Validate required metadata for EPUB
-          if (
-            !metadata ||
-            !metadata.creator ||
-            !metadata.publisher ||
-            !metadata.title ||
-            !metadata.rights ||
-            !metadata.lang
-          ) {
-            console.log("Missing required metadata for EPUB export", metadata)
-            return res.status(400).json({
-              error: "Missing required metadata for EPUB export",
-              required: ["creator", "publisher", "title", "rights", "lang"],
+          case "scorm": {
+            outputPath = await exportToScorm({
+              courseSlug: course_slug,
+              format: "scorm",
+              bucket,
+              outDir: path.join(__dirname, "../output/directory"),
             })
+            filename = `${course_slug}-scorm.zip`
+
+            break
           }
 
-          outputPath = await exportToEpub(
-            {
+          case "zip": {
+            outputPath = await exportToZip({
               courseSlug: course_slug,
-              format: "epub",
+              format: "zip",
               bucket,
               outDir: path.join(__dirname, "../output/directory"),
-              language: language,
-            },
-            metadata
-          )
-          filename = `${course_slug}.epub`
-        
-        break
-        }
+            })
+            filename = `${course_slug}.zip`
 
-        default: {
-          return res.status(400).json({
-            error: "Invalid format. Supported formats: scorm, epub, zip",
-          })
-        }
+            break
+          }
+
+          case "epub": {
+            console.log("EPUB export", metadata)
+            // Validate required metadata for EPUB
+            if (
+              !metadata ||
+              !metadata.creator ||
+              !metadata.publisher ||
+              !metadata.title ||
+              !metadata.rights ||
+              !metadata.lang
+            ) {
+              console.log(
+                "Missing required metadata for EPUB export",
+                metadata
+              )
+              return res.status(400).json({
+                error: "Missing required metadata for EPUB export",
+                required: ["creator", "publisher", "title", "rights", "lang"],
+              })
+            }
+
+            outputPath = await exportToEpub(
+              {
+                courseSlug: course_slug,
+                format: "epub",
+                bucket,
+                outDir: path.join(__dirname, "../output/directory"),
+                language: language,
+              },
+              metadata
+            )
+            filename = `${course_slug}.epub`
+
+            break
+          }
+
+          default: {
+            return res.status(400).json({
+              error: "Invalid format. Supported formats: scorm, epub, zip",
+            })
+          }
         }
 
         // Send the file and clean up