@learnpack/learnpack 5.0.291 → 5.0.292

package/lib/commands/serve.js

@@ -77,6 +77,32 @@ const PARAMS = {
     max_rewrite_attempts: 2,
     max_title_length: 50,
 };
+async function fetchComponentsYml() {
+    try {
+        const [assessmentResponse, explanatoryResponse] = await Promise.all([
+            axios_1.default.get("https://raw.githubusercontent.com/learnpack/ide/refs/heads/master/docs/assessment_components.yml"),
+            axios_1.default.get("https://raw.githubusercontent.com/learnpack/ide/refs/heads/master/docs/explanatory_components.yml"),
+        ]);
+        const combinedContent = `
+# ASSESSMENT COMPONENTS
+These components are designed for evaluation and knowledge assessment:
+
+${assessmentResponse.data}
+
+---
+
+# EXPLANATORY COMPONENTS  
+These components are designed for explanation and learning support:
+
+${explanatoryResponse.data}
+`;
+        return combinedContent;
+    }
+    catch (error) {
+        console.error("Failed to fetch components YAML files:", error);
+        return "";
+    }
+}
 const processImage = async (url, description, rigoToken, courseSlug) => {
     try {
         const filename = (0, creatorUtilities_1.getFilenameFromUrl)(url);
@@ -114,6 +140,9 @@ const cleanFormState = (formState) => {
     const { description, technologies, purpose, hasContentIndex, duration, isCompleted, variables, currentStep, language } = formState, rest = tslib_1.__rest(formState, ["description", "technologies", "purpose", "hasContentIndex", "duration", "isCompleted", "variables", "currentStep", "language"]);
     return rest;
 };
+const cleanFormStateForSyllabus = (formState) => {
+    return Object.assign(Object.assign({}, formState), { description: formState.description, technologies: formState.technologies, purposse: undefined, duration: undefined, hasContentIndex: undefined, variables: undefined, currentStep: undefined, language: undefined, isCompleted: undefined });
+};
 const createMultiLangAsset = async (bucket, rigoToken, bcToken, courseSlug, courseJson, deployUrl) => {
     const availableLangs = Object.keys(courseJson.title);
     console.log("AVAILABLE LANGUAGES to upload asset", availableLangs);
@@ -146,7 +175,6 @@ async function startExerciseGeneration(rigoToken, steps, packageContext, exercis
     const exSlug = (0, creatorUtilities_2.slugify)(exercise.id + "-" + exercise.title);
     console.log("Starting generation of", exSlug);
     const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/exercise-processor/${exercise.id}/${rigoToken}`;
-    console.log("WEBHOOK URL", webhookUrl);
     const res = await (0, rigoActions_1.readmeCreator)(rigoToken.trim(), {
         title: `${exercise.id} - ${exercise.title}`,
         output_lang: packageContext.language || "en",
@@ -159,6 +187,67 @@ async function startExerciseGeneration(rigoToken, steps, packageContext, exercis
     console.log("README CREATOR RES", res);
     return res.id;
 }
+const lessonCleaner = (lesson) => {
+    return Object.assign(Object.assign({}, lesson), { description: lesson.description, duration: undefined, generated: undefined, status: undefined, translations: undefined, uid: undefined });
+};
+async function startInitialContentGeneration(rigoToken, steps, packageContext, exercise, courseSlug, purposeSlug, lastLesson = "") {
+    const exSlug = (0, creatorUtilities_2.slugify)(exercise.id + "-" + exercise.title);
+    console.log("Starting initial content generation for", exSlug);
+    const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/initial-content-processor/${exercise.id}/${rigoToken}`;
+    // Emit notification that initial content generation is starting
+    (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+        lesson: exSlug,
+        status: "generating",
+        log: `🔄 Starting initial content generation for lesson: ${exercise.title}`,
+    });
+    const fullSyllabus = {
+        steps: [steps.map(lessonCleaner)],
+        courseInfo: cleanFormStateForSyllabus(packageContext),
+        purpose: purposeSlug,
+    };
+    // Add random 6-digit number to avoid cache issues
+    // eslint-disable-next-line no-mixed-operators
+    const randomCacheEvict = Math.floor(100000 + Math.random() * 900000);
+    const res = await (0, rigoActions_1.initialContentGenerator)(rigoToken.trim(), {
+        // prev_lesson: lastLesson,
+        output_language: packageContext.language || "en",
+        current_syllabus: JSON.stringify(fullSyllabus),
+        lesson_description: JSON.stringify(lessonCleaner(exercise)) + `-${randomCacheEvict}`,
+    }, webhookUrl);
+    console.log("INITIAL CONTENT GENERATOR RES", res);
+    return res.id;
+}
+async function startInteractivityGeneration(rigoToken, steps, packageContext, exercise, courseSlug, purposeSlug, bucket, lastLesson = "") {
+    const exSlug = (0, creatorUtilities_2.slugify)(exercise.id + "-" + exercise.title);
+    console.log("Starting interactivity generation for", exSlug);
+    const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/interactivity-processor/${exercise.id}/${rigoToken}`;
+    // Emit notification that interactivity generation is starting
+    (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+        lesson: exSlug,
+        status: "generating",
+        log: `🔄 Starting interactivity generation for lesson: ${exercise.title}`,
+    });
+    const componentsYml = await fetchComponentsYml();
+    // Get current syllabus to include used_components
+    const currentSyllabus = await getSyllabus(courseSlug, bucket);
+    const fullSyllabus = {
+        steps: steps.map(lessonCleaner),
+        courseInfo: cleanFormStateForSyllabus(packageContext),
+        used_components: currentSyllabus.used_components || {},
+    };
+    // Add random 6-digit number to avoid cache issues
+    // eslint-disable-next-line no-mixed-operators
+    const randomCacheEvict = Math.floor(100000 + Math.random() * 900000);
+    const res = await (0, rigoActions_1.addInteractivity)(rigoToken.trim(), {
+        components: componentsYml,
+        prev_lesson: lastLesson,
+        initial_lesson: exercise.initialContent + `-${randomCacheEvict}`,
+        output_language: packageContext.language || "en",
+        current_syllabus: JSON.stringify(fullSyllabus),
+    }, webhookUrl);
+    console.log("INTERACTIVITY GENERATOR RES", res);
+    return res.id;
+}
 async function createInitialReadme(tutorialInfo, tutorialSlug, rigoToken) {
     const webhookUrl = `${process.env.HOST}/webhooks/${tutorialSlug}/initial-readme-processor`;
     console.log("Creating initial readme", webhookUrl);
@@ -172,6 +261,130 @@ async function createInitialReadme(tutorialInfo, tutorialSlug, rigoToken) {
         console.error("Error creating initial readme", error);
     }
 }
+async function getSyllabus(courseSlug, bucket) {
+    const syllabus = await bucket.file(`courses/${courseSlug}/.learn/initialSyllabus.json`);
+    const [content] = await syllabus.download();
+    return JSON.parse(content.toString());
+}
+async function saveSyllabus(courseSlug, syllabus, bucket) {
+    await uploadFileToBucket(bucket, JSON.stringify(syllabus), `courses/${courseSlug}/.learn/initialSyllabus.json`);
+}
+async function updateUsedComponents(courseSlug, usedComponents, bucket) {
+    const syllabus = await getSyllabus(courseSlug, bucket);
+    // Initialize used_components if undefined
+    if (!syllabus.used_components) {
+        syllabus.used_components = {};
+    }
+    // Update component counts
+    for (const componentName of usedComponents) {
+        if (syllabus.used_components[componentName]) {
+            syllabus.used_components[componentName] += 1;
+        }
+        else {
+            syllabus.used_components[componentName] = 1;
+        }
+    }
+    console.log("Updated component usage:", syllabus.used_components);
+    await saveSyllabus(courseSlug, syllabus, bucket);
+}
+async function updateLessonWithInitialContent(courseSlug, lessonID, initialResponse, bucket) {
+    const syllabus = await getSyllabus(courseSlug, bucket);
+    const lessonIndex = syllabus.lessons.findIndex(lesson => lesson.id === lessonID);
+    if (lessonIndex === -1) {
+        console.error(`Lesson ${lessonID} not found in syllabus`);
+        return;
+    }
+    const lesson = syllabus.lessons[lessonIndex];
+    // Update initial content
+    lesson.initialContent = initialResponse.lesson_content;
+    await saveSyllabus(courseSlug, syllabus, bucket);
+}
+async function updateLessonStatusToError(courseSlug, lessonID, bucket) {
+    try {
+        const syllabus = await getSyllabus(courseSlug, bucket);
+        const lessonIndex = syllabus.lessons.findIndex(lesson => lesson.id === lessonID);
+        if (lessonIndex === -1) {
+            console.error(`Lesson ${lessonID} not found in syllabus when updating status to error`);
+            return;
+        }
+        const lesson = syllabus.lessons[lessonIndex];
+        // Update lesson status to ERROR
+        lesson.status = "ERROR";
+        // Update translations to mark as failed
+        const currentTranslations = lesson.translations || {};
+        const language = syllabus.courseInfo.language || "en";
+        if (currentTranslations[language]) {
+            currentTranslations[language].completedAt = Date.now();
+        }
+        else {
+            currentTranslations[language] = {
+                completionId: 0,
+                startedAt: Date.now(),
+                completedAt: Date.now(),
+            };
+        }
+        lesson.translations = currentTranslations;
+        await saveSyllabus(courseSlug, syllabus, bucket);
+        console.log(`Updated lesson ${lessonID} status to ERROR in syllabus`);
+    }
+    catch (error) {
+        console.error(`Error updating lesson ${lessonID} status to ERROR:`, error);
+    }
+}
+async function continueWithNextLesson(courseSlug, currentExerciseIndex, rigoToken, finalContent, bucket) {
+    const syllabus = await getSyllabus(courseSlug, bucket);
+    const nextExercise = syllabus.lessons[currentExerciseIndex + 1] || null;
+    let nextCompletionId = null;
+    if (nextExercise &&
+        (currentExerciseIndex === 0 ||
+            !(currentExerciseIndex % 3 === 0) ||
+            syllabus.generationMode === "continue-with-all")) {
+        let feedback = "";
+        if (syllabus.feedback) {
+            feedback = `\n\nThe user added the following feedback with relation to the previous generations: ${syllabus.feedback}`;
+        }
+        nextCompletionId = await startInitialContentGeneration(rigoToken, syllabus.lessons, syllabus.courseInfo, nextExercise, courseSlug, syllabus.courseInfo.purpose, finalContent + "\n\n" + feedback);
+    }
+    else {
+        console.log("Stopping generation process at", currentExerciseIndex, syllabus.lessons[currentExerciseIndex].title, "because it's a multiple of 3");
+    }
+    // Update syllabus with next lesson status
+    const newSyllabus = Object.assign(Object.assign({}, syllabus), { lessons: syllabus.lessons.map((lesson, index) => {
+            if (index === currentExerciseIndex) {
+                const currentTranslations = lesson.translations || {};
+                let currentTranslation = currentTranslations[syllabus.courseInfo.language || "en"];
+                if (currentTranslation) {
+                    currentTranslation.completedAt = Date.now();
+                }
+                else {
+                    currentTranslation = {
+                        completionId: nextCompletionId || 0,
+                        startedAt: Date.now(),
+                        completedAt: Date.now(),
+                    };
+                }
+                currentTranslations[syllabus.courseInfo.language || "en"] =
+                    currentTranslation;
+                return Object.assign(Object.assign({}, lesson), { generated: true, status: "DONE", translations: {
+                        [syllabus.courseInfo.language || "en"]: {
+                            completionId: nextCompletionId || 0,
+                            startedAt: currentTranslation.startedAt,
+                            completedAt: Date.now(),
+                        },
+                    } });
+            }
+            if (nextExercise && nextExercise.id === lesson.id && nextCompletionId) {
+                return Object.assign(Object.assign({}, lesson), { generated: false, status: "GENERATING", translations: {
+                        [syllabus.courseInfo.language || "en"]: {
+                            completionId: nextCompletionId,
+                            startedAt: Date.now(),
+                        },
+                    } });
+            }
+            return Object.assign({}, lesson);
+        }) });
+    await saveSyllabus(courseSlug, newSyllabus, bucket);
+}
 const fixPreviewUrl = (slug, previewUrl) => {
     if (!previewUrl) {
         return null;
@@ -421,13 +634,11 @@ class ServeCommand extends SessionCommand_1.default {
                     error: "Rigo token is required. x-rigo-token header is missing",
                 });
             }
-            const syllabus = await bucket.file(`courses/${courseSlug}/.learn/initialSyllabus.json`);
-            const [content] = await syllabus.download();
-            const syllabusJson = JSON.parse(content.toString());
-            const exercise = syllabusJson.lessons[parseInt(position)];
+            const syllabus = await getSyllabus(courseSlug, bucket);
+            const exercise = syllabus.lessons[parseInt(position)];
             // previous exercise
             let previousReadme = "---";
-            const previousExercise = syllabusJson.lessons[parseInt(position) - 1];
+            const previousExercise = syllabus.lessons[parseInt(position) - 1];
             if (previousExercise) {
                 // Get the readme of the previous exercise
                 const exSlug = (0, creatorUtilities_2.slugify)(previousExercise.id + "-" + previousExercise.title);
@@ -443,27 +654,27 @@ class ServeCommand extends SessionCommand_1.default {
                     previousReadme = content.toString();
                 }
             }
-            const completionId = await startExerciseGeneration(rigoToken, syllabusJson.lessons, syllabusJson.courseInfo, exercise, courseSlug, syllabusJson.courseInfo.purpose, previousReadme +
+            // Use new two-phase generation workflow
+            const completionId = await startInitialContentGeneration(rigoToken, syllabus.lessons, syllabus.courseInfo, exercise, courseSlug, syllabus.courseInfo.purpose, previousReadme +
                 "\n\nThe user provided the following feedback related to the content of the course so far: \n\n" +
                 feedback);
-            syllabusJson.lessons[parseInt(position)].status = "GENERATING";
-            syllabusJson.lessons[parseInt(position)].translations = {
-                [syllabusJson.courseInfo.language || "en"]: {
+            syllabus.lessons[parseInt(position)].status = "GENERATING";
+            syllabus.lessons[parseInt(position)].translations = {
+                [syllabus.courseInfo.language || "en"]: {
                     completionId,
                     startedAt: Date.now(),
                     completedAt: 0,
                 },
             };
-            console.log("Lesson", syllabusJson.lessons[parseInt(position)]);
-            if (syllabusJson.feedback &&
-                typeof syllabusJson.feedback === "string") {
-                syllabusJson.feedback += "\n\n" + feedback;
+            console.log("Lesson", syllabus.lessons[parseInt(position)]);
+            if (syllabus.feedback && typeof syllabus.feedback === "string") {
+                syllabus.feedback += "\n\n" + feedback;
             }
             else {
-                syllabusJson.feedback = feedback;
+                syllabus.feedback = feedback;
             }
-            syllabusJson.generationMode = mode;
-            await uploadFileToBucket(bucket, JSON.stringify(syllabusJson), `courses/${courseSlug}/.learn/initialSyllabus.json`);
+            syllabus.generationMode = mode;
+            await saveSyllabus(courseSlug, syllabus, bucket);
             res.json({ status: "SUCCESS" });
         });
         // TODO: Check if this command is being used
@@ -606,6 +817,264 @@ class ServeCommand extends SessionCommand_1.default {
             });
             res.json({ status: "SUCCESS" });
         });
+        // Phase 1: Initial content generation webhook
+        app.post("/webhooks/:courseSlug/initial-content-processor/:lessonID/:rigoToken", async (req, res) => {
+            const { courseSlug, lessonID, rigoToken } = req.params;
+            const response = req.body;
+            console.log("RECEIVING INITIAL CONTENT WEBHOOK", response);
+            // Handle errors
+            if (response.status === "ERROR") {
+                await updateLessonStatusToError(courseSlug, lessonID, bucket);
+                (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                    lesson: lessonID,
+                    status: "error",
+                    log: `❌ Error generating initial content for lesson ${lessonID}`,
+                });
+                // Retry initial content generation
+                try {
+                    const syllabus = await getSyllabus(courseSlug, bucket);
+                    const lessonIndex = syllabus.lessons.findIndex(lesson => lesson.id === lessonID);
+                    const exercise = syllabus.lessons[lessonIndex];
+                    if (exercise) {
+                        (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                            lesson: lessonID,
+                            status: "generating",
+                            log: `🔄 Retrying initial content generation for lesson ${lessonID}`,
+                        });
+                        const retryCompletionId = await startInitialContentGeneration(rigoToken, syllabus.lessons, syllabus.courseInfo, exercise, courseSlug, syllabus.courseInfo.purpose, "");
+                        // Update lesson status to show it's retrying
+                        exercise.status = "GENERATING";
+                        exercise.translations = {
+                            [syllabus.courseInfo.language || "en"]: {
+                                completionId: retryCompletionId,
+                                startedAt: Date.now(),
+                                completedAt: 0,
+                            },
+                        };
+                        await saveSyllabus(courseSlug, syllabus, bucket);
+                    }
+                }
+                catch (retryError) {
+                    console.error("Error retrying initial content generation:", retryError);
+                    (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                        lesson: lessonID,
+                        status: "error",
+                        log: `❌ Failed to retry initial content generation for lesson ${lessonID}`,
+                    });
+                }
+                return res.json({ status: "ERROR" });
+            }
+            try {
+                (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                    lesson: lessonID,
+                    status: "generating",
+                    log: `✅ Initial content generated for lesson ${lessonID}`,
+                });
+                // Update lesson with initial content
+                await updateLessonWithInitialContent(courseSlug, lessonID, response.parsed, bucket);
+                // Start Phase 2: Add interactivity
+                const syllabus = await getSyllabus(courseSlug, bucket);
+                const lessonIndex = syllabus.lessons.findIndex(lesson => lesson.id === lessonID);
+                const exercise = syllabus.lessons[lessonIndex];
+                let lastLesson = "";
+                const prevLessonIndex = lessonIndex - 1;
+                if (prevLessonIndex >= 0) {
+                    try {
+                        const prevLesson = syllabus.lessons[prevLessonIndex];
+                        // search the lesson content in the bucket
+                        const prevLessonSlug = (0, creatorUtilities_2.slugify)(prevLesson.id + "-" + prevLesson.title);
+                        const file = bucket.file(`courses/${courseSlug}/exercises/${prevLessonSlug}/README${(0, creatorUtilities_1.getReadmeExtension)(syllabus.courseInfo.language || "en")}`);
+                        const [content] = await file.download();
+                        lastLesson = content.toString();
+                    }
+                    catch (error) {
+                        console.error("Error searching previous lesson content:", error);
+                    }
+                }
+                if (exercise) {
+                    const completionId = await startInteractivityGeneration(rigoToken, syllabus.lessons, syllabus.courseInfo, exercise, courseSlug, syllabus.courseInfo.purpose, bucket, lastLesson);
+                    // Update lesson status to show it's in Phase 2
+                    exercise.status = "GENERATING";
+                    exercise.translations = {
+                        [syllabus.courseInfo.language || "en"]: {
+                            completionId,
+                            startedAt: Date.now(),
+                            completedAt: 0,
+                        },
+                    };
+                    await saveSyllabus(courseSlug, syllabus, bucket);
+                    (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                        lesson: lessonID,
+                        status: "generating",
+                        log: `🔄 Starting interactivity phase for lesson ${exercise.title}`,
+                    });
+                }
+                (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                    lesson: lessonID,
+                    status: "initial-content-complete",
+                    log: `✅ Initial content generated for lesson ${lessonID}, starting interactivity phase`,
+                });
+                res.json({ status: "SUCCESS" });
+            }
+            catch (error) {
+                console.error("Error processing initial content webhook:", error);
+                await updateLessonStatusToError(courseSlug, lessonID, bucket);
+                (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                    lesson: lessonID,
+                    status: "error",
+                    log: `❌ Error processing initial content for lesson ${lessonID}`,
+                });
+                res
+                    .status(500)
+                    .json({ status: "ERROR", error: error.message });
+            }
+        });
+        // Phase 2: Interactivity generation webhook (replaces exercise-processor logic)
+        app.post("/webhooks/:courseSlug/interactivity-processor/:lessonID/:rigoToken", async (req, res) => {
+            const { courseSlug, lessonID, rigoToken } = req.params;
+            const response = req.body;
+            console.log("RECEIVING INTERACTIVITY WEBHOOK", response);
+            // Handle errors
+            if (response.status === "ERROR") {
+                await updateLessonStatusToError(courseSlug, lessonID, bucket);
+                (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                    lesson: lessonID,
+                    status: "error",
+                    log: `❌ Error adding interactivity to lesson ${lessonID}`,
+                });
+                // Retry interactivity generation
+                try {
+                    const syllabus = await getSyllabus(courseSlug, bucket);
+                    const lessonIndex = syllabus.lessons.findIndex(lesson => lesson.id === lessonID);
+                    const exercise = syllabus.lessons[lessonIndex];
+                    if (exercise && exercise.initialContent) {
+                        (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                            lesson: lessonID,
+                            status: "generating",
+                            log: `🔄 Retrying interactivity generation for lesson ${lessonID}`,
+                        });
+                        // Get previous lesson content for context
+                        let lastLesson = "";
+                        const prevLessonIndex = lessonIndex - 1;
+                        if (prevLessonIndex >= 0) {
+                            try {
+                                const prevLesson = syllabus.lessons[prevLessonIndex];
+                                const prevLessonSlug = (0, creatorUtilities_2.slugify)(prevLesson.id + "-" + prevLesson.title);
+                                const file = bucket.file(`courses/${courseSlug}/exercises/${prevLessonSlug}/README${(0, creatorUtilities_1.getReadmeExtension)(syllabus.courseInfo.language || "en")}`);
+                                const [content] = await file.download();
+                                lastLesson = content.toString();
+                            }
+                            catch (error) {
+                                console.error("Error getting previous lesson content for retry:", error);
+                            }
+                        }
+                        const retryCompletionId = await startInteractivityGeneration(rigoToken, syllabus.lessons, syllabus.courseInfo, exercise, courseSlug, syllabus.courseInfo.purpose, bucket, lastLesson);
+                        // Update lesson status to show it's retrying
+                        exercise.status = "GENERATING";
+                        exercise.translations = {
+                            [syllabus.courseInfo.language || "en"]: {
+                                completionId: retryCompletionId,
+                                startedAt: Date.now(),
+                                completedAt: 0,
+                            },
+                        };
+                        await saveSyllabus(courseSlug, syllabus, bucket);
+                    }
+                }
+                catch (retryError) {
+                    console.error("Error retrying interactivity generation:", retryError);
+                    (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                        lesson: lessonID,
+                        status: "error",
+                        log: `❌ Failed to retry interactivity generation for lesson ${lessonID}`,
+                    });
+                }
+                return res.json({ status: "ERROR" });
+            }
+            try {
+                const syllabus = await getSyllabus(courseSlug, bucket);
+                const exerciseIndex = syllabus.lessons.findIndex(lesson => lesson.id === lessonID);
+                if (exerciseIndex === -1) {
+                    console.error("Exercise not found receiving webhook:", lessonID);
+                    return res.json({ status: "ERROR", error: "Exercise not found" });
+                }
+                const exercise = syllabus.lessons[exerciseIndex];
+                const exSlug = (0, creatorUtilities_2.slugify)(exercise.id + "-" + exercise.title);
+                // Process readability of final content
+                const readability = (0, creatorUtilities_2.checkReadability)(response.parsed.final_content, PARAMS.max_words, 3);
+                (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                    lesson: exSlug,
+                    status: "generating",
+                    log: `🔄 The lesson ${exercise.title} has a readability score of ${readability.fkglResult.fkgl}`,
+                });
+                // Upload README with final content
+                const exercisesDir = `courses/${courseSlug}/exercises`;
+                const targetDir = `${exercisesDir}/${exSlug}`;
+                const readmeFilename = `README${(0, creatorUtilities_1.getReadmeExtension)(response.parsed.output_language ||
+                    syllabus.courseInfo.language ||
+                    "en")}`;
+                await uploadFileToBucket(bucket, readability.newMarkdown, `${targetDir}/${readmeFilename}`);
+                // Handle code files if it's a coding exercise
+                if (exercise.type.toLowerCase() === "code" &&
+                    response.parsed.codefile_content) {
+                    (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                        lesson: exSlug,
+                        status: "generating",
+                        log: `🔄 Creating code file for ${exercise.title}`,
+                    });
+                    await uploadFileToBucket(bucket, response.parsed.codefile_content, `${targetDir}/${response.parsed.codefile_name
+                        .toLowerCase()
+                        .trim()}`);
+                    (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                        lesson: exSlug,
+                        status: "generating",
+                        log: `✅ Code file created for ${exercise.title}`,
+                    });
+                    if (response.parsed.solution_content) {
+                        const codeFileName = response.parsed.codefile_name
+                            .toLowerCase()
+                            .trim();
+                        const solutionFileName = "solution.hide." + codeFileName;
+                        await uploadFileToBucket(bucket, response.parsed.solution_content, `${targetDir}/${solutionFileName}`);
+                        (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                            lesson: exSlug,
+                            status: "generating",
+                            log: `✅ Solution file created for ${exercise.title}`,
+                        });
+                    }
+                }
+                // Update used components if provided by the AI
+                if (response.parsed.used_components &&
+                    Array.isArray(response.parsed.used_components)) {
+                    await updateUsedComponents(courseSlug, response.parsed.used_components, bucket);
+                    (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                        lesson: exSlug,
+                        status: "generating",
+                        log: `📊 Updated component usage tracking: ${response.parsed.used_components.join(", ")}`,
+                    });
+                }
+                // Continue with next lesson
+                await continueWithNextLesson(courseSlug, exerciseIndex, rigoToken, response.parsed.final_content, bucket);
+                (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                    lesson: exSlug,
+                    status: "done",
+                    log: `✅ The lesson ${exercise.id} - ${exercise.title} has been generated successfully with interactivity!`,
+                });
+                res.json({ status: "SUCCESS" });
+            }
+            catch (error) {
+                console.error("Error processing interactivity webhook:", error);
+                await updateLessonStatusToError(courseSlug, lessonID, bucket);
+                (0, creatorSocket_1.emitToCourse)(courseSlug, "course-creation", {
+                    lesson: lessonID,
+                    status: "error",
+                    log: `❌ Error processing interactivity for lesson ${lessonID}`,
+                });
+                res
+                    .status(500)
+                    .json({ status: "ERROR", error: error.message });
+            }
+        });
         // The following endpoint is used to store an incoming translation where it supposed to be
         app.post("/webhooks/:courseSlug/:exSlug/save-translation", async (req, res) => {
             const { courseSlug, exSlug } = req.params;
@@ -666,7 +1135,6 @@ class ServeCommand extends SessionCommand_1.default {
                     .json({ error: "Course slug and rigo token required" });
             }
             try {
-                console.log("GET CONFIG, COURSE SLUG", courseSlug);
                 const { config, exercises } = await (0, configBuilder_1.buildConfig)(bucket, courseSlug);
                 console.log("CONFIG", config);
                 console.log("EXERCISES", exercises);
@@ -1004,7 +1472,8 @@ class ServeCommand extends SessionCommand_1.default {
             await uploadFileToBucket(bucket, JSON.stringify(sidebar), `${tutorialDir}/.learn/sidebar.json`);
             const firstLesson = syllabus.lessons[0];
             const lastResult = "---";
-            const completionId = await startExerciseGeneration(rigoToken, syllabus.lessons, syllabus.courseInfo, firstLesson, courseSlug, syllabus.courseInfo.purpose, lastResult);
+            // Use new two-phase generation workflow
+            const completionId = await startInitialContentGeneration(rigoToken, syllabus.lessons, syllabus.courseInfo, firstLesson, courseSlug, syllabus.courseInfo.purpose, lastResult);
             if (firstLesson) {
                 firstLesson.translations = {
                     [syllabus.courseInfo.language || "en"]: {
@@ -1196,27 +1665,17 @@ class ServeCommand extends SessionCommand_1.default {
         const YT_REGEX = /(?:youtube\.com\/watch\?v=|youtu\.be\/)([\w-]{11})/;
         app.get("/actions/fetch/:link", async (req, res) => {
             const { link } = req.params;
+            console.log("[DEBUG] FETCHING LINK", link);
             try {
                 // 1) Decode the URL
                 const decoded = buffer_1.Buffer.from(link, "base64url").toString("utf-8");
                 const ytMatch = decoded.match(YT_REGEX);
+                console.log("[DEBUG] YT MATCH", ytMatch);
                 if (ytMatch) {
                     const videoId = ytMatch[1];
                     const resFromRigo = await axios_1.default.get(`${api_1.RIGOBOT_REALTIME_HOST}/actions/youtube-transcript/${videoId}`);
+                    console.log("[DEBUG] RES FROM RIGO", resFromRigo.data);
                     const transcript = resFromRigo.data.transcript;
-                    // let meta: any = null
-                    // try {
-                    //   const { data: meta } = await axios.get(
-                    //     "https://www.youtube.com/oembed",
-                    //     {
-                    //       params: { url: decoded, format: "json" },
-                    //     }
-                    //   )
-                    //   console.log("META", meta)
-                    // } catch (error) {
-                    //   console.error("ERROR FETCHING META", error)
-                    //   meta = null
-                    // }
                     return res.json({
                         url: decoded,
                         title: resFromRigo.data.title || null,
@@ -1224,12 +1683,12 @@ class ServeCommand extends SessionCommand_1.default {
                         transcript,
                     });
                 }
-                console.log("NOT A YOUTUBE LINK", decoded);
                 const response = await axios_1.default.get(decoded, { responseType: "text" });
                 const html = response.data;
                 const title = getTitleFromHTML(html);
                 console.log("TITLE", title);
                 const text = (0, html_to_text_1.convert)(html);
+                console.log("[DEBUG] TEXT", text);
                 return res.json({
                     url: decoded,
                     text,