@learnpack/learnpack 5.0.363 → 5.0.365

package/src/commands/serve.ts

@@ -39,6 +39,7 @@ import {
   addInteractivity,
   generateCodeChallenge,
   generateStepSlug,
+  getCompletionJob,
 } from "../utils/rigoActions"
 import * as dotenv from "dotenv"
 
@@ -49,6 +50,17 @@ import {
   insertStepInCorrectPosition,
 } from "../utils/creatorUtilities"
 import { findFinishedLessons } from "../utils/lessonRecovery"
+import {
+  MAX_AUTO_RETRIES,
+  isStaleWebhook,
+  planGenerationRetry,
+  RetryPlan,
+} from "../utils/generationRetry"
+import {
+  currentJobId,
+  isGenerationRunning,
+  resolveGenerationMode,
+} from "../utils/continueGeneration"
 // import { handleAssetCreation } from "./publish"
 import axios from "axios"
 import * as FormData from "form-data"
@@ -129,6 +141,18 @@ import type { RedisClientType } from "redis"
 import { HistoryManager } from "../managers/historyManager"
 import { ReadmeHistoryService } from "../managers/readmeHistoryService"
 import { sanitizeReadmeNewlines } from "../utils/readmeSanitizer"
+import {
+  buildInteractivityGuide,
+  ComponentsGuideUnavailableError,
+  componentsGuideLoader,
+  decidedByFromGuide,
+} from "../utils/interactivity/componentsGuide"
+import {
+  describeInteractivityFailure,
+  isInteractivityV2Enabled,
+  planInteractivityRequest,
+} from "../utils/interactivity/interactivityPlan"
+import { processInteractivityResult } from "../utils/interactivity/structuredComponents"
 
 // eslint-disable-next-line
 const frontMatter = require("front-matter");
@@ -159,7 +183,11 @@ function opId(): string {
   return crypto.randomBytes(4).toString("hex")
 }
 
-// Serialization per course to prevent syllabus race conditions
+// Serialization per course to prevent syllabus race conditions. The queue lives
+// in memory, so it only orders requests within one process: serve runs on a
+// single Heroku dyno. With more than one, it has to move to Redis.
+// A function running in the queue must not call serializeByCourse for the same
+// course, or it waits for itself.
 const courseQueues = new Map<string, Promise<unknown>>()
 
 async function serializeByCourse<T>(
@@ -167,11 +195,11 @@ async function serializeByCourse<T>(
   fn: () => Promise<T>
 ): Promise<T> {
   const prev = courseQueues.get(courseSlug) ?? Promise.resolve()
+  // A failure belongs to its own caller. Without the catch, every function
+  // queued after it would be skipped and fail with the same error.
   const next = prev
+    .catch(() => null)
     .then(() => fn())
-    .catch(error => {
-      throw error
-    })
     .finally(() => {
       if (courseQueues.get(courseSlug) === next) {
         courseQueues.delete(courseSlug)
@@ -696,12 +724,45 @@ async function startInteractivityGeneration(
   courseSlug: string,
   purposeSlug: string,
   bucket: Bucket,
-  lastLesson = ""
+  lastLesson = "",
+  options: { forceV1?: boolean } = {}
 ): Promise<number> {
   const exSlug = slugify(exercise.id + "-" + exercise.title)
   console.log("Starting interactivity generation for", exSlug)
 
-  const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/interactivity-processor/${exercise.uid}/${rigoToken}`
+  const request = planInteractivityRequest({
+    v2Enabled: isInteractivityV2Enabled(),
+    forceV1: options.forceV1,
+  })
+
+  // The structured template (v2) needs the components guide: without it the
+  // model does not know what each component field means. The guide is loaded
+  // before anything else so that rigobot is never called without it.
+  let componentsYml: string
+  if (request.tpl === "v2") {
+    try {
+      componentsYml = buildInteractivityGuide(
+        await componentsGuideLoader.load()
+      )
+    } catch (error) {
+      if (error instanceof ComponentsGuideUnavailableError) {
+        console.error("Components guide unavailable:", error.message)
+        emitToCourse(courseSlug, "course-creation", {
+          lesson: exSlug,
+          status: "error",
+          log: `❌ Could not load the components guide for lesson ${exercise.title}; retry the generation`,
+        })
+      }
+
+      throw error
+    }
+  } else {
+    componentsYml = await fetchComponentsYml()
+  }
+
+  // The template and the attempt travel in the webhook URL: rigobot posts back
+  // to this exact URL, and its error payload says neither.
+  const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/interactivity-processor/${exercise.uid}/${rigoToken}${request.webhookQuery}`
 
   // Emit notification that interactivity generation is starting
   emitToCourse(courseSlug, "course-creation", {
@@ -710,8 +771,6 @@ async function startInteractivityGeneration(
     log: `🔄 Starting interactivity generation for lesson: ${exercise.title}`,
   })
 
-  const componentsYml = await fetchComponentsYml()
-
   const currentSyllabus = await getSyllabus(courseSlug, bucket)
 
   const fullSyllabus = {
@@ -734,9 +793,16 @@ async function startInteractivityGeneration(
       current_syllabus: JSON.stringify(fullSyllabus),
       lesson_info: JSON.stringify(lessonCleaner(exercise)),
     },
-    webhookUrl
+    webhookUrl,
+    request.slug
   )
 
+  if (!res) {
+    throw new Error(
+      `Could not start interactivity generation for lesson ${exSlug} (${request.slug})`
+    )
+  }
+
   return res.id
 }
 
@@ -923,6 +989,63 @@ async function updateLessonWithInitialContent(
   return lesson
 }
 
+/**
+ * Changes one lesson on a fresh read of the syllabus. Saving a copy read
+ * earlier would undo whatever was saved since, like the lesson's
+ * `initialContent`.
+ * @param courseSlug - The course.
+ * @param lessonUID - The lesson to change.
+ * @param bucket - The bucket that holds the syllabus.
+ * @param patch - Changes the lesson in place. Gets the course language.
+ * @returns Once the syllabus is saved, or right away if the lesson is gone.
+ */
+async function patchLesson(
+  courseSlug: string,
+  lessonUID: string,
+  bucket: Bucket,
+  patch: (lesson: Lesson, language: string) => void
+): Promise<void> {
+  const syllabus = await getSyllabus(courseSlug, bucket)
+  const lesson = syllabus.lessons.find(lesson => lesson.uid === lessonUID)
+
+  if (!lesson) {
+    console.error(`Lesson ${lessonUID} not found in syllabus`)
+    return
+  }
+
+  patch(lesson, syllabus.courseInfo.language || "en")
+  await saveSyllabus(courseSlug, syllabus, bucket)
+}
+
+/**
+ * The translation entry of a lesson that waits for a new job.
+ * @param completionId - The Rigobot job.
+ * @param autoRetries - The automatic retries of the current phase so far.
+ * @returns The entry to save under the course language.
+ */
+const startedJob = (completionId: number, autoRetries = 0) => ({
+  completionId,
+  startedAt: Date.now(),
+  completedAt: 0,
+  autoRetries,
+})
+
+/**
+ * The creator log for a failed job that is not retried.
+ * @param reason - Why there is no retry.
+ * @param phase - What the job generated.
+ * @param exSlug - The lesson slug.
+ * @returns The log line.
+ */
+const noRetryLog = (
+  reason: Extract<RetryPlan, { retry: false }>["reason"],
+  phase: string,
+  exSlug: string
+): string =>
+  reason === "permanent" ?
+    `❌ Error generating ${phase} for lesson ${exSlug}. Retrying cannot fix this error, so it was not retried` :
+    `❌ Error generating ${phase} for lesson ${exSlug} after ${MAX_AUTO_RETRIES} retries`
+
 async function updateLessonStatusToError(
   courseSlug: string,
   lessonUID: string,
@@ -2147,88 +2270,122 @@ class ServeCommand extends SessionCommand {
           )
         }
 
-        const syllabus = await getSyllabus(courseSlug, bucket)
-
-        const exercise = syllabus.lessons.find(
-          lesson => lesson.uid === lessonUid
-        )
-        const exerciseIndex = syllabus.lessons.findIndex(
-          lesson => lesson.uid === lessonUid
-        )
+        // In the course queue, so the webhook of this job cannot be handled
+        // before the lesson is saved as generating.
+        await serializeByCourse(courseSlug, async () => {
+          const syllabus = await getSyllabus(courseSlug, bucket)
 
-        // Validate that exercise exists
-        if (!exercise) {
-          throw new NotFoundError(
-            `Lesson with UID "${lessonUid}" not found in course "${courseSlug}"`
+          const exercise = syllabus.lessons.find(
+            lesson => lesson.uid === lessonUid
           )
-        }
-
-        // Validate that exercise index is valid (defensive check)
-        if (exerciseIndex === -1) {
-          throw new NotFoundError(
-            `Lesson with UID "${lessonUid}" not found in course "${courseSlug}"`
+          const exerciseIndex = syllabus.lessons.findIndex(
+            lesson => lesson.uid === lessonUid
           )
-        }
 
-        // Validate that exercise has required properties
-        if (!exercise.id || !exercise.title) {
-          throw new ValidationError(
-            `Lesson "${lessonUid}" is missing required properties (id or title)`
-          )
-        }
+          // Validate that exercise exists
+          if (!exercise) {
+            throw new NotFoundError(
+              `Lesson with UID "${lessonUid}" not found in course "${courseSlug}"`
+            )
+          }
 
-        // previous exercise
-        let previousReadme = "---"
-        const previousExercise = syllabus.lessons[exerciseIndex - 1]
-        if (previousExercise) {
-          // Get the readme of the previous exercise
-          const exSlug = slugify(
-            previousExercise.id + "-" + previousExercise.title
-          )
-          // llist the files un
-          const [files] = await bucket.getFiles({
-            prefix: `courses/${courseSlug}/exercises/${exSlug}/README`,
-          })
-          // select any README
-          const readmeFiles = files.map(f => f.name)
-          const readmeFile = readmeFiles.find(f => f.includes("README"))
-          if (readmeFile) {
-            const [content] = await bucket.file(readmeFile).download()
-            previousReadme = content.toString()
+          // Validate that exercise index is valid (defensive check)
+          if (exerciseIndex === -1) {
+            throw new NotFoundError(
+              `Lesson with UID "${lessonUid}" not found in course "${courseSlug}"`
+            )
           }
-        }
 
-        // Use new two-phase generation workflow
-        const completionId = await startInitialContentGeneration(
-          rigoToken,
-          syllabus.lessons,
-          syllabus.courseInfo,
-          exercise,
-          courseSlug,
-          syllabus.courseInfo.purpose,
-          previousReadme +
-            "\n\nThe user provided the following feedback related to the content of the course so far: \n\n" +
-            feedback
-        )
+          // Validate that exercise has required properties
+          if (!exercise.id || !exercise.title) {
+            throw new ValidationError(
+              `Lesson "${lessonUid}" is missing required properties (id or title)`
+            )
+          }
 
-        syllabus.lessons[exerciseIndex].status = "GENERATING"
-        syllabus.lessons[exerciseIndex].translations = {
-          [syllabus.courseInfo.language || "en"]: {
-            completionId,
-            startedAt: Date.now(),
-            completedAt: 0,
-          },
-        }
-        console.log("Lesson", syllabus.lessons[exerciseIndex])
-        if (syllabus.feedback && typeof syllabus.feedback === "string") {
-          syllabus.feedback += "\n\n" + feedback
-        } else {
-          syllabus.feedback = feedback
-        }
+          const language = syllabus.courseInfo.language || "en"
+          const previousStatus = exercise.status
+
+          // A second click, or a retry while the job still runs, would start
+          // another paid job for the same lesson.
+          if (previousStatus === "GENERATING") {
+            const jobId = currentJobId(exercise, language)
+            let job = null
+            if (jobId) {
+              try {
+                job = await getCompletionJob(rigoToken, jobId, {
+                  timeoutMs: 5000,
+                })
+              } catch (error) {
+                console.error(`Could not check job ${jobId}:`, error)
+              }
+            }
 
-        syllabus.generationMode = mode
+            if (isGenerationRunning(exercise, language, job, Date.now())) {
+              console.log(
+                `Lesson ${lessonUid} is already being generated (job ${jobId}), not starting another one`
+              )
+              throw new ConflictError(
+                `Lesson "${exercise.title}" is already being generated`,
+                { code: "LESSON_ALREADY_GENERATING", completionId: jobId }
+              )
+            }
+          }
+
+          // previous exercise
+          let previousReadme = "---"
+          const previousExercise = syllabus.lessons[exerciseIndex - 1]
+          if (previousExercise) {
+            // Get the readme of the previous exercise
+            const exSlug = slugify(
+              previousExercise.id + "-" + previousExercise.title
+            )
+            // llist the files un
+            const [files] = await bucket.getFiles({
+              prefix: `courses/${courseSlug}/exercises/${exSlug}/README`,
+            })
+            // select any README
+            const readmeFiles = files.map(f => f.name)
+            const readmeFile = readmeFiles.find(f => f.includes("README"))
+            if (readmeFile) {
+              const [content] = await bucket.file(readmeFile).download()
+              previousReadme = content.toString()
+            }
+          }
 
-        await saveSyllabus(courseSlug, syllabus, bucket)
+          // Use new two-phase generation workflow
+          const completionId = await startInitialContentGeneration(
+            rigoToken,
+            syllabus.lessons,
+            syllabus.courseInfo,
+            exercise,
+            courseSlug,
+            syllabus.courseInfo.purpose,
+            previousReadme +
+              "\n\nThe user provided the following feedback related to the content of the course so far: \n\n" +
+              feedback
+          )
+
+          // A new job by the user, so the automatic retries start over.
+          syllabus.lessons[exerciseIndex].status = "GENERATING"
+          syllabus.lessons[exerciseIndex].translations = {
+            [language]: startedJob(completionId),
+          }
+          console.log("Lesson", syllabus.lessons[exerciseIndex])
+          if (syllabus.feedback && typeof syllabus.feedback === "string") {
+            syllabus.feedback += "\n\n" + feedback
+          } else {
+            syllabus.feedback = feedback
+          }
+
+          syllabus.generationMode = resolveGenerationMode(
+            previousStatus,
+            syllabus.generationMode,
+            mode
+          )
+
+          await saveSyllabus(courseSlug, syllabus, bucket)
+        })
 
         res.json({ status: "SUCCESS" })
       })
@@ -2546,223 +2703,152 @@ class ServeCommand extends SessionCommand {
 
         console.log("RECEIVING INITIAL CONTENT WEBHOOK", response)
 
-        const syllabus = await getSyllabus(courseSlug, bucket)
-
-        const exerciseIndex = syllabus.lessons.findIndex(
-          lesson => lesson.uid === lessonUID
-        )
-
-        if (exerciseIndex === -1) {
-          console.error("Exercise not found receiving webhook:", lessonUID)
-          return res.json({ status: "ERROR", error: "Exercise not found" })
-        }
-
-        let exercise: Lesson | null = syllabus.lessons[exerciseIndex]
-        const exSlug = slugify(exercise.id + "-" + exercise.title)
-        // Handle errors
-        if (response.status === "ERROR") {
-          await updateLessonStatusToError(courseSlug, lessonUID, bucket)
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: exSlug,
-            status: "error",
-            log: `❌ Error generating initial content for lesson ${exSlug}`,
-          })
-
-          // Retry initial content generation
-          try {
-            emitToCourse(courseSlug, "course-creation", {
-              lesson: exSlug,
-              status: "generating",
-              log: `🔄 Retrying initial content generation for lesson ${exSlug}`,
-            })
+        try {
+          // In the course queue, nothing else saves the syllabus between
+          // reading the lesson and saving it.
+          await serializeByCourse(courseSlug, async () => {
+            const syllabus = await getSyllabus(courseSlug, bucket)
+            const language = syllabus.courseInfo.language || "en"
 
-            const retryCompletionId = await startInitialContentGeneration(
-              rigoToken,
-              syllabus.lessons,
-              syllabus.courseInfo,
-              exercise,
-              courseSlug,
-              syllabus.courseInfo.purpose,
-              ""
+            const exerciseIndex = syllabus.lessons.findIndex(
+              lesson => lesson.uid === lessonUID
             )
 
-            // Update lesson status to show it's retrying
-            exercise.status = "GENERATING"
-            exercise.translations = {
-              [syllabus.courseInfo.language || "en"]: {
-                completionId: retryCompletionId,
-                startedAt: Date.now(),
-                completedAt: 0,
-              },
+            if (exerciseIndex === -1) {
+              console.error("Exercise not found receiving webhook:", lessonUID)
+              return res.json({ status: "ERROR", error: "Exercise not found" })
             }
-            await saveSyllabus(courseSlug, syllabus, bucket)
-          } catch (retryError) {
-            console.error(
-              "Error retrying initial content generation:",
-              retryError
-            )
-            emitToCourse(courseSlug, "course-creation", {
-              lesson: lessonUID,
-              status: "error",
-              log: `❌ Failed to retry initial content generation for lesson ${lessonUID}`,
-            })
-          }
-
-          return res.json({ status: "ERROR" })
-        }
-
-        try {
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: exSlug,
-            status: "generating",
-            log: `✅ Initial content generated for lesson ${exSlug}`,
-          })
-
-          // Update lesson with initial content
-          exercise = await updateLessonWithInitialContent(
-            courseSlug,
-            lessonUID,
-            response.parsed,
-            bucket
-          )
-
-          if (!exercise) {
-            console.error("Exercise not found after updating initial content")
-            return res.json({ status: "ERROR", error: "Exercise not found" })
-          }
 
-          let lastLesson = ""
+            let exercise: Lesson | null = syllabus.lessons[exerciseIndex]
+            const exSlug = slugify(exercise.id + "-" + exercise.title)
 
-          const prevLessonIndex = exerciseIndex - 1
-          if (prevLessonIndex >= 0) {
-            try {
-              const prevLesson = syllabus.lessons[prevLessonIndex]
-              // search the lesson content in the bucket
-              const prevLessonSlug = slugify(
-                prevLesson.id + "-" + prevLesson.title
-              )
-              const file = bucket.file(
-                `courses/${courseSlug}/exercises/${prevLessonSlug}/README${getReadmeExtension(
-                  syllabus.courseInfo.language || "en"
-                )}`
+            if (isStaleWebhook(exercise, language, response.id)) {
+              console.log(
+                `Ignoring initial content webhook of job ${response.id} for lesson ${exSlug}: the lesson no longer waits for it`
               )
-              const [content] = await file.download()
-              lastLesson = content.toString()
-            } catch (error) {
-              console.error("Error searching previous lesson content:", error)
+              return res.json({ status: "IGNORED" })
             }
-          }
-
-          const completionId = await startInteractivityGeneration(
-            rigoToken,
-            syllabus.lessons,
-            syllabus.courseInfo,
-            exercise,
-            courseSlug,
-            syllabus.courseInfo.purpose,
-            bucket,
-            lastLesson
-          )
 
-          // Update lesson status to show it's in Phase 2
-          exercise.status = "GENERATING"
-          exercise.translations = {
-            [syllabus.courseInfo.language || "en"]: {
-              completionId,
-              startedAt: Date.now(),
-              completedAt: 0,
-            },
-          }
-          await saveSyllabus(courseSlug, syllabus, bucket)
+            // Handle errors
+            if (response.status === "ERROR") {
+              console.error(
+                `Initial content job ${response.id} failed for lesson ${exSlug}:`,
+                response.status_text
+              )
 
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: exSlug,
-            status: "generating",
-            log: `🔄 Starting interactivity phase for lesson ${exercise.title}`,
-          })
+              const retryPlan = planGenerationRetry(
+                exercise,
+                language,
+                response.status_text
+              )
 
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: exSlug,
-            status: "initial-content-complete",
-            log: `✅ Initial content generated for lesson ${exSlug}, starting interactivity phase`,
-          })
+              if (!retryPlan.retry) {
+                await updateLessonStatusToError(courseSlug, lessonUID, bucket)
+                emitToCourse(courseSlug, "course-creation", {
+                  lesson: exSlug,
+                  status: "error",
+                  log: noRetryLog(retryPlan.reason, "initial content", exSlug),
+                })
+                return res.json({ status: "ERROR" })
+              }
 
-          res.json({ status: "SUCCESS" })
-        } catch (error) {
-          console.error("Error processing initial content webhook:", error)
-          await updateLessonStatusToError(courseSlug, lessonUID, bucket)
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: exSlug,
-            status: "error",
-            log: `❌ Error processing initial content for lesson ${exSlug}`,
-          })
-          res
-            .status(500)
-            .json({ status: "ERROR", error: (error as Error).message })
-        }
-      }
-    )
+              const attempt = retryPlan.attempt
 
-    // Phase 2: Interactivity generation webhook (replaces exercise-processor logic)
-    app.post(
-      "/webhooks/:courseSlug/interactivity-processor/:lessonUID/:rigoToken",
-      async (req, res) => {
-        const { courseSlug, lessonUID, rigoToken } = req.params
-        const response = req.body
+              try {
+                emitToCourse(courseSlug, "course-creation", {
+                  lesson: exSlug,
+                  status: "generating",
+                  log: `🔄 Retrying initial content generation for lesson ${exSlug} (${attempt} of ${MAX_AUTO_RETRIES})`,
+                })
 
-        console.log("RECEIVING INTERACTIVITY WEBHOOK")
-        // console.log("LESSON UID", lessonUID)
-        // console.log("RESPONSE", response)
+                const retryCompletionId = await startInitialContentGeneration(
+                  rigoToken,
+                  syllabus.lessons,
+                  syllabus.courseInfo,
+                  exercise,
+                  courseSlug,
+                  syllabus.courseInfo.purpose,
+                  ""
+                )
 
-        // Handle errors
-        if (response.status === "ERROR") {
-          await updateLessonStatusToError(courseSlug, lessonUID, bucket)
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: lessonUID,
-            status: "error",
-            log: `❌ Error adding interactivity to lesson ${lessonUID}`,
-          })
+                await patchLesson(
+                  courseSlug,
+                  lessonUID,
+                  bucket,
+                  (lesson, lang) => {
+                    lesson.status = "GENERATING"
+                    lesson.translations = {
+                      [lang]: startedJob(retryCompletionId, attempt),
+                    }
+                  }
+                )
+              } catch (retryError) {
+                console.error(
+                  "Error retrying initial content generation:",
+                  retryError
+                )
+                await updateLessonStatusToError(courseSlug, lessonUID, bucket)
+                emitToCourse(courseSlug, "course-creation", {
+                  lesson: exSlug,
+                  status: "error",
+                  log: `❌ Failed to retry initial content generation for lesson ${exSlug}`,
+                })
+              }
 
-          // Retry interactivity generation
-          try {
-            const syllabus = await getSyllabus(courseSlug, bucket)
-            const lessonIndex = syllabus.lessons.findIndex(
-              lesson => lesson.uid === lessonUID
-            )
-            const exercise = syllabus.lessons[lessonIndex]
+              return res.json({ status: "ERROR" })
+            }
 
-            if (exercise && exercise.initialContent) {
+            try {
               emitToCourse(courseSlug, "course-creation", {
-                lesson: lessonUID,
+                lesson: exSlug,
                 status: "generating",
-                log: `🔄 Retrying interactivity generation for lesson ${lessonUID}`,
+                log: `✅ Initial content generated for lesson ${exSlug}`,
               })
 
-              // Get previous lesson content for context
+              // Update lesson with initial content
+              exercise = await updateLessonWithInitialContent(
+                courseSlug,
+                lessonUID,
+                response.parsed,
+                bucket
+              )
+
+              if (!exercise) {
+                console.error(
+                  "Exercise not found after updating initial content"
+                )
+                return res.json({
+                  status: "ERROR",
+                  error: "Exercise not found",
+                })
+              }
+
               let lastLesson = ""
-              const prevLessonIndex = lessonIndex - 1
+
+              const prevLessonIndex = exerciseIndex - 1
               if (prevLessonIndex >= 0) {
                 try {
                   const prevLesson = syllabus.lessons[prevLessonIndex]
+                  // search the lesson content in the bucket
                   const prevLessonSlug = slugify(
                     prevLesson.id + "-" + prevLesson.title
                   )
                   const file = bucket.file(
                     `courses/${courseSlug}/exercises/${prevLessonSlug}/README${getReadmeExtension(
-                      syllabus.courseInfo.language || "en"
+                      language
                     )}`
                   )
                   const [content] = await file.download()
                   lastLesson = content.toString()
                 } catch (error) {
                   console.error(
-                    "Error getting previous lesson content for retry:",
+                    "Error searching previous lesson content:",
                     error
                   )
                 }
               }
 
-              const retryCompletionId = await startInteractivityGeneration(
+              const completionId = await startInteractivityGeneration(
                 rigoToken,
                 syllabus.lessons,
                 syllabus.courseInfo,
@@ -2773,120 +2859,337 @@ class ServeCommand extends SessionCommand {
                 lastLesson
               )
 
-              // Update lesson status to show it's retrying
-              exercise.status = "GENERATING"
-              exercise.translations = {
-                [syllabus.courseInfo.language || "en"]: {
-                  completionId: retryCompletionId,
-                  startedAt: Date.now(),
-                  completedAt: 0,
-                },
-              }
-              await saveSyllabus(courseSlug, syllabus, bucket)
+              // Saved on a fresh read: `syllabus` was read before the initial
+              // content was saved, and saving it would erase that content.
+              await patchLesson(
+                courseSlug,
+                lessonUID,
+                bucket,
+                (lesson, lang) => {
+                  lesson.status = "GENERATING"
+                  lesson.translations = { [lang]: startedJob(completionId) }
+                }
+              )
+
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: exSlug,
+                status: "generating",
+                log: `🔄 Starting interactivity phase for lesson ${exercise.title}`,
+              })
+
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: exSlug,
+                status: "initial-content-complete",
+                log: `✅ Initial content generated for lesson ${exSlug}, starting interactivity phase`,
+              })
+
+              res.json({ status: "SUCCESS" })
+            } catch (error) {
+              console.error("Error processing initial content webhook:", error)
+              await updateLessonStatusToError(courseSlug, lessonUID, bucket)
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: exSlug,
+                status: "error",
+                log: `❌ Error processing initial content for lesson ${exSlug}`,
+              })
+              res
+                .status(500)
+                .json({ status: "ERROR", error: (error as Error).message })
             }
-          } catch (retryError) {
-            console.error(
-              "Error retrying interactivity generation:",
-              retryError
-            )
-            emitToCourse(courseSlug, "course-creation", {
-              lesson: lessonUID,
-              status: "error",
-              log: `❌ Failed to retry interactivity generation for lesson ${lessonUID}`,
-            })
+          })
+        } catch (error) {
+          console.error("Error handling initial content webhook:", error)
+          if (!res.headersSent) {
+            res
+              .status(500)
+              .json({ status: "ERROR", error: (error as Error).message })
           }
-
-          return res.json({ status: "ERROR" })
         }
+      }
+    )
+
+    // Phase 2: Interactivity generation webhook (replaces exercise-processor logic)
+    app.post(
+      "/webhooks/:courseSlug/interactivity-processor/:lessonUID/:rigoToken",
+      async (req, res) => {
+        const { courseSlug, lessonUID, rigoToken } = req.params
+        const response = req.body
+
+        console.log("RECEIVING INTERACTIVITY WEBHOOK")
+        // console.log("LESSON UID", lessonUID)
+        // console.log("RESPONSE", response)
 
         try {
-          const syllabus = await getSyllabus(courseSlug, bucket)
-          const exerciseIndex = syllabus.lessons.findIndex(
-            lesson => lesson.uid === lessonUID
-          )
+          // In the course queue, nothing else saves the syllabus between
+          // reading the lesson and saving it.
+          await serializeByCourse(courseSlug, async () => {
+            const syllabus = await getSyllabus(courseSlug, bucket)
+            const language = syllabus.courseInfo.language || "en"
 
-          if (exerciseIndex === -1) {
-            console.error("Exercise not found receiving webhook:", lessonUID)
-            return res.json({ status: "ERROR", error: "Exercise not found" })
-          }
+            const exerciseIndex = syllabus.lessons.findIndex(
+              lesson => lesson.uid === lessonUID
+            )
 
-          const exercise = syllabus.lessons[exerciseIndex]
-          const exSlug = slugify(exercise.id + "-" + exercise.title)
+            if (exerciseIndex === -1) {
+              console.error("Exercise not found receiving webhook:", lessonUID)
+              return res.json({ status: "ERROR", error: "Exercise not found" })
+            }
 
-          // Process readability of final content
-          const readability = checkReadability(
-            response.parsed.final_content,
-            PARAMS.max_words,
-            3
-          )
+            const exercise = syllabus.lessons[exerciseIndex]
+            const exSlug = slugify(exercise.id + "-" + exercise.title)
 
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: exSlug,
-            status: "generating",
-            log: `🔄 The lesson ${exercise.title} has a readability score of ${readability.fkglResult.fkgl}`,
-          })
+            if (isStaleWebhook(exercise, language, response.id)) {
+              console.log(
+                `Ignoring interactivity webhook of job ${response.id} for lesson ${exSlug}: the lesson no longer waits for it`
+              )
+              return res.json({ status: "IGNORED" })
+            }
 
-          // Upload README with final content
-          const exercisesDir = `courses/${courseSlug}/exercises`
-          const targetDir = `${exercisesDir}/${exSlug}`
-          const readmeFilename = `README${getReadmeExtension(
-            response.parsed.output_language ||
-              syllabus.courseInfo.language ||
-              "en"
-          )}`
+            // Handle errors
+            if (response.status === "ERROR") {
+              console.error(
+                `Interactivity job ${response.id} failed for lesson ${exSlug}:`,
+                response.status_text
+              )
 
-          await uploadFileToBucket(
-            bucket,
-            readability.newMarkdown,
-            `${targetDir}/${readmeFilename}`
-          )
+              // Every retry uses v1. A failed v2 job gets its v1 retry even
+              // when its error would repeat, because the error belongs to the
+              // v2 template.
+              const { fromV2, retryStatusText } = describeInteractivityFailure(
+                req.query,
+                response.status_text
+              )
 
-          // Update used components if provided by the AI
-          if (
-            response.parsed.used_components &&
-            Array.isArray(response.parsed.used_components)
-          ) {
-            await updateUsedComponents(
-              courseSlug,
-              response.parsed.used_components,
-              bucket
-            )
-            emitToCourse(courseSlug, "course-creation", {
-              lesson: exSlug,
-              status: "generating",
-              log: `📊 Updated component usage tracking: ${response.parsed.used_components.join(
-                ", "
-              )}`,
-            })
-          }
+              const retryPlan = planGenerationRetry(
+                exercise,
+                language,
+                retryStatusText
+              )
 
-          // Continue with next lesson
-          await continueWithNextLesson(
-            courseSlug,
-            exerciseIndex,
-            rigoToken,
-            response.parsed.final_content,
-            bucket
-          )
+              // The retry builds on the initial content, so without it there
+              // is nothing to retry.
+              if (!retryPlan.retry || !exercise.initialContent) {
+                await updateLessonStatusToError(courseSlug, lessonUID, bucket)
+                emitToCourse(courseSlug, "course-creation", {
+                  lesson: exSlug,
+                  status: "error",
+                  log: retryPlan.retry ?
+                    `❌ Error adding interactivity to lesson ${exSlug}. Its initial content is missing, so it was not retried` :
+                    noRetryLog(retryPlan.reason, "interactivity", exSlug),
+                })
+                return res.json({ status: "ERROR" })
+              }
 
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: exSlug,
-            status: "done",
-            log: `✅ The lesson ${exercise.id} - ${exercise.title} has been generated successfully with interactivity!`,
-          })
+              const attempt = retryPlan.attempt
 
-          res.json({ status: "SUCCESS" })
-        } catch (error) {
-          console.error("Error processing interactivity webhook:", error)
-          await updateLessonStatusToError(courseSlug, lessonUID, bucket)
-          emitToCourse(courseSlug, "course-creation", {
-            lesson: lessonUID,
-            status: "error",
-            log: `❌ Error processing interactivity for lesson ${lessonUID}`,
+              try {
+                emitToCourse(courseSlug, "course-creation", {
+                  lesson: exSlug,
+                  status: "generating",
+                  log: fromV2 ?
+                    `⚠️ Structured interactivity failed for lesson ${exSlug}, retrying with the previous method (${attempt} of ${MAX_AUTO_RETRIES})` :
+                    `🔄 Retrying interactivity generation for lesson ${exSlug} (${attempt} of ${MAX_AUTO_RETRIES})`,
+                })
+
+                // Get previous lesson content for context
+                let lastLesson = ""
+                const prevLessonIndex = exerciseIndex - 1
+                if (prevLessonIndex >= 0) {
+                  try {
+                    const prevLesson = syllabus.lessons[prevLessonIndex]
+                    const prevLessonSlug = slugify(
+                      prevLesson.id + "-" + prevLesson.title
+                    )
+                    const file = bucket.file(
+                      `courses/${courseSlug}/exercises/${prevLessonSlug}/README${getReadmeExtension(
+                        language
+                      )}`
+                    )
+                    const [content] = await file.download()
+                    lastLesson = content.toString()
+                  } catch (error) {
+                    console.error(
+                      "Error getting previous lesson content for retry:",
+                      error
+                    )
+                  }
+                }
+
+                const retryCompletionId = await startInteractivityGeneration(
+                  rigoToken,
+                  syllabus.lessons,
+                  syllabus.courseInfo,
+                  exercise,
+                  courseSlug,
+                  syllabus.courseInfo.purpose,
+                  bucket,
+                  lastLesson,
+                  { forceV1: true }
+                )
+
+                await patchLesson(
+                  courseSlug,
+                  lessonUID,
+                  bucket,
+                  (lesson, lang) => {
+                    lesson.status = "GENERATING"
+                    lesson.translations = {
+                      [lang]: startedJob(retryCompletionId, attempt),
+                    }
+                  }
+                )
+              } catch (retryError) {
+                console.error(
+                  "Error retrying interactivity generation:",
+                  retryError
+                )
+                await updateLessonStatusToError(courseSlug, lessonUID, bucket)
+                emitToCourse(courseSlug, "course-creation", {
+                  lesson: exSlug,
+                  status: "error",
+                  log: `❌ Failed to retry interactivity generation for lesson ${exSlug}`,
+                })
+              }
+
+              return res.json({ status: "ERROR" })
+            }
+
+            try {
+              let finalContent: string = response.parsed?.final_content
+              let usedComponents: unknown = response.parsed?.used_components
+
+              // The structured template (v2) returns component data and
+              // placeholders; the CLI writes the component markdown.
+              if (req.query.tpl === "v2") {
+                if (!Array.isArray(response.parsed?.multiple_choice_question)) {
+                  console.warn(
+                    `Structured interactivity response for ${exSlug} has no component lists`
+                  )
+                }
+
+                const guide = componentsGuideLoader.peek()
+                const result = processInteractivityResult(response.parsed, {
+                  stepType: exercise.type,
+                  isFirstOrLast:
+                    exerciseIndex === 0 ||
+                    exerciseIndex === syllabus.lessons.length - 1,
+                  lessonId: exercise.id,
+                  decidedBy: guide ? decidedByFromGuide(guide) : undefined,
+                })
+                finalContent = result.markdown
+                usedComponents = result.usedComponents
+
+                if (result.warnings.length > 0) {
+                  console.warn(
+                    `Component warnings for ${exSlug}:\n- ${result.warnings
+                      .map(warning => warning.message)
+                      .join("\n- ")}`
+                  )
+                }
+
+                if (result.dropped.length > 0) {
+                  console.warn(
+                    `Components removed from ${exSlug}:\n- ${result.dropped
+                      .map(issue => issue.message)
+                      .join("\n- ")}`
+                  )
+                  emitToCourse(courseSlug, "course-creation", {
+                    lesson: exSlug,
+                    status: "generating",
+                    log: `⚠️ ${
+                      result.dropped.length
+                    } component(s) removed from lesson ${
+                      exercise.title
+                    } because they could not be rendered: ${result.dropped
+                      .map(issue => issue.message)
+                      .join("; ")}`,
+                  })
+                }
+
+                if (exercise.type === "QUIZ" && result.assessmentCount === 0) {
+                  emitToCourse(courseSlug, "course-creation", {
+                    lesson: exSlug,
+                    status: "generating",
+                    log: `⚠️ The quiz ${exercise.title} has no questions; review it or regenerate it`,
+                  })
+                }
+              }
+
+              // Process readability of final content
+              const readability = checkReadability(
+                finalContent,
+                PARAMS.max_words,
+                3
+              )
+
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: exSlug,
+                status: "generating",
+                log: `🔄 The lesson ${exercise.title} has a readability score of ${readability.fkglResult.fkgl}`,
+              })
+
+              // Upload README with final content
+              const exercisesDir = `courses/${courseSlug}/exercises`
+              const targetDir = `${exercisesDir}/${exSlug}`
+              const readmeFilename = `README${getReadmeExtension(
+                response.parsed.output_language || language
+              )}`
+
+              await uploadFileToBucket(
+                bucket,
+                readability.newMarkdown,
+                `${targetDir}/${readmeFilename}`
+              )
+
+              // Update used components (v1: listed by the AI; v2: the
+              // components that were actually written)
+              if (Array.isArray(usedComponents)) {
+                await updateUsedComponents(courseSlug, usedComponents, bucket)
+                emitToCourse(courseSlug, "course-creation", {
+                  lesson: exSlug,
+                  status: "generating",
+                  log: `📊 Updated component usage tracking: ${usedComponents.join(
+                    ", "
+                  )}`,
+                })
+              }
+
+              // Continue with next lesson
+              await continueWithNextLesson(
+                courseSlug,
+                exerciseIndex,
+                rigoToken,
+                finalContent,
+                bucket
+              )
+
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: exSlug,
+                status: "done",
+                log: `✅ The lesson ${exercise.id} - ${exercise.title} has been generated successfully with interactivity!`,
+              })
+
+              res.json({ status: "SUCCESS" })
+            } catch (error) {
+              console.error("Error processing interactivity webhook:", error)
+              await updateLessonStatusToError(courseSlug, lessonUID, bucket)
+              emitToCourse(courseSlug, "course-creation", {
+                lesson: exSlug,
+                status: "error",
+                log: `❌ Error processing interactivity for lesson ${exSlug}`,
+              })
+              res
+                .status(500)
+                .json({ status: "ERROR", error: (error as Error).message })
+            }
           })
-          res
-            .status(500)
-            .json({ status: "ERROR", error: (error as Error).message })
+        } catch (error) {
+          console.error("Error handling interactivity webhook:", error)
+          if (!res.headersSent) {
+            res
+              .status(500)
+              .json({ status: "ERROR", error: (error as Error).message })
+          }
         }
       }
     )
@@ -3855,42 +4158,43 @@ class ServeCommand extends SessionCommand {
         uid: stepSlug,
       }
 
-      const newLessons = insertStepInCorrectPosition(
-        initialSyllabus.lessons,
-        newLesson
-      )
-      // Use new two-phase generation workflow
-      const completionId = await startInitialContentGeneration(
-        rigoToken,
-        newLessons,
-        initialSyllabus.courseInfo,
-        newLesson,
-        courseSlug,
-        initialSyllabus.courseInfo.purpose,
-        "lastResult"
-      )
-      newLesson.translations = {
-        [initialSyllabus.courseInfo.language || "en"]: {
-          completionId,
-          startedAt: Date.now(),
-          completedAt: 0,
-        },
-      }
+      // In the course queue, so the webhook of this job finds the new lesson
+      // saved, and the syllabus saved here is not older than the last save.
+      await serializeByCourse(courseSlug, async () => {
+        const syllabus = await getSyllabus(courseSlug, bucket)
+        const newLessons = insertStepInCorrectPosition(
+          syllabus.lessons,
+          newLesson
+        )
+        // Use new two-phase generation workflow
+        const completionId = await startInitialContentGeneration(
+          rigoToken,
+          newLessons,
+          syllabus.courseInfo,
+          newLesson,
+          courseSlug,
+          syllabus.courseInfo.purpose,
+          "lastResult"
+        )
+        newLesson.translations = {
+          [syllabus.courseInfo.language || "en"]: startedJob(completionId),
+        }
 
-      await uploadFileToBucket(
-        bucket,
-        JSON.stringify({ ...initialSyllabus, lessons: newLessons }),
-        `courses/${courseSlug}/.learn/initialSyllabus.json`
-      )
+        await uploadFileToBucket(
+          bucket,
+          JSON.stringify({ ...syllabus, lessons: newLessons }),
+          `courses/${courseSlug}/.learn/initialSyllabus.json`
+        )
 
-      const targetDir = `courses/${courseSlug}/exercises/${stepSlug}`
+        const targetDir = `courses/${courseSlug}/exercises/${stepSlug}`
 
-      await uploadInitialReadme(
-        bucket,
-        stepSlug,
-        targetDir,
-        initialSyllabus.courseInfo
-      )
+        await uploadInitialReadme(
+          bucket,
+          stepSlug,
+          targetDir,
+          syllabus.courseInfo
+        )
+      })
 
       res.json({ status: "SUCCESS", message: "Exercise generati on started!" })
     })
@@ -5214,11 +5518,19 @@ class ServeCommand extends SessionCommand {
         syllabus.courseInfo.language
       )
 
+      const language = syllabus.courseInfo.language || "en"
       const initialSyllabus = {
         ...syllabus,
         lessons: syllabus.lessons.map((lesson, index) => {
           if (index < 1) {
-            return { ...lesson, generated: false, status: "GENERATING" }
+            // Its start time, so a retry right away is refused even before
+            // the job id is saved below.
+            return {
+              ...lesson,
+              generated: false,
+              status: "GENERATING",
+              translations: { [language]: startedJob(0) },
+            }
           }
 
           return {
@@ -5243,25 +5555,25 @@ class ServeCommand extends SessionCommand {
 
       const lastResult = "---"
 
-      // Use new two-phase generation workflow
-      const completionId = await startInitialContentGeneration(
-        rigoToken,
-        syllabus.lessons,
-        syllabus.courseInfo,
-        firstLesson,
-        courseSlug,
-        syllabus.courseInfo.purpose,
-        lastResult
-      )
-      if (firstLesson) {
-        firstLesson.translations = {
-          [syllabus.courseInfo.language || "en"]: {
-            completionId,
-            startedAt: Date.now(),
-            completedAt: 0,
-          },
-        }
-      }
+      // In the course queue, so the webhook of this job waits until its id
+      // is saved. The id was never saved before, so the first lesson had no
+      // protection against a second job.
+      await serializeByCourse(courseSlug, async () => {
+        // Use new two-phase generation workflow
+        const completionId = await startInitialContentGeneration(
+          rigoToken,
+          syllabus.lessons,
+          syllabus.courseInfo,
+          firstLesson,
+          courseSlug,
+          syllabus.courseInfo.purpose,
+          lastResult
+        )
+
+        await patchLesson(courseSlug, firstLesson.uid, bucket, lesson => {
+          lesson.translations = { [language]: startedJob(completionId) }
+        })
+      })
 
       await createInitialReadme(
         JSON.stringify(syllabus.courseInfo),