@learnpack/learnpack 5.0.204 → 5.0.209

package/src/commands/serve.ts

@@ -23,11 +23,11 @@ import { Bucket, Storage } from "@google-cloud/storage"
 import { downloadEditor, decompress } from "../managers/file"
 import * as fs from "fs"
 import {
-  createCodeFile,
+  // createCodeFile,
   translateExercise,
   isValidRigoToken,
   readmeCreator,
-  makeReadmeReadable,
+  // makeReadmeReadable,
   generateImage,
   isPackageAuthor,
 } from "../utils/rigoActions"
@@ -50,28 +50,19 @@ const frontMatter = require("front-matter")
 
 dotenv.config()
 
-const validateLanguage = (language: string) => {
-  if (!language || language.length !== 2) {
-    return "en"
+function findLast<T>(
+  array: T[],
+  predicate: (item: T) => boolean
+): T | undefined {
+  for (let i = array.length - 1; i >= 0; i--) {
+    if (predicate(array[i])) return array[i]
   }
 
-  return language
-}
-
-function fixSlugLength(slug: string): string {
-  let clean = slug.toLowerCase()
-  clean = clean.replace(/[^\da-z-]+/g, "-")
-  clean = clean.replace(/-+/g, "-")
-  clean = clean.slice(0, 49)
-  clean = clean.replace(/^-+/, "").replace(/-+$/, "")
-  clean = clean.replace(/^[^\da-z]+/, "").replace(/[^\da-z]+$/, "")
-  if (!clean) throw new Error("Invalid slug after cleaning")
-
-  return clean
+  return undefined
 }
 
 export const createLearnJson = (courseInfo: FormState) => {
-  console.log("courseInfo to create learn json", courseInfo)
+  // console.log("courseInfo to create learn json", courseInfo)
 
   const expectedPreviewUrl = `https://${slugify(
     courseInfo.title as string
@@ -167,27 +158,118 @@ export const processImage = async (
   }
 }
 
-const createInitialSidebar = async (slugs: string[]) => {
+const createInitialSidebar = async (
+  slugs: string[],
+  initialLanguage = "en"
+) => {
+  const language = initialLanguage === "en" ? "us" : initialLanguage
+
   const sidebar: Record<string, Record<string, string>> = {}
   for (const slug of slugs) {
     sidebar[slug] = {
-      us: slug,
+      [language]: slug,
     }
   }
 
   return sidebar
 }
 
+const uploadInitialReadme = async (
+  bucket: Bucket,
+  exSlug: string,
+  targetDir: string,
+  packageContext: FormState
+) => {
+  const isGeneratingText = `
+  \`\`\`loader slug="${exSlug}"
+  :rigo
+  \`\`\`
+      `
+  const readmeFilename = `README.${
+    packageContext.language && packageContext.language !== "en" ?
+      `${packageContext.language}.` :
+      ""
+  }md`
+
+  await uploadFileToBucket(
+    bucket,
+    isGeneratingText,
+    `${targetDir}/${readmeFilename}`
+  )
+}
+
+const cleanFormState = (formState: FormState) => {
+  // keysToDelete: description, technologies, purpose
+  const {
+    description,
+    technologies,
+    purpose,
+    hasContentIndex,
+    duration,
+    isCompleted,
+    variables,
+    currentStep,
+    language,
+    ...rest
+  } = formState
+  return rest
+}
+
+async function startExerciseGeneration(
+  bucket: Bucket,
+  rigoToken: string,
+  steps: Lesson[],
+  packageContext: FormState,
+  exercise: Lesson,
+  tutorialDir: string,
+  courseSlug: string,
+  purposeSlug: string,
+  lastLesson = ""
+): Promise<void> {
+  const exercisesDir = `${tutorialDir}/exercises`
+
+  const exSlug = slugify(exercise.id + "-" + exercise.title)
+  console.log("exSlug", exSlug)
+
+  const readmeFilename = `README.${
+    packageContext.language && packageContext.language !== "en" ?
+      `${packageContext.language}.` :
+      ""
+  }md`
+
+  const webhookUrl = `${process.env.HOST}/webhooks/${courseSlug}/exercise-processor/${exercise.id}/${rigoToken}`
+
+  const res = await readmeCreator(
+    rigoToken,
+    {
+      title: `${exercise.id} - ${exercise.title}`,
+      output_lang: packageContext.language || "en",
+      list_of_exercises: JSON.stringify(
+        steps.map(step => step.id + "-" + step.title)
+      ),
+      tutorial_description: JSON.stringify(cleanFormState(packageContext)),
+      lesson_description: exercise.description,
+      kind: exercise.type.toLowerCase(),
+      last_lesson: lastLesson,
+    },
+    purposeSlug,
+    webhookUrl
+  )
+  // console.log("res processing in background", res)
+}
+
 export async function processExercise(
   bucket: Bucket,
   rigoToken: string,
   steps: Lesson[],
   packageContext: FormState,
   exercise: Lesson,
-  exercisesDir: string,
+  tutorialDir: string,
   courseSlug: string,
-  purposeSlug: string
+  purposeSlug: string,
+  lastLesson = ""
 ): Promise<string> {
+  const exercisesDir = `${tutorialDir}/exercises`
   //   const tid = toast.loading("Generating lesson...")
   const exSlug = slugify(exercise.id + "-" + exercise.title)
   console.log("exSlug", exSlug)
@@ -201,99 +283,102 @@ export async function processExercise(
 
   console.log("✍🏻 Generating lesson", exercise.id, exercise.title)
 
-  const isGeneratingText = `
-  \`\`\`loader slug="${exSlug}"
-  :rigo
-  \`\`\`
-      `
-
-  await uploadFileToBucket(
-    bucket,
-    isGeneratingText,
-    `${targetDir}/${readmeFilename}`
-  )
-
   const readme = await readmeCreator(
     rigoToken,
     {
       title: `${exercise.id} - ${exercise.title}`,
       output_lang: packageContext.language || "en",
-      list_of_exercises: JSON.stringify(steps),
-      tutorial_description: JSON.stringify(packageContext),
+      list_of_exercises: JSON.stringify(
+        steps.map(step => step.id + "-" + step.title)
+      ),
+      tutorial_description: JSON.stringify(cleanFormState(packageContext)),
       lesson_description: exercise.description,
       kind: exercise.type.toLowerCase(),
+      last_lesson: lastLesson,
     },
     purposeSlug
   )
 
   const duration = exercise.duration
-  let attempts = 0
-  let readability = checkReadability(readme.parsed.content, 250, duration || 3)
-
-  while (
-    readability.fkglResult.fkgl > PARAMS.max_fkgl &&
-    attempts < PARAMS.max_rewrite_attempts
-  ) {
-    emitToCourse(courseSlug, "course-creation", {
-      lesson: exSlug,
-      status: "generating",
-      log: `🔄 The lesson ${exercise.id} - ${exercise.title} has a readability score of ${readability.fkglResult.fkgl}`,
-    })
-
-    // eslint-disable-next-line
-    const reducedReadme = await makeReadmeReadable(
-      rigoToken,
-      {
-        lesson: readability.body,
-        number_of_words: readability.minutes.toString(),
-        expected_number_words: PARAMS.max_words.toString(),
-        fkgl_results: JSON.stringify(readability.fkglResult),
-        expected_grade_level: PARAMS.expected_grade_level,
-      },
-      purposeSlug
-    )
-
-    if (!reducedReadme) break
-
-    readability = checkReadability(
-      reducedReadme.parsed.content,
-      PARAMS.max_words,
-      duration || 2
-    )
+  // let attempts = 0
+  const readability = checkReadability(
+    readme.parsed.content,
+    PARAMS.max_words,
+    duration || 3
+  )
 
-    attempts++
-  }
+  emitToCourse(courseSlug, "course-creation", {
+    lesson: exSlug,
+    status: "generating",
+    log: `🔄 The lesson ${exercise.id} - ${exercise.title} has a readability score of ${readability.fkglResult.fkgl}`,
+  })
+  // while (
+  //   readability.fkglResult.fkgl > PARAMS.max_fkgl &&
+  //   attempts < PARAMS.max_rewrite_attempts
+  // ) {
+
+  //   // eslint-disable-next-line
+  //   const reducedReadme = await makeReadmeReadable(
+  //     rigoToken,
+  //     {
+  //       lesson: readability.body,
+  //       number_of_words: readability.minutes.toString(),
+  //       expected_number_words: PARAMS.max_words.toString(),
+  //       fkgl_results: JSON.stringify(readability.fkglResult),
+  //       expected_grade_level: PARAMS.expected_grade_level,
+  //     },
+  //     purposeSlug
+  //   )
+
+  //   if (!reducedReadme) break
+
+  //   readability = checkReadability(
+  //     reducedReadme.parsed.content,
+  //     PARAMS.max_words,
+  //     duration || 3
+  //   )
+
+  //   attempts++
+  // }
 
   await uploadFileToBucket(
     bucket,
     readability.newMarkdown,
     `${targetDir}/${readmeFilename}`
   )
-  emitToCourse(courseSlug, "course-creation", {
-    lesson: exSlug,
-    status: "done",
-    log: `✅ The lesson ${exercise.id} - ${exercise.title} has been generated successfully!`,
-  })
 
-  if (exercise.type.toLowerCase() === "code") {
+  if (
+    exercise.type.toLowerCase() === "code" &&
+    readme.parsed.codefile_content
+  ) {
     console.log("🔍 Creating code file for", exercise.title)
 
-    const codeFile = await createCodeFile(rigoToken, {
-      readme: readability.newMarkdown,
-      tutorial_info: JSON.stringify(packageContext),
-    })
     await uploadFileToBucket(
       bucket,
-      codeFile.parsed.content,
-      `${targetDir}/index.${codeFile.parsed.extension.replace(".", "")}`
+      readme.parsed.codefile_content,
+      `${targetDir}/${readme.parsed.codefile_name.toLowerCase().trim()}`
     )
+  }
+
+  const imagesArray: any[] = extractImagesFromMarkdown(readability.newMarkdown)
+
+  if (imagesArray.length > 0) {
     emitToCourse(courseSlug, "course-creation", {
       lesson: exSlug,
       status: "done",
-      log: `✅ The code file for ${exercise.title} has been generated successfully!`,
+      log: `🔄 Generating images for ${exercise.title}`,
     })
+    for (const image of imagesArray) {
+      // eslint-disable-next-line no-await-in-loop
+      await processImage(bucket, tutorialDir, image.url, image.alt, rigoToken)
+    }
   }
 
+  emitToCourse(courseSlug, "course-creation", {
+    lesson: exSlug,
+    status: "done",
+    log: `✅ The lesson ${exercise.id} - ${exercise.title} has been generated successfully!`,
+  })
   return readability.newMarkdown
 }
 
@@ -352,6 +437,15 @@ export default class ServeCommand extends SessionCommand {
       process.env.GCP_BUCKET_NAME || "learnpack-packages"
     )
 
+    const host = process.env.HOST
+
+    if (!host) {
+      console.log("HOST is not set")
+      process.exit(1)
+    } else {
+      console.log("HOST is set to", host)
+    }
+
     // async function listFilesWithPrefix(prefix: string) {
     //   const [files] = await bucket.getFiles({ prefix })
     //   return files
@@ -555,6 +649,154 @@ export default class ServeCommand extends SessionCommand {
       res.json({ id, status: "SUCCESS" })
     })
 
+    app.post(
+      "/webhooks/:courseSlug/exercise-processor/:lessonID/:rigoToken",
+      async (req, res) => {
+        // console.log("Receiving a webhook to exercise processor")
+        const { courseSlug, lessonID, rigoToken } = req.params
+        const readme = req.body
+
+        const syllabus = await bucket.file(
+          `courses/${courseSlug}/.learn/initialSyllabus.json`
+        )
+        const [content] = await syllabus.download()
+        const syllabusJson: Syllabus = JSON.parse(content.toString())
+        const exerciseIndex = syllabusJson.lessons.findIndex(
+          lesson => lesson.id === lessonID
+        )
+        if (exerciseIndex === -1) {
+          console.log(
+            "Exercise not found receiving webhook, this should not happen",
+            lessonID
+          )
+          return res.json({ status: "ERROR", error: "Exercise not found" })
+        }
+
+        const exercise = syllabusJson.lessons[exerciseIndex]
+
+        const nextExercise = syllabusJson.lessons[exerciseIndex + 1] || null
+
+        if (!exercise) {
+          console.log(
+            "Exercise not found receiving webhook, this should not happen",
+            lessonID
+          )
+          return res.json({ status: "SUCCESS" })
+        }
+
+        const exSlug = slugify(exercise.id + "-" + exercise.title)
+
+        const readability = checkReadability(
+          readme.parsed.content,
+          PARAMS.max_words,
+          3
+        )
+
+        emitToCourse(courseSlug, "course-creation", {
+          lesson: exSlug,
+          status: "generating",
+          log: `🔄 The lesson ${exercise.title} has a readability score of ${readability.fkglResult.fkgl}`,
+        })
+
+        const exercisesDir = `courses/${courseSlug}/exercises`
+        const targetDir = `${exercisesDir}/${exSlug}`
+
+        const readmeFilename = `README.${
+          readme.parsed.language_code && readme.parsed.language_code !== "en" ?
+            `${readme.parsed.language_code}.` :
+            ""
+        }md`
+
+        await uploadFileToBucket(
+          bucket,
+          readability.newMarkdown,
+          `${targetDir}/${readmeFilename}`
+        )
+
+        if (
+          exercise.type.toLowerCase() === "code" &&
+          readme.parsed.codefile_content
+        ) {
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: exSlug,
+            status: "generating",
+            log: `🔄 Creating code file for ${exercise.title}`,
+          })
+
+          await uploadFileToBucket(
+            bucket,
+            readme.parsed.codefile_content,
+            `${targetDir}/${readme.parsed.codefile_name.toLowerCase().trim()}`
+          )
+        }
+
+        if (nextExercise) {
+          startExerciseGeneration(
+            bucket,
+            rigoToken,
+            syllabusJson.lessons,
+            syllabusJson.courseInfo,
+            nextExercise,
+            `courses/${courseSlug}`,
+            courseSlug,
+            syllabusJson.courseInfo.purpose,
+            readme.parsed.content
+          )
+        }
+
+        const imagesArray: any[] = extractImagesFromMarkdown(
+          readability.newMarkdown
+        )
+
+        if (imagesArray.length > 0) {
+          console.log(
+            "This course requires images and I don't have the token :)"
+          )
+
+          emitToCourse(courseSlug, "course-creation", {
+            lesson: exSlug,
+            status: "pending",
+            log: `🔄 Generating images for ${exercise.title}`,
+          })
+          for (const image of imagesArray) {
+            // eslint-disable-next-line no-await-in-loop
+            await processImage(
+              bucket,
+              `courses/${courseSlug}`,
+              image.url,
+              image.alt,
+              rigoToken
+            )
+          }
+        }
+
+        emitToCourse(courseSlug, "course-creation", {
+          lesson: exSlug,
+          status: "done",
+          log: `✅ The lesson ${exercise.id} - ${exercise.title} has been generated successfully!`,
+        })
+
+        const newSyllabus = {
+          ...syllabusJson,
+          lessons: syllabusJson.lessons.map(lesson => {
+            if (lesson.id === exercise.id) {
+              return { ...lesson, generated: true }
+            }
+
+            return lesson
+          }),
+        }
+
+        await uploadFileToBucket(
+          bucket,
+          JSON.stringify(newSyllabus),
+          `courses/${courseSlug}/.learn/initialSyllabus.json`
+        )
+
+        res.json({ status: "SUCCESS" })
+      }
+    )
+
     app.get("/check-preview-image/:slug", async (req, res) => {
       const { slug } = req.params
       const file = bucket.file(`courses/${slug}/preview.png`)
@@ -931,76 +1173,95 @@ export default class ServeCommand extends SessionCommand {
         return res.status(400).json({ error: "Missing tokens" })
       }
 
-      const slug = slugify(syllabus.courseInfo.title)
+      const courseSlug = slugify(syllabus.courseInfo.title)
 
-      const tutorialDir = `courses/${slug}`
+      const tutorialDir = `courses/${courseSlug}`
       const learnJson = createLearnJson(syllabus.courseInfo)
 
       try {
-        await api.createRigoPackage(rigoToken, slug, learnJson)
+        await api.createRigoPackage(rigoToken, courseSlug, learnJson)
       } catch (error) {
         console.error("Failed to create Rigo package:", error)
         return res.status(400).json({ error: "Failed to create Rigo package" })
       }
 
-      await uploadFileToBucket(
-        bucket,
-        JSON.stringify(learnJson),
-        `${tutorialDir}/learn.json`
-      )
-      console.log("🔄 Learn.json uploaded to bucket to", tutorialDir)
-
-      console.log(
-        "🔄 Processing lessons with purpose",
-        syllabus.courseInfo.purpose
-      )
-      const lessonsPromises = syllabus.lessons.map(lesson =>
-        processExercise(
+      try {
+        await uploadFileToBucket(
           bucket,
-          rigoToken,
-          syllabus.lessons,
-          syllabus.courseInfo,
-          lesson,
-          tutorialDir + "/exercises",
-          slugify(syllabus.courseInfo.title),
-          syllabus.courseInfo.purpose
+          JSON.stringify(learnJson),
+          `${tutorialDir}/learn.json`
         )
-      )
-      const readmeContents = await Promise.all(lessonsPromises)
+      } catch (error) {
+        console.error("Failed to upload learn.json:", error)
+        return res.status(400).json({ error: "Failed to upload learn.json" })
+      }
 
-      let imagesArray: any[] = []
+      console.log(
+        "🔄 Learn.json and initialSyllabus.json uploaded to bucket to",
+        tutorialDir
+      )
 
-      for (const content of readmeContents) {
-        imagesArray = [...imagesArray, ...extractImagesFromMarkdown(content)]
-      }
+      for (let i = 0; i < syllabus.lessons.length; i++) {
+        const lesson = syllabus.lessons[i]
+        const exSlug = slugify(lesson.id + "-" + lesson.title)
 
-      console.log("📷 Generating images...")
+        const targetDir = `${tutorialDir}/exercises/${exSlug}`
 
-      const imagePromises = imagesArray.map(
-        async (image: { alt: string; url: string }) => {
-          return processImage(
-            bucket,
-            tutorialDir,
-            image.url,
-            image.alt,
-            rigoToken
-          )
-        }
-      )
-      await Promise.all(imagePromises)
+        // eslint-disable-next-line no-await-in-loop
+        await uploadInitialReadme(
+          bucket,
+          exSlug,
+          targetDir,
+          syllabus.courseInfo
+        )
+      }
 
       const sidebar = await createInitialSidebar(
         syllabus.lessons.map(lesson =>
           slugify(lesson.id + "-" + lesson.title)
-        )
+        ),
+        syllabus.courseInfo.language
       )
+
+      const initialSyllabus = {
+        ...syllabus,
+        lessons: syllabus.lessons.map((lesson, index) => {
+          if (index < 1) {
+            return { ...lesson, generated: true }
+          }
+
+          return {
+            ...lesson,
+            generated: false,
+          }
+        }),
+      }
+      await uploadFileToBucket(
+        bucket,
+        JSON.stringify(initialSyllabus),
+        `${tutorialDir}/.learn/initialSyllabus.json`
+      )
+
       await uploadFileToBucket(
         bucket,
         JSON.stringify(sidebar),
         `${tutorialDir}/.learn/sidebar.json`
       )
+      const firstLesson = syllabus.lessons[0]
 
-      console.log("SIDEBAR UPLOADED", sidebar)
+      const lastResult = "Nothing"
+
+      startExerciseGeneration(
+        bucket,
+        rigoToken,
+        syllabus.lessons,
+        syllabus.courseInfo,
+        firstLesson,
+        tutorialDir,
+        courseSlug,
+        syllabus.courseInfo.purpose,
+        lastResult
+      )
 
       return res.json({
         message: "Course created",
@@ -1008,6 +1269,71 @@ export default class ServeCommand extends SessionCommand {
       })
     })
 
+    app.get("/courses/:courseSlug/syllabus", async (req, res) => {
+      try {
+        console.log("GET /courses/:courseSlug/syllabus")
+        const { courseSlug } = req.params
+        const syllabus = await bucket.file(
+          `courses/${courseSlug}/.learn/initialSyllabus.json`
+        )
+        const [content] = await syllabus.download()
+        const syllabusJson: Syllabus = JSON.parse(content.toString())
+        res.json(syllabusJson)
+      } catch (error) {
+        console.error("Error getting syllabus:", error)
+        return res.status(500).json({ error: "Error getting syllabus" })
+      }
+    })
+
+    app.post("/actions/continue-course/:courseSlug", async (req, res) => {
+      console.log("POST /actions/continue-course/:courseSlug")
+      const { courseSlug } = req.params
+
+      const { feedback }: { feedback: string } = req.body
+
+      const rigoToken = req.header("x-rigo-token")
+      const bcToken = req.header("x-breathecode-token")
+      if (!rigoToken || !bcToken) {
+        return res.status(400).json({ error: "Missing tokens" })
+      }
+
+      const syllabus = await bucket.file(
+        `courses/${courseSlug}/.learn/initialSyllabus.json`
+      )
+      const [content] = await syllabus.download()
+      const syllabusJson: Syllabus = JSON.parse(content.toString())
+      const notGeneratedLessons = syllabusJson.lessons.filter(
+        lesson => !lesson.generated
+      )
+
+      const lastGeneratedLesson = findLast(
+        syllabusJson.lessons,
+        lesson => lesson.generated ?? false
+      )
+
+      console.log("ABout to generate", notGeneratedLessons.length, "lessons")
+
+      const firstLessonToGenerate = notGeneratedLessons[0]
+
+      await startExerciseGeneration(
+        bucket,
+        rigoToken,
+        syllabusJson.lessons,
+        syllabusJson.courseInfo,
+        firstLessonToGenerate,
+        `courses/${courseSlug}`,
+        courseSlug,
+        syllabusJson.courseInfo.purpose,
+        JSON.stringify(lastGeneratedLesson) +
+          `\n\nThe user provided this feedback in relation to the course: ${feedback}`
+      )
+
+      return res.json({
+        message: "Course continued",
+        slug: courseSlug,
+      })
+    })
+
     app.get(
       "/courses/:courseSlug/exercises/:exerciseSlug/",
       async (req, res) => {
@@ -1069,11 +1395,7 @@ export default class ServeCommand extends SessionCommand {
           slug
         )
 
-        // console.log(slug, "SLUG")
-
-        const fixedSlug = fixSlugLength(slug)
-
-        // console.log(fixedSlug, "FIXED SLUG")
+        // const fixedSlug = fixSlugLength(slug)
 
         const prefix = `courses/${slug}/`
         const tmpRoot = path.join(os.tmpdir(), `learnpack-${slug}`)
@@ -1106,7 +1428,7 @@ export default class ServeCommand extends SessionCommand {
           .replace(/{{description}}/g, config.description.us)
           .replace(
             /{{preview}}/g,
-            fixPreviewUrl(fixedSlug, "") ||
+            fixPreviewUrl(slug, "") ||
               "https://raw.githubusercontent.com/learnpack/ide/master/public/learnpack.svg"
           )
           .replace(/{{slug}}/g, slug)
@@ -1136,9 +1458,9 @@ export default class ServeCommand extends SessionCommand {
 
         // 8) Crear el config.json en buildRoot con lo que retorna buildConfig
         const fullConfig = { config, exercises }
-        fullConfig.config.slug = fixedSlug
+        fullConfig.config.slug = slug
         fullConfig.config.preview =
-          fixPreviewUrl(fixedSlug, "") ||
+          fixPreviewUrl(slug, "") ||
           "https://raw.githubusercontent.com/learnpack/ide/master/public/learnpack.svg"
 
         fs.writeFileSync(
@@ -1148,7 +1470,7 @@ export default class ServeCommand extends SessionCommand {
         )
 
         // 9) Empaquetar en ZIP (solo contenido de buildRoot)
-        const zipName = `${fixedSlug}.zip`
+        const zipName = `${slug}.zip`
         const zipPath = path.join(tmpRoot, zipName)
         const output = fs.createWriteStream(zipPath)
         const archive = archiver("zip", { zlib: { level: 9 } })