@learnpack/learnpack 5.0.130 → 5.0.132

package/src/commands/serve.ts

@@ -10,21 +10,260 @@ import * as mkdirp from "mkdirp"
 import { convert } from "html-to-text"
 import * as rimraf from "rimraf"
 import SessionCommand from "../utils/SessionCommand"
-import { Storage } from "@google-cloud/storage"
+import { Bucket, Storage } from "@google-cloud/storage"
 import { downloadEditor, decompress } from "../managers/file"
 import * as fs from "fs"
-import { translateExercise, isValidRigoToken } from "../utils/rigoActions"
+import {
+  createCodeFile,
+  translateExercise,
+  isValidRigoToken,
+  readmeCreator,
+  makeReadmeReadable,
+  generateImage,
+} from "../utils/rigoActions"
 import * as dotenv from "dotenv"
+import {
+  extractImagesFromMarkdown,
+  getFilenameFromUrl,
+} from "../utils/creatorUtilities"
 // import { handleAssetCreation } from "./publish"
 import axios from "axios"
 import * as FormData from "form-data"
 import { RIGOBOT_HOST } from "../utils/api"
 import { minutesToISO8601Duration } from "../utils/misc"
 import { buildConfig, ConfigResponse } from "../utils/configBuilder"
+import { checkReadability, slugify } from "../utils/creatorUtilities"
+const frontMatter = require("front-matter")
 
 dotenv.config()
 
-const frontMatter = require("front-matter")
+export const createLearnJson = (courseInfo: FormState) => {
+  console.log("courseInfo to create learn json", courseInfo)
+
+  const learnJson = {
+    slug: slugify(courseInfo.title as string),
+    title: {
+      us: courseInfo.title,
+    },
+    technologies: courseInfo.technologies || [],
+    difficulty: "beginner",
+    description: {
+      us: courseInfo.description,
+    },
+    grading: "isolated",
+    telemetry: {
+      batch: "https://breathecode.herokuapp.com/v1/assignment/me/telemetry",
+    },
+    preview: "preview.png",
+  }
+  return learnJson
+}
+
+const uploadFileToBucket = async (
+  bucket: Bucket,
+  file: string,
+  path: string
+) => {
+  const fileRef = bucket.file(path)
+  await fileRef.save(file)
+}
+
+const uploadImageToBucket = async (
+  bucket: Bucket,
+  url: string,
+  path: string
+) => {
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`Failed to download image: ${response.statusText}`)
+  }
+
+  const contentType =
+    response.headers.get("content-type") || "application/octet-stream"
+  const buffer = await response.arrayBuffer()
+
+  const fileRef = bucket.file(path)
+  await fileRef.save(Buffer.from(buffer), {
+    resumable: false,
+    contentType,
+  })
+}
+
+const PARAMS = {
+  expected_grade_level: "7",
+  max_fkgl: 9,
+  max_words: 200,
+  max_rewrite_attempts: 2,
+  max_title_length: 50,
+}
+
+export const processImage = async (
+  bucket: Bucket,
+  tutorialDir: string,
+  url: string,
+  description: string,
+  rigoToken: string
+) => {
+  try {
+    const filename = getFilenameFromUrl(url)
+
+    const imagePath = `${tutorialDir}/.learn/assets/${filename}`
+
+    console.log("🖼️ Generating image", imagePath)
+
+    const res = await generateImage(rigoToken, { prompt: description })
+    await uploadImageToBucket(bucket, res.image_url, imagePath)
+
+    console.log("✅ Image", imagePath, "generated successfully!")
+    return true
+  } catch {
+    return false
+  }
+}
+
+export async function processExercise(
+  bucket: Bucket,
+  rigoToken: string,
+  steps: Lesson[],
+  packageContext: string,
+  exercise: Lesson,
+  exercisesDir: string
+): Promise<string> {
+  //   const tid = toast.loading("Generating lesson...")
+  const exSlug = slugify(exercise.id + "-" + exercise.title)
+  const readmeFilename = "README.md"
+  const targetDir = `${exercisesDir}/${exSlug}`
+
+  console.log("✍🏻 Generating lesson", exercise.id, exercise.title)
+  const isGeneratingText = `The lesson ${exercise.id} - ${exercise.title} is being generated by Rigobot, wait a sec!
+  
+  \`\`\`loader slug="${exSlug}"
+  # This lesson is being generated by Rigobot, wait a sec!
+  \`\`\`
+  `
+
+  await uploadFileToBucket(
+    bucket,
+    isGeneratingText,
+    `${targetDir}/${readmeFilename}`
+  )
+
+  const readme = await readmeCreator(rigoToken, {
+    title: `${exercise.id} - ${exercise.title}`,
+    output_lang: "en",
+    list_of_exercises: JSON.stringify(steps),
+    tutorial_description: packageContext,
+    lesson_description: exercise.description,
+    kind: exercise.type.toLowerCase(),
+  })
+
+  const duration = exercise.duration
+  let attempts = 0
+  let readability = checkReadability(readme.parsed.content, 200, duration || 1)
+
+  while (
+    readability.fkglResult.fkgl > PARAMS.max_fkgl &&
+    attempts < PARAMS.max_rewrite_attempts
+  ) {
+    console.log(
+      "🔄 The lesson",
+      exercise.id,
+      exercise.title,
+      "has a readability score of",
+      readability.fkglResult.fkgl
+    )
+
+    // eslint-disable-next-line
+    const reducedReadme = await makeReadmeReadable(rigoToken, {
+      lesson: readability.body,
+      number_of_words: readability.minutes.toString(),
+      expected_number_words: PARAMS.max_words.toString(),
+      fkgl_results: JSON.stringify(readability.fkglResult),
+      expected_grade_level: PARAMS.expected_grade_level,
+    })
+
+    if (!reducedReadme) 
+break
+
+    readability = checkReadability(
+      reducedReadme.parsed.content,
+      PARAMS.max_words,
+      duration || 1
+    )
+
+    attempts++
+  }
+
+  console.log(
+    `✅ After ${attempts} attempts, the lesson ${
+      exercise.title
+    } has a readability score of ${
+      readability.fkglResult.fkgl
+    } using FKGL. And it has ${readability.minutes.toFixed(
+      2
+    )} minutes of reading time.`
+  )
+
+  await uploadFileToBucket(
+    bucket,
+    readability.newMarkdown,
+    `${targetDir}/${readmeFilename}`
+  )
+
+  if (exercise.type.toLowerCase() === "code") {
+    console.log("🔍 Creating code file for", exercise.title)
+
+    const codeFile = await createCodeFile(rigoToken, {
+      readme: readability.newMarkdown,
+      tutorial_info: packageContext,
+    })
+    await uploadFileToBucket(
+      bucket,
+      codeFile.parsed.content,
+      `${targetDir}/index.${codeFile.parsed.extension.replace(".", "")}`
+    )
+  }
+
+  return readability.newMarkdown
+}
+
+interface Lesson {
+  id: string
+  uid: string
+  title: string
+  type: "READ" | "CODE" | "QUIZ"
+  description: string
+  duration?: number
+}
+
+interface ParsedLink {
+  url: string
+  title?: string
+  text?: string
+  transcript?: string
+  description?: string
+  author?: string
+  duration?: number
+  thumbnail?: string
+}
+type FormState = {
+  description: string
+  duration: number
+  targetAudience: string
+  hasContentIndex: boolean
+  contentIndex: string
+  sources: ParsedLink[]
+  isCompleted: boolean
+  variables: string[]
+  currentStep: string
+  title: string
+  technologies?: string[]
+}
+
+type Syllabus = {
+  lessons: Lesson[]
+  courseInfo: FormState
+}
 
 const fixPreviewUrl = (slug: string, previewUrl: string) => {
   if (!previewUrl) {
@@ -151,7 +390,7 @@ export default class ServeCommand extends SessionCommand {
 
     app.post(
       "/upload-image",
-      express.json({ limit: "20mb" }),
+      express.json({ limit: 20_000_000 }),
       async (req, res) => {
         const { image_url, destination } = req.body
         if (!image_url || !destination) {
@@ -428,6 +667,64 @@ export default class ServeCommand extends SessionCommand {
       res.sendFile(file)
     })
 
+    app.post("/actions/create-course", async (req, res) => {
+      console.log("POST /actions/create-course")
+      const { syllabus }: { syllabus: Syllabus } = req.body
+      const rigoToken = req.header("x-rigo-token")
+      const bcToken = req.header("x-breathecode-token")
+
+      if (!rigoToken || !bcToken) {
+        return res.status(400).json({ error: "Missing tokens" })
+      }
+
+      const tutorialDir = "courses/" + slugify(syllabus.courseInfo.title)
+
+      const learnJson = createLearnJson(syllabus.courseInfo)
+      await uploadFileToBucket(
+        bucket,
+        JSON.stringify(learnJson),
+        `courses/${slugify(syllabus.courseInfo.title)}/learn.json`
+      )
+
+      const lessonsPromises = syllabus.lessons.map(lesson =>
+        processExercise(
+          bucket,
+          rigoToken,
+          syllabus.lessons,
+          JSON.stringify(syllabus.courseInfo),
+          lesson,
+          tutorialDir + "/exercises"
+        )
+      )
+      const readmeContents = await Promise.all(lessonsPromises)
+
+      let imagesArray: any[] = []
+
+      for (const content of readmeContents) {
+        imagesArray = [...imagesArray, ...extractImagesFromMarkdown(content)]
+      }
+
+      console.log("📷 Generating images...")
+
+      const imagePromises = imagesArray.map(
+        async (image: { alt: string; url: string }) => {
+          return processImage(
+            bucket,
+            tutorialDir,
+            image.url,
+            image.alt,
+            rigoToken
+          )
+        }
+      )
+      await Promise.all(imagePromises)
+
+      return res.json({
+        message: "Course created",
+        slug: slugify(syllabus.courseInfo.title),
+      })
+    })
+
     app.post("/actions/publish/:slug", async (req, res) => {
       try {
         const { slug } = req.params

package/src/creator/src/components/syllabus/SyllabusEditor.tsx

@@ -4,19 +4,12 @@ import useStore from "../../utils/store"
 import { interactiveCreation } from "../../utils/rigo"
 import {
   parseLesson,
-  uploadFileToBucket,
   useConsumableCall,
   validateTokens,
-  extractImagesFromMarkdown,
   checkParams,
   loginWithToken,
+  createCourse,
 } from "../../utils/lib"
-import {
-  createLearnJson,
-  processExercise,
-  slugify,
-  processImage,
-} from "../../utils/creatorUtils"
 
 import Loader from "../Loader"
 import { TMessage } from "../Message"
@@ -26,10 +19,10 @@ import toast from "react-hot-toast"
 import { ContentIndex } from "./ContentIndex"
 import { Sidebar } from "./Sidebar"
 import Login from "../Login"
-import { eventBus } from "../../utils/eventBus"
 import { useNavigate } from "react-router"
 import PreviewGenerator from "../PreviewGenerator"
 import { ParamsChecker } from "../ParamsChecker"
+import { slugify } from "../../utils/creatorUtils"
 
 const SyllabusEditor: React.FC = () => {
   const navigate = useNavigate()
@@ -138,7 +131,6 @@ const SyllabusEditor: React.FC = () => {
       toast.error("Please provide a title for the course")
       return
     }
-    setIsGenerating(true)
 
     let tokenToUse = auth.rigoToken
     const onValidRigoToken = (rigotoken: string) => {
@@ -159,54 +151,19 @@ const SyllabusEditor: React.FC = () => {
       return
     }
     setIsGenerating(true)
-
-    const tutorialDir = "courses/" + slugify(syllabus.courseInfo.title)
-    const lessonsPromises = syllabus.lessons.map((lesson) =>
-      processExercise(
-        tokenToUse,
-        syllabus.lessons,
-        JSON.stringify(syllabus.courseInfo),
-        lesson,
-        tutorialDir + "/exercises"
-      )
-    )
-    const readmeContents = await Promise.all(lessonsPromises)
-
-    let imagesArray: any[] = []
-
-    for (const content of readmeContents) {
-      imagesArray = [...imagesArray, ...extractImagesFromMarkdown(content)]
-    }
-
-    eventBus.emit("course-generation", {
-      message: "📷 Generating images...",
-    })
-
-    const imagePromises = imagesArray.map(
-      async (image: { alt: string; url: string }) => {
-        return processImage(tutorialDir, image.url, image.alt, tokenToUse)
-      }
-    )
-    await Promise.all(imagePromises)
-
-    const learnJson = createLearnJson(syllabus.courseInfo)
-    await uploadFileToBucket(
-      JSON.stringify(learnJson),
-      "courses/" + slugify(syllabus.courseInfo.title) + "/learn.json"
-    )
-
-    cleanHistory()
-
-    window.location.href = `/preview/${slugify(
-      syllabus.courseInfo.title
-    )}?token=${auth.bcToken}`
-    // setIsGenerating(false)
+    createCourse(syllabus, tokenToUse, auth.bcToken)
+
+    setTimeout(() => {
+      cleanHistory()
+      setIsGenerating(false)
+      window.location.href = `/preview/${slugify(
+        syllabus.courseInfo.title || ""
+      )}?token=${auth.bcToken}`
+    }, 1000)
   }
 
   if (!syllabus) return null
 
-  // console.log(auth.user)
-
   return isGenerating ? (
     <>
       <Loader

package/src/utils/cloudStorage.ts

@@ -1,24 +1,24 @@
-import { Bucket } from "@google-cloud/storage"
-import { Storage } from "@google-cloud/storage"
-import * as path from "path"
-
-/**
- * Sube contenido de texto directamente a un bucket de GCS.
- * @param bucket Instancia de Bucket
- * @param destination Ruta destino dentro del bucket (incluye carpeta/nombre)
- * @param textContent Contenido textual a subir
- */
-export async function uploadTextToBucket(
-  bucket: Bucket,
-  destination: string,
-  textContent: string
-): Promise<void> {
-  const file = bucket.file(destination)
-
-  await file.save(textContent, {
-    resumable: false,
-    contentType: "text/plain",
-  })
-
-  console.log(`✅ Texto subido a gs://${bucket.name}/${destination}`)
-}
+import { Bucket } from "@google-cloud/storage"
+import { Storage } from "@google-cloud/storage"
+import * as path from "path"
+
+/**
+ * Sube contenido de texto directamente a un bucket de GCS.
+ * @param bucket Instancia de Bucket
+ * @param destination Ruta destino dentro del bucket (incluye carpeta/nombre)
+ * @param textContent Contenido textual a subir
+ */
+export async function uploadTextToBucket(
+  bucket: Bucket,
+  destination: string,
+  textContent: string
+): Promise<void> {
+  const file = bucket.file(destination)
+
+  await file.save(textContent, {
+    resumable: false,
+    contentType: "text/plain",
+  })
+
+  console.log(`✅ Texto subido a gs://${bucket.name}/${destination}`)
+}