@learnpack/learnpack 5.0.355 → 5.0.358

package/src/commands/serve.ts

@@ -26,6 +26,7 @@ import * as fs from "fs"
 import {
   // createCodeFile,
   translateExercise,
+  translateExerciseAndWait,
   isValidRigoToken,
   readmeCreator,
   // makeReadmeReadable,
@@ -51,7 +52,6 @@ import {
 import axios from "axios"
 import * as FormData from "form-data"
 import api, {
-  BREATHECODE_TELEMETRY_URL,
   RIGOBOT_HOST,
   RIGOBOT_REALTIME_HOST,
   listUserAcademies,
@@ -74,7 +74,23 @@ import {
   SyncNotification,
 } from "../models/creator"
 import { exportToScorm, exportToEpub, exportToZip } from "../utils/export"
-import { generateAndPersistPackageManifest } from "../utils/packageManifest"
+import { createIngestRouter } from "../services/ingest/http/router"
+import {
+  uploadFileToBucket,
+  uploadBinaryToBucket,
+} from "../utils/coursePackage/bucketIo"
+import { createLearnJson } from "../utils/coursePackage/learnJson"
+import { createInitialSidebar } from "../utils/coursePackage/sidebar"
+import {
+  generateAndPersistPackageManifest,
+  resolvePreviewUrl,
+} from "../utils/packageManifest"
+import {
+  DRAFT_PREVIEW_DESCRIPTION,
+  DRAFT_PREVIEW_TITLE,
+  getFallbackPreviewUrl,
+  getPublicHost,
+} from "../utils/branding"
 import { requireGcsBucketName } from "../utils/gcsBucketName"
 import {
   createGcsSyllabusSyncStorage,
@@ -161,56 +177,6 @@ async function serializeByCourse<T>(
   return next
 }
 
-export const createLearnJson = (courseInfo: FormState) => {
-  // console.log("courseInfo to create learn json", courseInfo)
-
-  const expectedPreviewUrl = `https://${courseInfo.slug}.learn-pack.com/preview.png`
-
-  const language = courseInfo.language || "en"
-
-  const learnJson = {
-    slug: courseInfo.slug,
-    title: {
-      [language]: courseInfo.title,
-    },
-    technologies: courseInfo.technologies || [],
-    difficulty: "beginner",
-    description: {
-      [language]: courseInfo.description,
-    },
-    grading: "isolated",
-    telemetry: {
-      batch: BREATHECODE_TELEMETRY_URL,
-    },
-    preview: expectedPreviewUrl,
-  }
-  return learnJson
-}
-
-const uploadFileToBucket = async (
-  bucket: Bucket,
-  file: string,
-  path: string
-) => {
-  const isReadme = /readme(\.\w+)?\.md$/i.test(path)
-  const content = isReadme ? sanitizeReadmeNewlines(file) : file
-  const fileRef = bucket.file(path)
-  await fileRef.save(Buffer.from(content, "utf8"))
-}
-
-const uploadBinaryToBucket = async (
-  bucket: Bucket,
-  buffer: Buffer,
-  path: string,
-  contentType?: string
-) => {
-  const fileRef = bucket.file(path)
-  await fileRef.save(buffer, {
-    resumable: false,
-    ...(contentType && { contentType }),
-  })
-}
-
 const getGithubCredentials = () => {
   const token = process.env.GITHUB_TOKEN?.trim()
   const username = process.env.GITHUB_USERNAME?.trim()
@@ -518,20 +484,6 @@ export const processImage = async (
   }
 }
 
-const createInitialSidebar = async (
-  slugs: string[],
-  initialLanguage = "en"
-) => {
-  const sidebar: Record<string, Record<string, string>> = {}
-  for (const slug of slugs) {
-    sidebar[slug] = {
-      [initialLanguage]: slug,
-    }
-  }
-
-  return sidebar
-}
-
 const uploadInitialReadme = async (
   bucket: Bucket,
   exSlug: string,
@@ -1238,17 +1190,43 @@ async function continueWithNextLesson(
   await saveSyllabus(courseSlug, newSyllabus, bucket)
 }
 
-const fixPreviewUrl = (slug: string, previewUrl: string) => {
-  if (!previewUrl) {
-    return null
+/**
+ * The `og:image` of a published course.
+ *
+ * A published package is served from `{slug}.learn-pack.com`, so the course's
+ * own `preview.png` is the right card whenever it exists. This used to go
+ * through a helper that returned null for an empty preview, and both call sites
+ * passed an empty string — so every published course, preview or not, fell back
+ * to a shared SVG that no unfurl crawler can render.
+ */
+const resolveCoursePreviewUrl = async (
+  bucket: Bucket,
+  slug: string,
+  configuredPreview?: string
+): Promise<string> => {
+  // `config` is parsed from a config.json in the bucket, so its declared types
+  // are an assertion rather than a guarantee: an old or hand-edited course can
+  // carry anything here, and a throw would fail the whole publication.
+  if (
+    typeof configuredPreview === "string" &&
+    configuredPreview.startsWith("http")
+  ) {
+    return configuredPreview
   }
 
-  if (previewUrl.startsWith("http")) {
-    return previewUrl
+  try {
+    const [exists] = await bucket.file(`courses/${slug}/preview.png`).exists()
+    if (exists) {
+      return resolvePreviewUrl(slug)
+    }
+  } catch (error) {
+    console.error(
+      `WARNING: could not check preview.png for ${slug}, using the brand card`,
+      error
+    )
   }
 
-  const expectedUrl = `https://${slug}.learn-pack.com/preview.png`
-  return expectedUrl
+  return getFallbackPreviewUrl()
 }
 
 const getTitleFromHTML = (html: string) => {
@@ -1257,6 +1235,101 @@ const getTitleFromHTML = (html: string) => {
   return titleMatch ? titleMatch[1] : null
 }
 
+const escapeHtmlAttribute = (value: string) =>
+  value.replace(/&/g, "&amp;").replace(/"/g, "&quot;").replace(/</g, "&lt;")
+
+/**
+ * The IDE shell served to draft courses, with its meta tags filled in.
+ *
+ * The shell ships with `{{title}}`, `{{description}}` and `{{preview}}`
+ * placeholders that only get substituted when a course is published. Drafts were
+ * served the raw template, so a link shared in Slack or WhatsApp unfurled to a
+ * literal `{{title}}` / `{{description}}`.
+ */
+let draftShellTemplate: string | null = null
+
+const loadDraftShellTemplate = (): string => {
+  const file = path.resolve(__dirname, "../ui/_app/index.html")
+
+  draftShellTemplate = fs
+    .readFileSync(file, "utf-8")
+    // A draft is not a real Course, so it must not emit Course structured data.
+    // The match is anchored to the typed opening tag and non-greedy on purpose:
+    // the shell also carries an inline <script> that loads the app bundle from
+    // CloudFront, and a looser pattern would swallow it and blank the page.
+    .replace(
+      /[^\S\n]*<script type="application\/ld\+json">[\S\s]*?<\/script>\n?/,
+      ""
+    )
+
+  return draftShellTemplate
+}
+
+const renderDraftShell = (req: express.Request): string => {
+  const template = draftShellTemplate || loadDraftShellTemplate()
+
+  // Only the path: a preview URL can carry tokens in its query string, and
+  // og:url ends up in whatever crawler cache unfurls the link.
+  const canonicalUrl = escapeHtmlAttribute(`${getPublicHost()}${req.path}`)
+
+  return (
+    template
+      .replace(/<title>.*<\/title>/, `<title>${DRAFT_PREVIEW_TITLE}</title>`)
+      .replace(/{{title}}/g, DRAFT_PREVIEW_TITLE)
+      .replace(/{{description}}/g, DRAFT_PREVIEW_DESCRIPTION)
+      .replace(/{{preview}}/g, getFallbackPreviewUrl())
+      .replace(
+        "</head>",
+        `  <meta property="og:url" content="${canonicalUrl}" />\n` +
+          `    <meta name="robots" content="noindex, nofollow, noarchive" />\n  </head>`
+      )
+      // Safety net for placeholders a future shell may add: shipping an empty
+      // value to a crawler beats shipping "{{whatever}}".
+      .replace(/{{\w+}}/g, "")
+  )
+}
+
+/**
+ * Drafts are unlisted work in progress. `noindex` goes out as both a meta tag
+ * and a header so it reaches crawlers that skip the markup; a `robots.txt`
+ * `Disallow` would be the wrong tool, since it blocks the fetch itself and
+ * would stop Slack and X from unfurling the link at all.
+ */
+const sendDraftShell = (req: express.Request, res: express.Response) => {
+  res
+    .type("html")
+    .set("X-Robots-Tag", "noindex, nofollow")
+    .send(renderDraftShell(req))
+}
+
+/** How often the notification is touched while a translation is running. */
+const SYNC_HEARTBEAT_INTERVAL_MS = 60 * 1000
+
+/**
+ * A synchronization with no news for this long is considered stuck: the GET
+ * endpoint flags it as failed and it can be accepted again.
+ */
+const SYNC_PROCESSING_TIMEOUT_MS = 3 * 60 * 1000
+
+// True when a processing notification stopped reporting progress.
+function isSyncProcessingStale(notification: SyncNotification): boolean {
+  const lastUpdate =
+    notification.processingLastUpdate || notification.updatedAt
+  return Date.now() - lastUpdate > SYNC_PROCESSING_TIMEOUT_MS
+}
+
+function findSyncNotification(
+  syllabus: Syllabus,
+  exerciseSlug: string,
+  notificationId: string
+) {
+  const lesson = syllabus.lessons.find(
+    lesson => slugify(lesson.id + "-" + lesson.title) === exerciseSlug
+  )
+
+  return lesson?.syncNotifications?.find(n => n.id === notificationId)
+}
+
 async function processSyncTranslationsSequentially(
   courseSlug: string,
   exerciseSlug: string,
@@ -1269,30 +1342,72 @@ async function processSyncTranslationsSequentially(
 ) {
   try {
     const sanitizedSource = sanitizeReadmeNewlines(sourceReadmeContent)
+
+    let lastHeartbeatAt = Date.now()
+
+    // Keeps `processingLastUpdate` fresh while a language is being translated,
+    // otherwise the PROCESSING_TIMEOUT in GET /sync-notifications can flag a
+    // running synchronization as failed. Throttled, since every beat rewrites
+    // the syllabus in the bucket.
+    const heartbeat = async (targetLang: string) => {
+      if (Date.now() - lastHeartbeatAt < SYNC_HEARTBEAT_INTERVAL_MS) {
+        return
+      }
+
+      lastHeartbeatAt = Date.now()
+
+      try {
+        await serializeByCourse(courseSlug, async () => {
+          const syllabus = await getSyllabus(courseSlug, bucket)
+          const notification = findSyncNotification(
+            syllabus,
+            exerciseSlug,
+            notificationId
+          )
+
+          if (!notification) {
+            return
+          }
+
+          notification.processingLastUpdate = Date.now()
+
+          if (notification.syncProgress) {
+            notification.syncProgress.currentLanguage = targetLang
+          }
+
+          await saveSyllabus(courseSlug, syllabus, bucket)
+        })
+      } catch (error) {
+        console.warn(
+          `🔄 SYNC: heartbeat failed for ${exerciseSlug}:`,
+          (error as Error).message
+        )
+      }
+    }
+
     // Process translations sequentially (no race conditions)
     for (const targetLang of targetLanguages) {
       try {
-        // Call Rigobot directly with synchronous execution (no webhook needed)
+        const languageStartedAt = Date.now()
+
+        // The translation runs as an asynchronous Rigobot job that we poll:
+        // waiting for it inside a single request takes 28-45s for a full
+        // README and Heroku kills anything above 30s.
         // eslint-disable-next-line no-await-in-loop
-        const response = await axios.post(
-          `${RIGOBOT_HOST}/v1/prompting/completion/translate-asset-markdown/`,
+        const translationResult = await translateExerciseAndWait(
+          rigoToken,
           {
-            inputs: {
-              text_to_translate: sanitizedSource,
-              output_language: targetLang,
-            },
-            include_purpose_objective: false,
-            execute_async: false, // Synchronous execution
+            text_to_translate: sanitizedSource,
+            output_language: targetLang,
           },
-          {
-            headers: {
-              "Content-Type": "application/json",
-              Authorization: "Token " + rigoToken,
-            },
-          }
+          { onPoll: () => heartbeat(targetLang) }
         )
 
-        const translationResult = response.data
+        console.log(
+          `🔄 SYNC: ${targetLang} translated in ${Math.round(
+            (Date.now() - languageStartedAt) / 1000
+          )}s (job ${translationResult.id})`
+        )
 
         // Check if translation was successful
         if (!translationResult.parsed?.translation) {
@@ -1428,10 +1543,22 @@ async function processSyncTranslationsSequentially(
           (notification.syncProgress.failedLanguages?.length || 0)
 
         if (totalProcessed === notification.syncProgress.totalLanguages) {
-          notification.status =
-            notification.syncProgress.completedLanguages.length === 0 ?
-              "error" :
-              "completed"
+          const completedCount =
+            notification.syncProgress.completedLanguages.length
+          const failedCount =
+            notification.syncProgress.failedLanguages?.length || 0
+
+          // A partial result stays actionable: the notification remains
+          // visible so the user can retry just the languages that failed.
+          if (completedCount === 0) {
+            notification.status = "error"
+          } else if (failedCount > 0) {
+            notification.status = "partial"
+          } else {
+            notification.status = "completed"
+          }
+
+          notification.updatedAt = Date.now()
 
           await saveSyllabus(courseSlug, syllabus, bucket)
 
@@ -1598,6 +1725,7 @@ class ServeCommand extends SessionCommand {
     // app.use(express.static(distPath))
     app.use(express.json({ limit: "20mb" }))
     app.use(cors())
+    app.use("/generation", createIngestRouter())
 
     const appPath = path.resolve(__dirname, "../ui/_app")
     const tarPath = path.resolve(__dirname, "../ui/app.tar.gz")
@@ -1619,6 +1747,11 @@ class ServeCommand extends SessionCommand {
     const localAppPath = path.resolve(__dirname, "../ui/_app")
     // app.use(express.static(localAppPath))
 
+    // Warm the draft shell now that _app is on disk. Doing it here rather than
+    // on first request means a corrupt tarball fails at boot, with a stack
+    // trace, instead of on some user's preview link.
+    loadDraftShellTemplate()
+
     app.post("/upload", async (req, res) => {
       const { content, destination } = req.body
       //   console.log("UPLOAD", content, destination)
@@ -3104,8 +3237,7 @@ class ServeCommand extends SessionCommand {
       const slug = req.query.slug
 
       if (slug) {
-        const file = path.resolve(__dirname, "../ui/_app/index.html")
-        res.sendFile(file)
+        sendDraftShell(req, res)
       } else {
         res.redirect("https://learnpack.co")
       }
@@ -3114,13 +3246,18 @@ class ServeCommand extends SessionCommand {
     app.get("/preview/:slug", async (req, res) => {
       console.log("GET COURSE PREVIEW /")
 
-      const file = path.resolve(__dirname, "../ui/_app/index.html")
-      res.sendFile(file)
+      sendDraftShell(req, res)
     })
 
     app.get("/preview/:slug/webview", async (req, res) => {
-      const file = path.resolve(__dirname, "../ui/_app/index.html")
-      res.sendFile(file)
+      sendDraftShell(req, res)
+    })
+
+    // The brand card used as og:image by drafts, and as the last-resort
+    // fallback by published courses without a preview of their own.
+    app.get("/og-image.png", (req, res) => {
+      res.set("Cache-Control", "public, max-age=86400")
+      res.sendFile(path.resolve(__dirname, "../ui/static/og-image.png"))
     })
 
     app.get("/config", async (req, res) => {
@@ -4343,45 +4480,42 @@ class ServeCommand extends SessionCommand {
           return res.status(404).json({ error: "Syllabus not found" })
         }
 
-        const PROCESSING_TIMEOUT = 3 * 60 * 1000 // 3 minutes
         let modified = false
 
-        // Collect active notifications (pending, processing, or error)
+        // Collect active notifications (pending, processing, partial or error)
         const notifications: any[] = []
 
         for (const lesson of syllabus.lessons) {
           if (lesson.syncNotifications && lesson.syncNotifications.length > 0) {
             for (const notification of lesson.syncNotifications) {
               // Check for timeout in processing notifications
-              if (notification.status === "processing") {
-                // Use processingLastUpdate if available, otherwise fallback to updatedAt
-                const processingLastUpdateTime =
-                  notification.processingLastUpdate || notification.updatedAt
-                const timeSinceProcessingStarted =
-                  Date.now() - processingLastUpdateTime
-
-                if (timeSinceProcessingStarted > PROCESSING_TIMEOUT) {
-                  notification.status = "error"
-                  notification.error = {
-                    message: "Synchronization timeout - process took too long",
-                    code: "PROCESSING_TIMEOUT",
-                    timestamp: Date.now(),
-                  }
-                  modified = true
-
-                  // Emit error event
-                  emitToCourse(courseSlug, "sync-notification-error", {
-                    exerciseSlug: slugify(lesson.id + "-" + lesson.title),
-                    notificationId: notification.id,
-                    error: "Processing timeout",
-                  })
+              if (
+                notification.status === "processing" &&
+                isSyncProcessingStale(notification)
+              ) {
+                notification.status = "error"
+                notification.error = {
+                  message: "Synchronization timeout - process took too long",
+                  code: "PROCESSING_TIMEOUT",
+                  timestamp: Date.now(),
                 }
+                modified = true
+
+                // Emit error event
+                emitToCourse(courseSlug, "sync-notification-error", {
+                  exerciseSlug: slugify(lesson.id + "-" + lesson.title),
+                  notificationId: notification.id,
+                  error: "Processing timeout",
+                })
               }
 
-              // Include active notifications (pending, processing, or error)
+              // Include active notifications (pending, processing, partial or
+              // error): a partial sync is still actionable, the user can retry
+              // the languages that failed.
               if (
                 notification.status === "pending" ||
                 notification.status === "processing" ||
+                notification.status === "partial" ||
                 notification.status === "error"
               ) {
                 notifications.push({
@@ -4521,9 +4655,24 @@ class ServeCommand extends SessionCommand {
             return res.status(404).json({ error: "Notification not found" })
           }
 
-          if (notification.status !== "pending") {
+          // A synchronization can be accepted again unless it is actively
+          // running: after a failure or a partial result this same
+          // notification is what the Retry button acts on.
+          if (
+            notification.status === "processing" &&
+            !isSyncProcessingStale(notification)
+          ) {
+            return res.status(409).json({
+              error: "Notification is already being synchronized",
+              code: "ALREADY_PROCESSING",
+              currentStatus: notification.status,
+            })
+          }
+
+          if (notification.status === "completed") {
             return res.status(400).json({
-              error: "Notification is not pending",
+              error: "Notification is already synchronized",
+              code: "ALREADY_COMPLETED",
               currentStatus: notification.status,
             })
           }
@@ -4545,10 +4694,16 @@ class ServeCommand extends SessionCommand {
             })
           }
 
-          // Determine target languages
+          // Determine target languages. On a retry only what is still missing
+          // is translated again, so the languages that already succeeded are
+          // neither paid for twice nor overwritten.
+          const alreadyCompleted =
+            notification.syncProgress?.completedLanguages || []
           const availableLanguages = Object.keys(lesson.translations || {})
           const targetLanguages = availableLanguages.filter(
-            lang => lang !== notification.sourceLanguage
+            lang =>
+              lang !== notification.sourceLanguage &&
+              !alreadyCompleted.includes(lang)
           )
 
           if (targetLanguages.length === 0) {
@@ -4563,8 +4718,12 @@ class ServeCommand extends SessionCommand {
             lesson.syncNotifications?.filter(n => n.id === notificationId) ||
             []
 
+          // Start from a clean slate so a previous failure does not leak into
+          // this run.
           notification.status = "processing"
+          notification.updatedAt = Date.now()
           notification.processingLastUpdate = Date.now()
+          delete notification.error
           notification.syncProgress = {
             totalLanguages: targetLanguages.length,
             completedLanguages: [], // Array of completed language codes
@@ -5358,16 +5517,18 @@ class ServeCommand extends SessionCommand {
           selectedLang = availableLangs[0]
         }
 
+        const previewUrl = await resolveCoursePreviewUrl(
+          bucket,
+          slug,
+          config.preview
+        )
+
         const idxTpl = fs.readFileSync(path.join(uiSrc, "index.html"), "utf-8")
         const idxHtml = idxTpl
           .replace(/{{title}}/g, title)
           .replace(/<title>.*<\/title>/, `<title>${title}</title>`)
           .replace(/{{description}}/g, config.description[selectedLang])
-          .replace(
-            /{{preview}}/g,
-            fixPreviewUrl(slug, "") ||
-              "https://raw.githubusercontent.com/learnpack/ide/master/public/learnpack.svg"
-          )
+          .replace(/{{preview}}/g, previewUrl)
           .replace(/{{slug}}/g, slug)
           .replace(/{{duration}}/g, minutesToISO8601Duration(config.duration))
         fs.writeFileSync(path.join(buildRoot, "index.html"), idxHtml)
@@ -5415,9 +5576,9 @@ class ServeCommand extends SessionCommand {
         // 8) Crear el config.json en buildRoot con lo que retorna buildConfig
         const fullConfig = { config, exercises }
         fullConfig.config.slug = slug
-        fullConfig.config.preview =
-          fixPreviewUrl(slug, "") ||
-          "https://raw.githubusercontent.com/learnpack/ide/master/public/learnpack.svg"
+        // Same URL the meta tags got: this used to overwrite the real preview
+        // of every published package with the unusable SVG fallback.
+        fullConfig.config.preview = previewUrl
 
         fs.writeFileSync(
           path.join(buildRoot, "config.json"),