@learnpack/learnpack 5.0.352 → 5.0.353

package/src/commands/serve.ts

@@ -51,6 +51,7 @@ import {
 import axios from "axios"
 import * as FormData from "form-data"
 import api, {
+  BREATHECODE_TELEMETRY_URL,
   RIGOBOT_HOST,
   RIGOBOT_REALTIME_HOST,
   listUserAcademies,
@@ -74,6 +75,25 @@ import {
 } from "../models/creator"
 import { exportToScorm, exportToEpub, exportToZip } from "../utils/export"
 import { generateAndPersistPackageManifest } from "../utils/packageManifest"
+import { requireGcsBucketName } from "../utils/gcsBucketName"
+import {
+  createGcsSyllabusSyncStorage,
+  synchronizeSyllabusWithBucket,
+} from "../utils/syllabusSync"
+import {
+  createGcsJournalStorage,
+  createJournal,
+  markStage,
+} from "../utils/publishJournal"
+import {
+  countPendingDescriptions,
+  runDescriptionsStageAfterPublish,
+} from "../utils/descriptions/publishStage"
+import {
+  buildPackagePublishedEvent,
+  fetchPackageInfo,
+  sendPublishEvent,
+} from "../utils/publishEvents"
 import {
   errorHandler,
   notFoundHandler,
@@ -160,7 +180,7 @@ export const createLearnJson = (courseInfo: FormState) => {
     },
     grading: "isolated",
     telemetry: {
-      batch: "https://breathecode.herokuapp.com/v1/assignment/me/telemetry",
+      batch: BREATHECODE_TELEMETRY_URL,
     },
     preview: expectedPreviewUrl,
   }
@@ -1471,9 +1491,7 @@ class ServeCommand extends SessionCommand {
       credentials,
     })
 
-    const bucket = bucketStorage.bucket(
-      process.env.GCP_BUCKET_NAME || "learnpack-packages"
-    )
+    const bucket = bucketStorage.bucket(requireGcsBucketName())
 
     const rawHost = process.env.HOST || ""
     const host = rawHost.trim().split(/\s+#/)[0].trim()
@@ -4716,320 +4734,22 @@ class ServeCommand extends SessionCommand {
       }
 
       try {
-        // Get the current syllabus
+        // Reconciliation logic extracted to src/utils/syllabusSync.ts so the
+        // sweep/backfill can reuse it. The manual dev button keeps the original
+        // destructive behavior via prune: true.
         const syllabus = await getSyllabus(courseSlug, bucket)
-        const removedLessons: Array<{
-          id: string;
-          title: string;
-          slug: string;
-        }> = []
-        const keptLessons: Array<{
-          id: string;
-          title: string;
-          slug: string;
-          fileCount?: number;
-        }> = []
-        const duplicatesRemoved: Array<{
-          id: string;
-          title: string;
-          slug: string;
-          fileCount: number;
-        }> = []
-        const addedLessons: Array<{
-          id: string;
-          title: string;
-          slug: string;
-        }> = []
-        let repairedTranslationsInLessons = 0
-        let repairedTranslationEntries = 0
-        let fixedLessons = 0
-
-        console.log(
-          `📋 Checking ${syllabus.lessons.length} lessons in syllabus...`
+        const storage = createGcsSyllabusSyncStorage(bucket)
+        const result = await synchronizeSyllabusWithBucket(
+          storage,
+          courseSlug,
+          syllabus,
+          { prune: true }
         )
 
-        // First pass: Check each lesson to see if it exists in the bucket and count files.
-        // We try two possible folder names because they can differ by source:
-        // - Initial upload / most flows: folder = slugify(lesson.id + "-" + lesson.title)
-        // - create-step flow: folder = lesson.uid (Rigo's stepSlug), which may not match slugify()
-        //   (e.g. casing or character handling). GCS prefixes are case-sensitive.
-        const existingLessons: Array<{
-          lesson: any;
-          slug: string;
-          fileCount: number;
-        }> = []
-
-        for (const lesson of syllabus.lessons) {
-          const lessonSlug = slugify(lesson.id + "-" + lesson.title)
-          let filePrefix = `courses/${courseSlug}/exercises/${lessonSlug}/`
-
-          // eslint-disable-next-line no-await-in-loop
-          let [files] = await bucket.getFiles({
-            prefix: filePrefix,
-          })
-
-          if (files.length === 0 && lesson.uid) {
-            filePrefix = `courses/${courseSlug}/exercises/${lesson.uid}/`;
-            // eslint-disable-next-line no-await-in-loop
-            [files] = await bucket.getFiles({ prefix: filePrefix })
-          }
-
-          const resolvedSlug =
-            files.length > 0 ?
-              filePrefix
-                  .replace(`courses/${courseSlug}/exercises/`, "")
-                  .replace(/\/$/, "") :
-              lessonSlug
-
-          if (files.length === 0) {
-            removedLessons.push({
-              id: lesson.id,
-              title: lesson.title,
-              slug: lessonSlug,
-            })
-            console.log(
-              `❌ Lesson not found: ${lessonSlug}${
-                lesson.uid ? ` (also tried uid: ${lesson.uid})` : ""
-              }`
-            )
-          } else {
-            existingLessons.push({
-              lesson,
-              slug: resolvedSlug,
-              fileCount: files.length,
-            })
-            console.log(
-              `✅ Lesson exists: ${resolvedSlug} (${files.length} files)`
-            )
-          }
-        }
-
-        // Second pass: Detect and resolve duplicates
-        const slugMap = new Map<
-          string,
-          Array<{ lesson: any; slug: string; fileCount: number }>
-        >()
-
-        // Group by slug
-        for (const item of existingLessons) {
-          if (!slugMap.has(item.slug)) {
-            slugMap.set(item.slug, [])
-          }
-
-          slugMap.get(item.slug)!.push(item)
-        }
-
-        // Process each group
-        for (const [slug, items] of slugMap.entries()) {
-          if (items.length > 1) {
-            // Duplicates found! Sort by file count (descending) and keep the most complete
-            items.sort((a, b) => b.fileCount - a.fileCount)
-
-            const winner = items[0]
-            const losers = items.slice(1)
-
-            console.log(`🔍 Duplicate slug detected: ${slug}`)
-            console.log(
-              `   ✅ Keeping: ${winner.lesson.id} - ${winner.lesson.title} (${winner.fileCount} files)`
-            )
-
-            // Keep the winner
-            keptLessons.push({
-              id: winner.lesson.id,
-              title: winner.lesson.title,
-              slug: winner.slug,
-              fileCount: winner.fileCount,
-            })
-
-            // Mark losers for removal
-            for (const loser of losers) {
-              console.log(
-                `   ❌ Removing duplicate: ${loser.lesson.id} - ${loser.lesson.title} (${loser.fileCount} files)`
-              )
-              duplicatesRemoved.push({
-                id: loser.lesson.id,
-                title: loser.lesson.title,
-                slug: loser.slug,
-                fileCount: loser.fileCount,
-              })
-            }
-          } else {
-            // No duplicates, keep as is
-            keptLessons.push({
-              id: items[0].lesson.id,
-              title: items[0].lesson.title,
-              slug: items[0].slug,
-              fileCount: items[0].fileCount,
-            })
-          }
-        }
-
-        // Update syllabus: keep only the winners and remove non-existent + duplicates
-        const totalRemoved = removedLessons.length + duplicatesRemoved.length
-
-        if (totalRemoved > 0) {
-          syllabus.lessons = syllabus.lessons.filter(lesson => {
-            const lessonSlug = slugify(lesson.id + "-" + lesson.title)
-            const isRemoved = removedLessons.some(
-              removed => removed.slug === lessonSlug
-            )
-            const isDuplicate = duplicatesRemoved.some(
-              dup => dup.id === lesson.id && dup.title === lesson.title
-            )
-            return !isRemoved && !isDuplicate
-          })
-        }
-
-        // Third pass: add lessons that exist in the bucket but are missing from the syllabus
-        const existingSlugs = new Set(existingLessons.map(e => e.slug))
-        const exercisesPrefix = `courses/${courseSlug}/exercises/`
-        const [allExerciseFiles] = await bucket.getFiles({
-          prefix: exercisesPrefix,
-        })
-        const bucketFolderSlugs = new Set<string>()
-        for (const file of allExerciseFiles) {
-          if (!file.name.startsWith(exercisesPrefix)) continue
-          const afterPrefix = file.name.slice(exercisesPrefix.length)
-          const segment = afterPrefix.split("/")[0]
-          if (segment) bucketFolderSlugs.add(segment)
-        }
-
-        for (const folderSlug of bucketFolderSlugs) {
-          if (existingSlugs.has(folderSlug)) continue
-          const idMatch = folderSlug.match(/^(\d+(?:\.\d+)?)(?:-(.*))?$/)
-          const id = idMatch ? idMatch[1] : folderSlug
-          const titlePart = idMatch && idMatch[2] ? idMatch[2] : folderSlug
-          const title = titlePart.replace(/-/g, " ").trim() || folderSlug
-          const newLesson: Lesson = {
-            id,
-            uid: folderSlug,
-            title,
-            type: "READ",
-            description: title,
-            duration: 2,
-            generated: true,
-            status: "DONE",
-          }
-          syllabus.lessons.push(newLesson)
-          addedLessons.push({ id, title, slug: folderSlug })
-          console.log(`➕ Added missing lesson from bucket: ${folderSlug}`)
-        }
-
-        if (addedLessons.length > 0) {
-          syllabus.lessons.sort((a, b) => {
-            const na = parseFloat(a.id)
-            const nb = parseFloat(b.id)
-            if (!Number.isNaN(na) && !Number.isNaN(nb)) return na - nb
-            return String(a.id).localeCompare(String(b.id))
-          })
-        }
-
-        // Fourth pass: reconcile lesson.translations from actual README files in bucket.
-        // This fixes lessons that exist but lost translations metadata after renames or syncs.
-        try {
-          const { exercises } = await buildConfig(bucket, courseSlug)
-          const translationsBySlug = new Map<string, string[]>()
-
-          for (const exercise of exercises) {
-            const langs = Object.keys(exercise.translations || {})
-              .map(lang => lang.toLowerCase())
-              .filter(Boolean)
-            if (langs.length > 0) {
-              translationsBySlug.set(exercise.slug, [...new Set(langs)])
-            }
-          }
-
-          for (const lesson of syllabus.lessons) {
-            const candidateSlugs = [
-              slugify(lesson.id + "-" + lesson.title),
-              lesson.uid,
-            ].filter(Boolean)
-            const matchedSlug = candidateSlugs.find(s =>
-              translationsBySlug.has(s)
-            )
-            if (!matchedSlug) continue
-
-            const languageCodes = translationsBySlug.get(matchedSlug) || []
-            if (languageCodes.length === 0) continue
-
-            const currentTranslations = lesson.translations || {}
-            let lessonChanged = false
-
-            for (const lang of languageCodes) {
-              const now = Date.now()
-              if (!currentTranslations[lang]) {
-                currentTranslations[lang] = {
-                  completionId: 0,
-                  startedAt: now,
-                  completedAt: now,
-                }
-                repairedTranslationEntries += 1
-                lessonChanged = true
-              } else {
-                // If README exists for this language but metadata is incomplete, mark it as completed.
-                if (!currentTranslations[lang].startedAt) {
-                  currentTranslations[lang].startedAt = now
-                  lessonChanged = true
-                }
-
-                if (!currentTranslations[lang].completedAt) {
-                  currentTranslations[lang].completedAt = now
-                  repairedTranslationEntries += 1
-                  lessonChanged = true
-                }
-              }
-            }
-
-            if (lessonChanged) {
-              lesson.translations = currentTranslations
-              repairedTranslationsInLessons += 1
-            }
-          }
-
-          // Fifth pass: fix lessons stuck in GENERATING or ERROR when the file exists in the bucket
-          const primaryLanguage = syllabus.courseInfo.language || "en"
-          for (const lesson of syllabus.lessons) {
-            if (lesson.generated !== false) continue
-            if (lesson.status !== "GENERATING" && lesson.status !== "ERROR")
-              continue
-
-            const candidateSlugs = [
-              slugify(lesson.id + "-" + lesson.title),
-              lesson.uid,
-            ].filter(Boolean) as string[]
-
-            const matchedSlug = candidateSlugs.find(s =>
-              translationsBySlug.has(s)
-            )
-            if (!matchedSlug) continue
-
-            const langs = translationsBySlug.get(matchedSlug) || []
-            if (!langs.includes(primaryLanguage)) continue
-
-            const prevStatus = lesson.status
-            lesson.generated = true
-            lesson.status = "DONE"
-            fixedLessons += 1
-            console.log(
-              `🔧 Fixed lesson: ${lesson.id} - "${lesson.title}" (was generated:false status:${prevStatus}, primary language "${primaryLanguage}" exists in bucket)`
-            )
-          }
-        } catch (error) {
-          console.error(
-            "⚠️ Could not reconcile lesson translations during syllabus sync:",
-            error
-          )
-        }
-
-        if (
-          totalRemoved > 0 ||
-          addedLessons.length > 0 ||
-          repairedTranslationsInLessons > 0 ||
-          fixedLessons > 0
-        ) {
-          await saveSyllabus(courseSlug, syllabus, bucket)
+        if (result.changed) {
+          await saveSyllabus(courseSlug, result.syllabus, bucket)
           console.log(
-            `✅ Syllabus synchronized. Removed ${removedLessons.length} non-existent, ${duplicatesRemoved.length} duplicate(s); added ${addedLessons.length} from bucket; repaired translations in ${repairedTranslationsInLessons} lesson(s); fixed ${fixedLessons} stuck lesson(s).`
+            `✅ Syllabus synchronized. Removed ${result.counters.removedLessons} non-existent + duplicate(s); added ${result.counters.addedLessons} from bucket; repaired translations in ${result.counters.repairedTranslationsInLessons} lesson(s); fixed ${result.counters.fixedLessons} stuck lesson(s).`
           )
         } else {
           console.log(`✅ Syllabus is already in sync. No changes.`)
@@ -5038,18 +4758,20 @@ class ServeCommand extends SessionCommand {
         res.json({
           status: "SUCCESS",
           message: `Syllabus synchronized successfully`,
-          totalLessons: syllabus.lessons.length,
-          keptLessons: keptLessons.length,
-          removedLessons: removedLessons.length,
-          duplicatesResolved: duplicatesRemoved.length,
-          addedLessons: addedLessons.length,
-          fixedLessons,
-          repairedTranslationsInLessons,
-          repairedTranslationEntries,
-          removed: removedLessons,
-          duplicates: duplicatesRemoved,
-          kept: keptLessons,
-          added: addedLessons,
+          totalLessons: result.counters.totalLessons,
+          keptLessons: result.counters.keptLessons,
+          removedLessons: result.counters.removedLessons,
+          duplicatesResolved: result.counters.duplicatesResolved,
+          addedLessons: result.counters.addedLessons,
+          fixedLessons: result.counters.fixedLessons,
+          repairedTranslationsInLessons:
+            result.counters.repairedTranslationsInLessons,
+          repairedTranslationEntries:
+            result.counters.repairedTranslationEntries,
+          removed: result.details.removed,
+          duplicates: result.details.duplicates,
+          kept: result.details.kept,
+          added: result.details.added,
         })
       } catch (error) {
         console.error("❌ Error synchronizing syllabus:", error)
@@ -5601,6 +5323,11 @@ class ServeCommand extends SessionCommand {
         rimraf.sync(tmpRoot)
         mkdirp.sync(buildRoot)
 
+        // Bookkeeping for the whole publication, so an interrupted run can be
+        // resumed. Best effort: never blocks the publish.
+        const journalStorage = createGcsJournalStorage(bucket)
+        const journal = await createJournal(journalStorage, slug)
+
         // 4) Copiar UI descomprimida (_app)
         const uiSrc = path.resolve(__dirname, "../ui/_app")
         const copyDir = (src: string, dest: string) => {
@@ -5666,10 +5393,24 @@ class ServeCommand extends SessionCommand {
           })
         )
 
-        await generateAndPersistPackageManifest(bucket, slug, {
-          publishedAt: new Date().toISOString(),
-          localDir: buildRoot,
-        })
+        const publishedManifest = await generateAndPersistPackageManifest(
+          bucket,
+          slug,
+          {
+            publishedAt: new Date().toISOString(),
+            localDir: buildRoot,
+          }
+        )
+
+        // Decide here, while the build is still on disk, whether this package
+        // will need descriptions: the answer is what the response and the
+        // package_published event promise to breathecode.
+        const pendingDescriptions = countPendingDescriptions(
+          buildRoot,
+          exercises
+        )
+        const descriptionsOutcome =
+          pendingDescriptions > 0 ? "queued" : "up_to_date"
 
         // 8) Crear el config.json en buildRoot con lo que retorna buildConfig
         const fullConfig = { config, exercises }
@@ -5709,6 +5450,9 @@ class ServeCommand extends SessionCommand {
               }
             )
             rigoPublishUrl = rigoRes.data.url
+            await markStage(journalStorage, journal, "deploy", "done", {
+              meta: { url: rigoPublishUrl },
+            })
 
             let errors: AssetSyncError[]
             try {
@@ -5722,6 +5466,15 @@ class ServeCommand extends SessionCommand {
                 academyId
               )
               errors = assetResults.errors
+              await markStage(
+                journalStorage,
+                journal,
+                "assetsSync",
+                errors.length > 0 ? "failed" : "done",
+                {
+                  error: errors.length > 0 ? JSON.stringify(errors) : undefined,
+                }
+              )
             } catch (error) {
               console.error("Asset sync failed unexpectedly:", error)
               errors = [
@@ -5730,16 +5483,71 @@ class ServeCommand extends SessionCommand {
                   error: { detail: "Asset sync failed unexpectedly." },
                 },
               ]
+              await markStage(journalStorage, journal, "assetsSync", "failed", {
+                error: (error as Error).message,
+              })
             }
 
-            if (res.headersSent) return
-            console.log("RigoRes", rigoRes.data)
-            res.json({
-              url: rigoPublishUrl,
-              errors,
+            if (!res.headersSent) {
+              console.log("RigoRes", rigoRes.data)
+              res.json({
+                url: rigoPublishUrl,
+                errors,
+                descriptions: descriptionsOutcome,
+              })
+            }
+
+            // The package is published: tell breathecode, announcing whether a
+            // second event is coming. Emitted after the asset sync so the
+            // assets the payload refers to already exist.
+            const eventContext = {
+              publishId: journal.publishId,
+              courseSlug: slug,
+              packageInfo: await fetchPackageInfo(slug, rigoToken),
+              manifest: publishedManifest,
+            }
+
+            const publishedDelivered = await sendPublishEvent(
+              buildPackagePublishedEvent(eventContext, descriptionsOutcome),
+              bcToken
+            )
+            await markStage(
+              journalStorage,
+              journal,
+              "publishedEvent",
+              publishedDelivered ? "done" : "failed",
+              {
+                error: publishedDelivered ? undefined : "delivery failed",
+                // Recorded so a sweep resuming this publication knows whether a
+                // manifest event was promised to breathecode.
+                meta: { descriptions: descriptionsOutcome },
+              }
+            )
+
+            // Stage B, after responding: a course may need several ~15s
+            // completions, far past any request limit. The package is already
+            // published and durable; this only enriches it.
+            runDescriptionsStageAfterPublish({
+              courseSlug: slug,
+              gcsBucket: bucket,
+              journal,
+              journalStorage,
+              hasWork: pendingDescriptions > 0,
+              eventContext,
+              breathecodeToken: bcToken,
+            }).catch(error => {
+              console.error("[descriptions] Detached stage rejected:", error)
             })
           } catch (error) {
             console.error(error)
+            await markStage(
+              journalStorage,
+              journal,
+              rigoPublishUrl === undefined ? "deploy" : "assetsSync",
+              "failed",
+              { error: (error as Error).message }
+            )
+
             if (res.headersSent) return
             if (rigoPublishUrl !== undefined) {
               res.json({
@@ -5750,6 +5558,7 @@ class ServeCommand extends SessionCommand {
                     error: { detail: "Asset sync failed unexpectedly." },
                   },
                 ],
+                descriptions: descriptionsOutcome,
               })
             } else {
               res.status(500).json({ error: (error as Error).message })

package/src/models/creator.ts

@@ -37,6 +37,15 @@ export interface Lesson {
       startedAt: number;
       completedAt?: number;
       error?: boolean;
+      // --- description fields (per language) ---
+      // The step description is a projection derived from the README content;
+      // it lives here (source of truth) and is projected into package-manifest.json.
+      description?: string | null;
+      descriptionStatus?: "generated" | "pending" | "error" | "none";
+      sourceContentHash?: string; // SHA-256 of the normalized README it was generated from
+      sourceSimHash?: string; // SimHash-64 (hex) used for the significance threshold
+      descriptionPromptVersion?: number;
+      descriptionSource?: "auto" | "human";
     };
   };
   syncNotifications?: SyncNotification[];