@learnpack/learnpack 5.0.354 → 5.0.357

package/lib/utils/socketRegistry.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Which sockets are listening on which channel.
+ *
+ * Extracted from `creatorSocket.ts` so the bookkeeping can be tested without
+ * standing up a socket.io server. The registry deals in socket ids only; the
+ * socket objects themselves stay with the transport layer.
+ *
+ * The behaviour that matters is `removeSocket`: a disconnecting client has to
+ * disappear from **every** channel it joined. The notification map used to be
+ * skipped on disconnect, so it grew for the lifetime of the process and
+ * accumulated ids of sockets that were long gone.
+ */
+export declare class SocketRegistry {
+    private readonly channels;
+    /**
+     * Adds a socket to a channel.
+     * @param channel - Channel key, such as a course slug or a notification id.
+     * @param socketId - The socket joining it.
+     */
+    register(channel: string, socketId: string): void;
+    /**
+     * Removes a socket from one channel, dropping the channel when it empties.
+     * @param channel - Channel to leave.
+     * @param socketId - The socket leaving it.
+     */
+    unregister(channel: string, socketId: string): void;
+    /**
+     * Removes a socket from every channel it joined.
+     *
+     * Called when the socket disconnects. Without it the registry keeps growing
+     * with ids that can never receive anything again.
+     * @param socketId - The socket that went away.
+     */
+    removeSocket(socketId: string): void;
+    /**
+     * The sockets currently listening on a channel.
+     * @param channel - Channel to look up.
+     * @returns Their socket ids, empty when nobody is listening.
+     */
+    listeners(channel: string): string[];
+    /**
+     * How many channels are being tracked. Exposed for tests and diagnostics.
+     * @returns The number of non-empty channels.
+     */
+    channelCount(): number;
+}

package/lib/utils/socketRegistry.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SocketRegistry = void 0;
+/**
+ * Which sockets are listening on which channel.
+ *
+ * Extracted from `creatorSocket.ts` so the bookkeeping can be tested without
+ * standing up a socket.io server. The registry deals in socket ids only; the
+ * socket objects themselves stay with the transport layer.
+ *
+ * The behaviour that matters is `removeSocket`: a disconnecting client has to
+ * disappear from **every** channel it joined. The notification map used to be
+ * skipped on disconnect, so it grew for the lifetime of the process and
+ * accumulated ids of sockets that were long gone.
+ */
+class SocketRegistry {
+    constructor() {
+        this.channels = new Map();
+    }
+    /**
+     * Adds a socket to a channel.
+     * @param channel - Channel key, such as a course slug or a notification id.
+     * @param socketId - The socket joining it.
+     */
+    register(channel, socketId) {
+        if (!channel || !socketId) {
+            return;
+        }
+        let listeners = this.channels.get(channel);
+        if (!listeners) {
+            listeners = new Set();
+            this.channels.set(channel, listeners);
+        }
+        listeners.add(socketId);
+    }
+    /**
+     * Removes a socket from one channel, dropping the channel when it empties.
+     * @param channel - Channel to leave.
+     * @param socketId - The socket leaving it.
+     */
+    unregister(channel, socketId) {
+        const listeners = this.channels.get(channel);
+        if (!listeners) {
+            return;
+        }
+        listeners.delete(socketId);
+        if (listeners.size === 0) {
+            this.channels.delete(channel);
+        }
+    }
+    /**
+     * Removes a socket from every channel it joined.
+     *
+     * Called when the socket disconnects. Without it the registry keeps growing
+     * with ids that can never receive anything again.
+     * @param socketId - The socket that went away.
+     */
+    removeSocket(socketId) {
+        for (const [channel, listeners] of this.channels) {
+            listeners.delete(socketId);
+            if (listeners.size === 0) {
+                this.channels.delete(channel);
+            }
+        }
+    }
+    /**
+     * The sockets currently listening on a channel.
+     * @param channel - Channel to look up.
+     * @returns Their socket ids, empty when nobody is listening.
+     */
+    listeners(channel) {
+        const listeners = this.channels.get(channel);
+        return listeners ? [...listeners] : [];
+    }
+    /**
+     * How many channels are being tracked. Exposed for tests and diagnostics.
+     * @returns The number of non-empty channels.
+     */
+    channelCount() {
+        return this.channels.size;
+    }
+}
+exports.SocketRegistry = SocketRegistry;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@learnpack/learnpack",
   "description": "Seamlessly build, sell and/or take interactive & auto-graded tutorials, start learning now or build a new tutorial to your audience.",
-  "version": "5.0.354",
+  "version": "5.0.357",
   "author": "Alejandro Sanchez @alesanchezr",
   "contributors": [
     {

package/src/commands/serve.ts

@@ -26,6 +26,7 @@ import * as fs from "fs"
 import {
   // createCodeFile,
   translateExercise,
+  translateExerciseAndWait,
   isValidRigoToken,
   readmeCreator,
   // makeReadmeReadable,
@@ -51,7 +52,6 @@ import {
 import axios from "axios"
 import * as FormData from "form-data"
 import api, {
-  BREATHECODE_TELEMETRY_URL,
   RIGOBOT_HOST,
   RIGOBOT_REALTIME_HOST,
   listUserAcademies,
@@ -74,6 +74,13 @@ import {
   SyncNotification,
 } from "../models/creator"
 import { exportToScorm, exportToEpub, exportToZip } from "../utils/export"
+import { createIngestRouter } from "../services/ingest/http/router"
+import {
+  uploadFileToBucket,
+  uploadBinaryToBucket,
+} from "../utils/coursePackage/bucketIo"
+import { createLearnJson } from "../utils/coursePackage/learnJson"
+import { createInitialSidebar } from "../utils/coursePackage/sidebar"
 import { generateAndPersistPackageManifest } from "../utils/packageManifest"
 import { requireGcsBucketName } from "../utils/gcsBucketName"
 import {
@@ -161,56 +168,6 @@ async function serializeByCourse<T>(
   return next
 }
 
-export const createLearnJson = (courseInfo: FormState) => {
-  // console.log("courseInfo to create learn json", courseInfo)
-
-  const expectedPreviewUrl = `https://${courseInfo.slug}.learn-pack.com/preview.png`
-
-  const language = courseInfo.language || "en"
-
-  const learnJson = {
-    slug: courseInfo.slug,
-    title: {
-      [language]: courseInfo.title,
-    },
-    technologies: courseInfo.technologies || [],
-    difficulty: "beginner",
-    description: {
-      [language]: courseInfo.description,
-    },
-    grading: "isolated",
-    telemetry: {
-      batch: BREATHECODE_TELEMETRY_URL,
-    },
-    preview: expectedPreviewUrl,
-  }
-  return learnJson
-}
-
-const uploadFileToBucket = async (
-  bucket: Bucket,
-  file: string,
-  path: string
-) => {
-  const isReadme = /readme(\.\w+)?\.md$/i.test(path)
-  const content = isReadme ? sanitizeReadmeNewlines(file) : file
-  const fileRef = bucket.file(path)
-  await fileRef.save(Buffer.from(content, "utf8"))
-}
-
-const uploadBinaryToBucket = async (
-  bucket: Bucket,
-  buffer: Buffer,
-  path: string,
-  contentType?: string
-) => {
-  const fileRef = bucket.file(path)
-  await fileRef.save(buffer, {
-    resumable: false,
-    ...(contentType && { contentType }),
-  })
-}
-
 const getGithubCredentials = () => {
   const token = process.env.GITHUB_TOKEN?.trim()
   const username = process.env.GITHUB_USERNAME?.trim()
@@ -518,20 +475,6 @@ export const processImage = async (
   }
 }
 
-const createInitialSidebar = async (
-  slugs: string[],
-  initialLanguage = "en"
-) => {
-  const sidebar: Record<string, Record<string, string>> = {}
-  for (const slug of slugs) {
-    sidebar[slug] = {
-      [initialLanguage]: slug,
-    }
-  }
-
-  return sidebar
-}
-
 const uploadInitialReadme = async (
   bucket: Bucket,
   exSlug: string,
@@ -1257,6 +1200,34 @@ const getTitleFromHTML = (html: string) => {
   return titleMatch ? titleMatch[1] : null
 }
 
+/** How often the notification is touched while a translation is running. */
+const SYNC_HEARTBEAT_INTERVAL_MS = 60 * 1000
+
+/**
+ * A synchronization with no news for this long is considered stuck: the GET
+ * endpoint flags it as failed and it can be accepted again.
+ */
+const SYNC_PROCESSING_TIMEOUT_MS = 3 * 60 * 1000
+
+// True when a processing notification stopped reporting progress.
+function isSyncProcessingStale(notification: SyncNotification): boolean {
+  const lastUpdate =
+    notification.processingLastUpdate || notification.updatedAt
+  return Date.now() - lastUpdate > SYNC_PROCESSING_TIMEOUT_MS
+}
+
+function findSyncNotification(
+  syllabus: Syllabus,
+  exerciseSlug: string,
+  notificationId: string
+) {
+  const lesson = syllabus.lessons.find(
+    lesson => slugify(lesson.id + "-" + lesson.title) === exerciseSlug
+  )
+
+  return lesson?.syncNotifications?.find(n => n.id === notificationId)
+}
+
 async function processSyncTranslationsSequentially(
   courseSlug: string,
   exerciseSlug: string,
@@ -1269,30 +1240,72 @@ async function processSyncTranslationsSequentially(
 ) {
   try {
     const sanitizedSource = sanitizeReadmeNewlines(sourceReadmeContent)
+
+    let lastHeartbeatAt = Date.now()
+
+    // Keeps `processingLastUpdate` fresh while a language is being translated,
+    // otherwise the PROCESSING_TIMEOUT in GET /sync-notifications can flag a
+    // running synchronization as failed. Throttled, since every beat rewrites
+    // the syllabus in the bucket.
+    const heartbeat = async (targetLang: string) => {
+      if (Date.now() - lastHeartbeatAt < SYNC_HEARTBEAT_INTERVAL_MS) {
+        return
+      }
+
+      lastHeartbeatAt = Date.now()
+
+      try {
+        await serializeByCourse(courseSlug, async () => {
+          const syllabus = await getSyllabus(courseSlug, bucket)
+          const notification = findSyncNotification(
+            syllabus,
+            exerciseSlug,
+            notificationId
+          )
+
+          if (!notification) {
+            return
+          }
+
+          notification.processingLastUpdate = Date.now()
+
+          if (notification.syncProgress) {
+            notification.syncProgress.currentLanguage = targetLang
+          }
+
+          await saveSyllabus(courseSlug, syllabus, bucket)
+        })
+      } catch (error) {
+        console.warn(
+          `🔄 SYNC: heartbeat failed for ${exerciseSlug}:`,
+          (error as Error).message
+        )
+      }
+    }
+
     // Process translations sequentially (no race conditions)
     for (const targetLang of targetLanguages) {
       try {
-        // Call Rigobot directly with synchronous execution (no webhook needed)
+        const languageStartedAt = Date.now()
+
+        // The translation runs as an asynchronous Rigobot job that we poll:
+        // waiting for it inside a single request takes 28-45s for a full
+        // README and Heroku kills anything above 30s.
         // eslint-disable-next-line no-await-in-loop
-        const response = await axios.post(
-          `${RIGOBOT_HOST}/v1/prompting/completion/translate-asset-markdown/`,
+        const translationResult = await translateExerciseAndWait(
+          rigoToken,
           {
-            inputs: {
-              text_to_translate: sanitizedSource,
-              output_language: targetLang,
-            },
-            include_purpose_objective: false,
-            execute_async: false, // Synchronous execution
+            text_to_translate: sanitizedSource,
+            output_language: targetLang,
           },
-          {
-            headers: {
-              "Content-Type": "application/json",
-              Authorization: "Token " + rigoToken,
-            },
-          }
+          { onPoll: () => heartbeat(targetLang) }
         )
 
-        const translationResult = response.data
+        console.log(
+          `🔄 SYNC: ${targetLang} translated in ${Math.round(
+            (Date.now() - languageStartedAt) / 1000
+          )}s (job ${translationResult.id})`
+        )
 
         // Check if translation was successful
         if (!translationResult.parsed?.translation) {
@@ -1428,10 +1441,22 @@ async function processSyncTranslationsSequentially(
           (notification.syncProgress.failedLanguages?.length || 0)
 
         if (totalProcessed === notification.syncProgress.totalLanguages) {
-          notification.status =
-            notification.syncProgress.completedLanguages.length === 0 ?
-              "error" :
-              "completed"
+          const completedCount =
+            notification.syncProgress.completedLanguages.length
+          const failedCount =
+            notification.syncProgress.failedLanguages?.length || 0
+
+          // A partial result stays actionable: the notification remains
+          // visible so the user can retry just the languages that failed.
+          if (completedCount === 0) {
+            notification.status = "error"
+          } else if (failedCount > 0) {
+            notification.status = "partial"
+          } else {
+            notification.status = "completed"
+          }
+
+          notification.updatedAt = Date.now()
 
           await saveSyllabus(courseSlug, syllabus, bucket)
 
@@ -1598,6 +1623,7 @@ class ServeCommand extends SessionCommand {
     // app.use(express.static(distPath))
     app.use(express.json({ limit: "20mb" }))
     app.use(cors())
+    app.use("/generation", createIngestRouter())
 
     const appPath = path.resolve(__dirname, "../ui/_app")
     const tarPath = path.resolve(__dirname, "../ui/app.tar.gz")
@@ -4343,45 +4369,42 @@ class ServeCommand extends SessionCommand {
           return res.status(404).json({ error: "Syllabus not found" })
         }
 
-        const PROCESSING_TIMEOUT = 3 * 60 * 1000 // 3 minutes
         let modified = false
 
-        // Collect active notifications (pending, processing, or error)
+        // Collect active notifications (pending, processing, partial or error)
         const notifications: any[] = []
 
         for (const lesson of syllabus.lessons) {
           if (lesson.syncNotifications && lesson.syncNotifications.length > 0) {
             for (const notification of lesson.syncNotifications) {
               // Check for timeout in processing notifications
-              if (notification.status === "processing") {
-                // Use processingLastUpdate if available, otherwise fallback to updatedAt
-                const processingLastUpdateTime =
-                  notification.processingLastUpdate || notification.updatedAt
-                const timeSinceProcessingStarted =
-                  Date.now() - processingLastUpdateTime
-
-                if (timeSinceProcessingStarted > PROCESSING_TIMEOUT) {
-                  notification.status = "error"
-                  notification.error = {
-                    message: "Synchronization timeout - process took too long",
-                    code: "PROCESSING_TIMEOUT",
-                    timestamp: Date.now(),
-                  }
-                  modified = true
-
-                  // Emit error event
-                  emitToCourse(courseSlug, "sync-notification-error", {
-                    exerciseSlug: slugify(lesson.id + "-" + lesson.title),
-                    notificationId: notification.id,
-                    error: "Processing timeout",
-                  })
+              if (
+                notification.status === "processing" &&
+                isSyncProcessingStale(notification)
+              ) {
+                notification.status = "error"
+                notification.error = {
+                  message: "Synchronization timeout - process took too long",
+                  code: "PROCESSING_TIMEOUT",
+                  timestamp: Date.now(),
                 }
+                modified = true
+
+                // Emit error event
+                emitToCourse(courseSlug, "sync-notification-error", {
+                  exerciseSlug: slugify(lesson.id + "-" + lesson.title),
+                  notificationId: notification.id,
+                  error: "Processing timeout",
+                })
               }
 
-              // Include active notifications (pending, processing, or error)
+              // Include active notifications (pending, processing, partial or
+              // error): a partial sync is still actionable, the user can retry
+              // the languages that failed.
               if (
                 notification.status === "pending" ||
                 notification.status === "processing" ||
+                notification.status === "partial" ||
                 notification.status === "error"
               ) {
                 notifications.push({
@@ -4521,9 +4544,24 @@ class ServeCommand extends SessionCommand {
             return res.status(404).json({ error: "Notification not found" })
           }
 
-          if (notification.status !== "pending") {
+          // A synchronization can be accepted again unless it is actively
+          // running: after a failure or a partial result this same
+          // notification is what the Retry button acts on.
+          if (
+            notification.status === "processing" &&
+            !isSyncProcessingStale(notification)
+          ) {
+            return res.status(409).json({
+              error: "Notification is already being synchronized",
+              code: "ALREADY_PROCESSING",
+              currentStatus: notification.status,
+            })
+          }
+
+          if (notification.status === "completed") {
             return res.status(400).json({
-              error: "Notification is not pending",
+              error: "Notification is already synchronized",
+              code: "ALREADY_COMPLETED",
               currentStatus: notification.status,
             })
           }
@@ -4545,10 +4583,16 @@ class ServeCommand extends SessionCommand {
             })
           }
 
-          // Determine target languages
+          // Determine target languages. On a retry only what is still missing
+          // is translated again, so the languages that already succeeded are
+          // neither paid for twice nor overwritten.
+          const alreadyCompleted =
+            notification.syncProgress?.completedLanguages || []
           const availableLanguages = Object.keys(lesson.translations || {})
           const targetLanguages = availableLanguages.filter(
-            lang => lang !== notification.sourceLanguage
+            lang =>
+              lang !== notification.sourceLanguage &&
+              !alreadyCompleted.includes(lang)
           )
 
           if (targetLanguages.length === 0) {
@@ -4563,8 +4607,12 @@ class ServeCommand extends SessionCommand {
             lesson.syncNotifications?.filter(n => n.id === notificationId) ||
             []
 
+          // Start from a clean slate so a previous failure does not leak into
+          // this run.
           notification.status = "processing"
+          notification.updatedAt = Date.now()
           notification.processingLastUpdate = Date.now()
+          delete notification.error
           notification.syncProgress = {
             totalLanguages: targetLanguages.length,
             completedLanguages: [], // Array of completed language codes
@@ -6731,7 +6779,7 @@ class ServeCommand extends SessionCommand {
 
             const buffer = Buffer.from(content, "base64")
 
-            if (isImageFile(item.relativePath)) {
+            if (item.isAsset || isImageFile(item.relativePath)) {
               // eslint-disable-next-line no-await-in-loop -- Sequential processing to avoid rate limits
               await uploadBinaryToBucket(
                 bucket,

package/src/creator/README.md

@@ -1,54 +1,66 @@
-# React + TypeScript + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
-- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
-- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
-
-## Expanding the ESLint configuration
-
-If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules:
-
-```js
-export default tseslint.config({
-  extends: [
-    // Remove ...tseslint.configs.recommended and replace with this
-    ...tseslint.configs.recommendedTypeChecked,
-    // Alternatively, use this for stricter rules
-    ...tseslint.configs.strictTypeChecked,
-    // Optionally, add this for stylistic rules
-    ...tseslint.configs.stylisticTypeChecked,
-  ],
-  languageOptions: {
-    // other options...
-    parserOptions: {
-      project: ["./tsconfig.node.json", "./tsconfig.app.json"],
-      tsconfigRootDir: import.meta.dirname,
-    },
-  },
-})
+# LearnPack Creator
+
+The React + Vite app behind the `/creator` routes: the wizard that collects a course
+brief, and the syllabus editor where the user refines the generated outline before the
+course is created.
+
+It is a separate TypeScript project from the CLI. The root `tsconfig.json` explicitly
+excludes `src/creator`, and this app is ESM (`"type": "module"`) while the CLI is
+CommonJS. Nothing here is compiled by the CLI build.
+
+## ⚠️ Read this before you commit
+
+A PR that changes `src/creator/src/**` **must not** include `src/creatorDist/**`.
+
+`src/creatorDist/` is the compiled output of this app, committed to git because it is
+what the CLI serves at runtime and ships to npm. It is a **release artifact**: it gets
+rebuilt and committed only in the release commit, by `autoUpload.sh` at the repository
+root.
+
+Committing it in a feature PR causes conflicts that cannot be resolved by merging —
+Vite emits content-hashed filenames, so every rebuild rewrites `index.html` and adds a
+differently named asset.
+
+Corollary: **merging a creator change does not deploy it.** It ships when someone
+publishes.
+
+Full rationale in [`docs/creator-bundle.md`](../../docs/creator-bundle.md).
+
+## Commands
+
+Run these from `src/creator/`:
+
+```bash
+npm install          # this app has its own node_modules; the root install does not cover it
+npm run dev          # Vite dev server with HMR
+npm run build        # tsc -b && vite build -> ../creatorDist (not committed, see above)
+npm run watch        # vite build --watch, for testing against a local CLI server
+npm run test         # vitest; this app has its own runner, the root `npm test` does not cover it
+npm run lint         # ESLint
 ```
 
-You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules:
-
-```js
-// eslint.config.js
-import reactX from "eslint-plugin-react-x"
-import reactDom from "eslint-plugin-react-dom"
-
-export default tseslint.config({
-  plugins: {
-    // Add the react-x and react-dom plugins
-    "react-x": reactX,
-    "react-dom": reactDom,
-  },
-  rules: {
-    // other rules...
-    // Enable its recommended typescript rules
-    ...reactX.configs["recommended-typescript"].rules,
-    ...reactDom.configs.recommended.rules,
-  },
-})
+## Layout
+
 ```
+src/
+  App.tsx                     wizard: brief -> purpose -> duration -> human check
+  components/
+    NotificationListener.tsx  socket.io subscription to a Rigobot completion webhook
+    syllabus/                 syllabus editor and its chat sidebar
+  utils/
+    rigo.ts                   calls to the Rigobot prompting API
+    socket.ts                 socket.io client wrapper
+    store.ts                  zustand store, persisted to localStorage
+  locales/                    i18n resources (en, es)
+```
+
+## How generation works
+
+Course generation is asynchronous. `publicInteractiveCreation` posts to Rigobot with a
+`webhook_url` pointing at the CLI server (`POST /notifications/:id`), and Rigobot answers
+`201` immediately with a pending job. The CLI relays the eventual webhook over socket.io,
+and `NotificationListener` resolves it in the browser.
+
+That means **an HTTP 2xx from Rigobot says nothing about whether generation succeeded**.
+The job can still fail later and arrive over the socket with `status: "ERROR"` and
+`parsed: null`. Any code consuming a notification payload has to handle that.