@learnpack/learnpack 5.0.328 → 5.0.331

package/src/commands/serve.ts

@@ -75,6 +75,10 @@ import {
   NotFoundError,
 } from "../utils/errorHandler"
 import { JSDOM } from "jsdom"
+import { createClient } from "redis"
+import type { RedisClientType } from "redis"
+import { HistoryManager } from "../managers/historyManager"
+import { ReadmeHistoryService } from "../managers/readmeHistoryService"
 
 const frontMatter = require("front-matter")
 
@@ -796,7 +800,8 @@ async function processSyncTranslationsSequentially(
   sourceReadmeContent: string,
   targetLanguages: string[],
   rigoToken: string,
-  bucket: Bucket
+  bucket: Bucket,
+  historyManager: HistoryManager | null
 ) {
   try {
     // Process translations sequentially (no race conditions)
@@ -829,14 +834,33 @@ async function processSyncTranslationsSequentially(
           throw new Error("Translation result is empty")
         }
 
-        // Save translated README
-        const readmePath = `courses/${courseSlug}/exercises/${exerciseSlug}/README${getReadmeExtension(
-          targetLang
-        )}`
+        // Save translated README with history tracking
+        const fileName = ReadmeHistoryService.getReadmeFileName(targetLang)
+        const readmeHistoryService = new ReadmeHistoryService(
+          historyManager,
+          bucket
+        )
+
         // eslint-disable-next-line no-await-in-loop
-        await bucket
-          .file(readmePath)
-          .save(translationResult.parsed.translation)
+        const historyResult = await readmeHistoryService.saveReadmeWithHistory({
+          courseSlug,
+          exerciseSlug,
+          lang: targetLang,
+          fileName,
+          newContent: translationResult.parsed.translation,
+          // Don't pass previousContent or currentVersion - let service fetch them
+          throwOnConflict: false, // Don't fail sync on version conflicts
+        })
+
+        if (historyResult.success) {
+          console.log(
+            `🔄 SYNC: Saved ${targetLang} translation with history (version ${historyResult.newVersion})`
+          )
+        } else {
+          console.warn(
+            `🔄 SYNC: Translation saved but history failed for ${targetLang}: ${historyResult.error}`
+          )
+        }
 
         // Update progress in syllabus
         // eslint-disable-next-line no-await-in-loop
@@ -969,9 +993,11 @@ async function processSyncTranslationsSequentially(
   }
 }
 
-export default class ServeCommand extends SessionCommand {
+class ServeCommand extends SessionCommand {
   static description = "Runs a small server to build tutorials"
 
+  private redis: ReturnType<typeof createClient> | null = null
+
   static flags = {
     ...SessionCommand.flags,
     port: flags.string({ char: "p", description: "server port" }),
@@ -1015,6 +1041,64 @@ export default class ServeCommand extends SessionCommand {
       console.log("INFO: HOST is set to", host)
     }
 
+    // Initialize Redis (official client)
+    const redisUrl = process.env.REDIS_URL || "redis://localhost:6379"
+
+    if (
+      process.env.NODE_ENV === "production" &&
+      redisUrl === "redis://localhost:6379"
+    ) {
+      console.error("❌ REDIS_URL not configured for production environment!")
+      console.warn("⚠️  History features will be unavailable")
+      this.redis = null
+    } else {
+      try {
+        const useTLS = redisUrl.startsWith("rediss://")
+
+        this.redis = createClient({
+          url: redisUrl,
+          socket: {
+            tls: useTLS,
+            rejectUnauthorized: useTLS,
+            reconnectStrategy: (retries: number) => {
+              if (retries > 10) {
+                console.error("❌ Too many Redis reconnection attempts")
+                return new Error("Too many retries")
+              }
+
+              // Exponential backoff: 50ms, 100ms, 200ms, ...
+              const delay = Math.min(retries * 50, 2000)
+              return delay
+            },
+          },
+        })
+
+        this.redis.on("error", (err: Error) => {
+          console.error("❌ Redis error:", err)
+        })
+
+        this.redis.on("connect", () => {
+          console.log("🔄 Connecting to Redis...")
+        })
+
+        this.redis.on("ready", () => {
+          console.log("✅ Connected to Redis")
+        })
+
+        // Connect the client
+        await this.redis.connect()
+      } catch (redisError) {
+        console.error("❌ Failed to connect to Redis:", redisError)
+        console.warn("⚠️  History features (undo/redo) will be unavailable")
+        this.redis = null
+      }
+    }
+
+    // Create HistoryManager instance if Redis is available
+    const historyManager = this.redis ?
+      new HistoryManager(this.redis as RedisClientType) :
+      null
+
     // async function listFilesWithPrefix(prefix: string) {
     //   const [files] = await bucket.getFiles({ prefix })
     //   return files
@@ -2573,11 +2657,15 @@ export default class ServeCommand extends SessionCommand {
 
     app.put(
       "/exercise/:slug/file/:fileName",
-      express.text(),
+      express.json(),
       asyncHandler(async (req, res) => {
         const { slug, fileName } = req.params
         const query = req.query
         const courseSlug = query.slug as string
+        const lang = (query.lang as string) || "en"
+
+        // Headers for history management
+        const versionId = (req.headers["x-history-version"] as string) || "0"
 
         console.log(`PUT /exercise/${slug}/file/${fileName}`)
 
@@ -2592,21 +2680,64 @@ export default class ServeCommand extends SessionCommand {
           throw new ValidationError("File name and exercise slug are required")
         }
 
+        // Extract content from body (can be text or JSON)
+        let fileContent: string
+        let contentToSaveInHistory: string | undefined
+
+        if (typeof req.body === "string") {
+          // Old format: plain text
+          fileContent = req.body
+        } else if (req.body && typeof req.body === "object") {
+          // New format: JSON with content and historyContent
+          fileContent = req.body.content
+          contentToSaveInHistory = req.body.historyContent
+        } else {
+          throw new ValidationError("Invalid request body format")
+        }
+
         try {
-          // Update the file in the bucket
-          const file = bucket.file(
-            `courses/${courseSlug}/exercises/${slug}/${fileName}`
-          )
+          let newVersion = versionId
+          let created = false
 
-          await file.save(req.body, {
-            resumable: false,
-          })
+          // Use ReadmeHistoryService for README files with history support
+          if (fileName.startsWith("README")) {
+            const readmeHistoryService = new ReadmeHistoryService(
+              historyManager,
+              bucket
+            )
+
+            const historyResult =
+              await readmeHistoryService.saveReadmeWithHistory({
+                courseSlug,
+                exerciseSlug: slug,
+                lang,
+                fileName,
+                newContent: fileContent,
+                previousContent: contentToSaveInHistory,
+                currentVersion: versionId,
+                throwOnConflict: true, // Throw error on version conflicts for manual edits
+              })
+
+            newVersion = historyResult.newVersion
+            created = true // File was saved by the service
+          } else {
+            // For non-README files, save directly without history
+            const file = bucket.file(
+              `courses/${courseSlug}/exercises/${slug}/${fileName}`
+            )
+
+            await file.save(fileContent, {
+              resumable: false,
+            })
 
-          const created = await file.exists()
+            const [exists] = await file.exists()
+            created = exists
+          }
 
-          res.send({
+          res.json({
             message: "File updated",
             created,
+            version: newVersion,
           })
         } catch (error: any) {
           // Handle Google Cloud Storage rate limit errors (429)
@@ -2637,6 +2768,304 @@ export default class ServeCommand extends SessionCommand {
         }
       })
     )
+
+    // Remove code_challenge_proposals from all README files of an exercise
+    app.post(
+      "/exercise/:slug/remove-code-proposals",
+      asyncHandler(async (req, res) => {
+        const { slug } = req.params
+        const { courseSlug } = req.body
+
+        // Validate required parameters
+        if (!courseSlug) {
+          throw new ValidationError("courseSlug is required in request body")
+        }
+
+        console.log(
+          `🧹 Cleaning code_challenge_proposals from exercise: ${slug} in course: ${courseSlug}`
+        )
+
+        try {
+          const exerciseDir = `courses/${courseSlug}/exercises/${slug}`
+
+          // Get all README files (all languages)
+          const [readmeFiles] = await bucket.getFiles({
+            prefix: `${exerciseDir}/README`,
+          })
+
+          if (readmeFiles.length === 0) {
+            return res.status(404).json({
+              success: false,
+              error: "No README files found for this exercise",
+            })
+          }
+
+          let cleanedCount = 0
+          const processedFiles: string[] = []
+
+          // Process each README file
+          for (const readmeFile of readmeFiles) {
+            try {
+              // eslint-disable-next-line no-await-in-loop
+              const [readmeContent] = await readmeFile.download()
+              const readmeText = readmeContent.toString()
+
+              // Remove code_challenge_proposal blocks, EXCEPT those in GENERATING state
+              // The GENERATING state is identified by "GENERATING(" marker
+              // Pattern: ```code_challenge_proposal ... ``` (but not if contains GENERATING)
+              const codeProposalRegex =
+                /(\r?\n)?```code_challenge_proposal(?![\S\s]*?GENERATING\()[\S\s]*?```(\r?\n)?/g
+
+              const updatedReadme = readmeText.replace(codeProposalRegex, "")
+
+              // Only update if there were actual changes
+              if (updatedReadme.trim() !== readmeText.trim()) {
+                // eslint-disable-next-line no-await-in-loop
+                await readmeFile.save(updatedReadme.trim(), {
+                  resumable: false,
+                })
+
+                cleanedCount++
+                processedFiles.push(readmeFile.name)
+                console.log(`✅ Cleaned proposals from: ${readmeFile.name}`)
+              }
+            } catch (readmeError) {
+              console.error(
+                `❌ Error cleaning README ${readmeFile.name}:`,
+                readmeError
+              )
+              // Continue processing other files even if one fails
+            }
+          }
+
+          const message =
+            cleanedCount > 0 ?
+              `Successfully cleaned ${cleanedCount} README file(s)` :
+              "No code challenge proposals found to clean"
+
+          console.log(`✅ ${message}`)
+
+          res.json({
+            success: true,
+            message,
+            cleanedCount,
+            processedFiles,
+          })
+        } catch (error: any) {
+          console.error(
+            "❌ Error during code challenge proposal cleanup:",
+            error
+          )
+
+          // Handle rate limit errors
+          if (
+            error.code === 429 ||
+            error.message?.includes("rate limit") ||
+            error.message?.includes("rateLimitExceeded")
+          ) {
+            throw new ConflictError(
+              "Storage rate limit exceeded. Please try again in a few moments.",
+              {
+                code: "STORAGE_RATE_LIMIT",
+                retryAfter: 60,
+              }
+            )
+          }
+
+          throw error
+        }
+      })
+    )
+
+    // GET /exercise/:slug/history/status
+    // Get history status (can undo/redo, current version)
+    app.get(
+      "/exercise/:slug/history/status",
+      asyncHandler(async (req, res) => {
+        const { slug } = req.params
+        const courseSlug = req.query.slug as string
+        const lang = (req.query.lang as string) || "en"
+
+        if (!courseSlug) {
+          throw new ValidationError("Course slug is required")
+        }
+
+        if (!historyManager) {
+          return res.json({
+            canUndo: false,
+            canRedo: false,
+            version: "0",
+            available: false,
+          })
+        }
+
+        const status = await historyManager.getHistoryStatus(
+          courseSlug,
+          slug,
+          lang
+        )
+
+        res.json({
+          ...status,
+          available: true,
+        })
+      })
+    )
+
+    // POST /exercise/:slug/history/undo
+    // Undo the last change
+    app.post(
+      "/exercise/:slug/history/undo",
+      express.json(),
+      asyncHandler(async (req, res) => {
+        const { slug } = req.params
+        const { courseSlug, currentContent, versionId, lang } = req.body
+
+        if (!courseSlug || !currentContent || !versionId) {
+          throw new ValidationError(
+            "Missing required fields: courseSlug, currentContent, versionId"
+          )
+        }
+
+        const language = lang || "en"
+
+        if (!historyManager) {
+          console.error("⏮️  HISTORY: History manager not available")
+          throw new InternalServerError("History service not available")
+        }
+
+        const result = await historyManager.undo(
+          courseSlug,
+          slug,
+          language,
+          currentContent,
+          versionId
+        )
+
+        if (!result.success) {
+          if (result.error === "VERSION_CONFLICT") {
+            throw new ConflictError(
+              "Version conflict. Please refresh and try again."
+            )
+          }
+
+          if (result.error === "NO_UNDO_AVAILABLE") {
+            throw new ValidationError("No more changes to undo")
+          }
+
+          throw new InternalServerError(
+            `Undo operation failed: ${result.error}`
+          )
+        }
+
+        if (!result.content) {
+          throw new InternalServerError(
+            "No content returned from undo operation"
+          )
+        }
+
+        // Save the restored content to GCS
+        const fileName = `README${getReadmeExtension(language)}`
+        const file = bucket.file(
+          `courses/${courseSlug}/exercises/${slug}/${fileName}`
+        )
+
+        try {
+          await file.save(result.content, {
+            resumable: false,
+          })
+        } catch (gcsError) {
+          console.error(
+            "⏮️  HISTORY: Failed to save restored content to GCS:",
+            gcsError
+          )
+          throw new InternalServerError("Failed to save restored content")
+        }
+
+        res.json({
+          content: result.content,
+          version: result.newVersion,
+        })
+      })
+    )
+
+    // POST /exercise/:slug/history/redo
+    // Redo the last undone change
+    app.post(
+      "/exercise/:slug/history/redo",
+      express.json(),
+      asyncHandler(async (req, res) => {
+        const { slug } = req.params
+        const { courseSlug, currentContent, versionId, lang } = req.body
+
+        if (!courseSlug || !currentContent || !versionId) {
+          throw new ValidationError(
+            "Missing required fields: courseSlug, currentContent, versionId"
+          )
+        }
+
+        const language = lang || "en"
+
+        if (!historyManager) {
+          console.error("⏮️  HISTORY: History manager not available")
+          throw new InternalServerError("History service not available")
+        }
+
+        const result = await historyManager.redo(
+          courseSlug,
+          slug,
+          language,
+          currentContent,
+          versionId
+        )
+
+        if (!result.success) {
+          if (result.error === "VERSION_CONFLICT") {
+            throw new ConflictError(
+              "Version conflict. Please refresh and try again."
+            )
+          }
+
+          if (result.error === "NO_REDO_AVAILABLE") {
+            throw new ValidationError("No more changes to redo")
+          }
+
+          throw new InternalServerError(
+            `Redo operation failed: ${result.error}`
+          )
+        }
+
+        if (!result.content) {
+          throw new InternalServerError(
+            "No content returned from redo operation"
+          )
+        }
+
+        // Save the restored content to GCS
+        const fileName = `README${getReadmeExtension(language)}`
+        const file = bucket.file(
+          `courses/${courseSlug}/exercises/${slug}/${fileName}`
+        )
+
+        try {
+          await file.save(result.content, {
+            resumable: false,
+          })
+        } catch (gcsError) {
+          console.error(
+            "⏮️  HISTORY: Failed to save restored content to GCS:",
+            gcsError
+          )
+          throw new InternalServerError("Failed to save restored content")
+        }
+
+        res.json({
+          content: result.content,
+          version: result.newVersion,
+        })
+      })
+    )
+
     // Create a new step for a course
     app.post("/course/:slug/create-step", async (req, res) => {
       console.log("POST /course/:slug/create-step")
@@ -3417,7 +3846,8 @@ export default class ServeCommand extends SessionCommand {
             sourceReadmeContent,
             targetLanguages,
             rigoToken,
-            bucket
+            bucket,
+            historyManager
           ).catch(error => {
             console.error("Error in sync translation processing:", error)
 
@@ -4668,3 +5098,5 @@ export default class ServeCommand extends SessionCommand {
     })
   }
 }
+
+export default ServeCommand

package/src/lua/redo.lua

@@ -0,0 +1,48 @@
+-- Lua script to redo the last undone change
+-- Implements Memento pattern with optimistic locking
+--
+-- KEYS[1]: undo:{course}:{exercise}:{lang}
+-- KEYS[2]: redo:{course}:{exercise}:{lang}
+-- KEYS[3]: undo:{course}:{exercise}:{lang}:version
+-- ARGV[1]: current_state_json (state before redo, to save in undo)
+-- ARGV[2]: version_id (current client version for optimistic locking)
+-- ARGV[3]: ttl (432000 seconds = 5 days)
+
+-- Get current version
+local current_version = redis.call('GET', KEYS[3])
+
+-- Verify version (optimistic locking)
+if current_version ~= false and current_version ~= ARGV[2] then
+    return {err = 'VERSION_CONFLICT'}
+end
+
+-- Verify there are states to redo
+local redo_length = redis.call('LLEN', KEYS[2])
+if redo_length == 0 then
+    return {err = 'NO_REDO_AVAILABLE'}
+end
+
+-- Get future state (the most recent in redo stack)
+local future_state = redis.call('LPOP', KEYS[2])
+
+if not future_state then
+    return {err = 'NO_FUTURE_STATE'}
+end
+
+-- Save current state in undo stack
+redis.call('LPUSH', KEYS[1], ARGV[1])
+
+-- Maintain state limit in undo (last 5)
+redis.call('LTRIM', KEYS[1], 0, 4)
+
+-- Renew TTL of both stacks
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[3]))
+redis.call('EXPIRE', KEYS[2], tonumber(ARGV[3]))
+
+-- Increment version and renew its TTL
+local new_version = redis.call('INCR', KEYS[3])
+redis.call('EXPIRE', KEYS[3], tonumber(ARGV[3]))
+
+-- Return array [newVersion, futureState] for compatibility with Redis client
+return {tostring(new_version), future_state}
+

package/src/lua/saveState.lua

@@ -0,0 +1,40 @@
+-- Lua script to save a new state in history
+-- Implements Memento pattern with optimistic locking
+-- NOTE: We save the state BEFORE changes, so users can undo to it
+--
+-- KEYS[1]: undo:{course}:{exercise}:{lang}
+-- KEYS[2]: redo:{course}:{exercise}:{lang}
+-- KEYS[3]: undo:{course}:{exercise}:{lang}:version
+-- ARGV[1]: state_json (complete README BEFORE changes - serialized)
+-- ARGV[2]: version_id (current client version for optimistic locking)
+-- ARGV[3]: ttl (432000 seconds = 5 days)
+-- ARGV[4]: limit (5 states maximum)
+
+-- Get current version
+local current_version = redis.call('GET', KEYS[3])
+
+-- Verify version (optimistic locking)
+-- If the sent version doesn't match the current one, reject
+if current_version ~= false and current_version ~= ARGV[2] then
+    return {err = 'VERSION_CONFLICT'}
+end
+
+-- Save new state in undo stack (at the beginning)
+redis.call('LPUSH', KEYS[1], ARGV[1])
+
+-- Invalidate redo stack (new change invalidates the "alternative future")
+redis.call('DEL', KEYS[2])
+
+-- Keep only the last N states (0 to limit-1)
+redis.call('LTRIM', KEYS[1], 0, tonumber(ARGV[4]) - 1)
+
+-- Renew TTL of undo stack
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[3]))
+
+-- Increment version and renew its TTL
+local new_version = redis.call('INCR', KEYS[3])
+redis.call('EXPIRE', KEYS[3], tonumber(ARGV[3]))
+
+-- Return new version (as string for compatibility with Redis client)
+return tostring(new_version)
+

package/src/lua/undo.lua

@@ -0,0 +1,50 @@
+-- Lua script to undo the last change
+-- Implements Memento pattern with optimistic locking
+-- NOTE: Stack contains states BEFORE changes, so index 0 is what we want to restore
+--
+-- KEYS[1]: undo:{course}:{exercise}:{lang}
+-- KEYS[2]: redo:{course}:{exercise}:{lang}
+-- KEYS[3]: undo:{course}:{exercise}:{lang}:version
+-- ARGV[1]: current_state_json (current state to save in redo for potential redo)
+-- ARGV[2]: version_id (current client version for optimistic locking)
+-- ARGV[3]: ttl (432000 seconds = 5 days)
+
+-- Get current version
+local current_version = redis.call('GET', KEYS[3])
+
+-- Verify version (optimistic locking)
+if current_version ~= false and current_version ~= ARGV[2] then
+    return {err = 'VERSION_CONFLICT'}
+end
+
+-- Verify there are states to undo
+-- We need at least 1 element (the state before the current change)
+local undo_length = redis.call('LLEN', KEYS[1])
+if undo_length < 1 then
+    return {err = 'NO_UNDO_AVAILABLE'}
+end
+
+-- Get the saved state (index 0 - the state before current changes)
+local previous_state = redis.call('LINDEX', KEYS[1], 0)
+
+if not previous_state then
+    return {err = 'NO_PREVIOUS_STATE'}
+end
+
+-- Move current state to redo stack
+redis.call('LPUSH', KEYS[2], ARGV[1])
+
+-- Remove current state from undo stack (index 0)
+redis.call('LPOP', KEYS[1])
+
+-- Renew TTL of both stacks
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[3]))
+redis.call('EXPIRE', KEYS[2], tonumber(ARGV[3]))
+
+-- Increment version and renew its TTL
+local new_version = redis.call('INCR', KEYS[3])
+redis.call('EXPIRE', KEYS[3], tonumber(ARGV[3]))
+
+-- Return array [newVersion, previousState] for compatibility with Redis client
+return {tostring(new_version), previous_state}
+