@learnpack/learnpack 5.0.364 → 5.0.366

package/src/commands/serve.ts

@@ -39,6 +39,7 @@ import {
   addInteractivity,
   generateCodeChallenge,
   generateStepSlug,
+  getCompletionJob,
 } from "../utils/rigoActions"
 import * as dotenv from "dotenv"
 
@@ -55,6 +56,11 @@ import {
   planGenerationRetry,
   RetryPlan,
 } from "../utils/generationRetry"
+import {
+  currentJobId,
+  isGenerationRunning,
+  resolveGenerationMode,
+} from "../utils/continueGeneration"
 // import { handleAssetCreation } from "./publish"
 import axios from "axios"
 import * as FormData from "form-data"
@@ -2297,6 +2303,35 @@ class ServeCommand extends SessionCommand {
             )
           }
 
+          const language = syllabus.courseInfo.language || "en"
+          const previousStatus = exercise.status
+
+          // A second click, or a retry while the job still runs, would start
+          // another paid job for the same lesson.
+          if (previousStatus === "GENERATING") {
+            const jobId = currentJobId(exercise, language)
+            let job = null
+            if (jobId) {
+              try {
+                job = await getCompletionJob(rigoToken, jobId, {
+                  timeoutMs: 5000,
+                })
+              } catch (error) {
+                console.error(`Could not check job ${jobId}:`, error)
+              }
+            }
+
+            if (isGenerationRunning(exercise, language, job, Date.now())) {
+              console.log(
+                `Lesson ${lessonUid} is already being generated (job ${jobId}), not starting another one`
+              )
+              throw new ConflictError(
+                `Lesson "${exercise.title}" is already being generated`,
+                { code: "LESSON_ALREADY_GENERATING", completionId: jobId }
+              )
+            }
+          }
+
           // previous exercise
           let previousReadme = "---"
           const previousExercise = syllabus.lessons[exerciseIndex - 1]
@@ -2334,7 +2369,7 @@ class ServeCommand extends SessionCommand {
           // A new job by the user, so the automatic retries start over.
           syllabus.lessons[exerciseIndex].status = "GENERATING"
           syllabus.lessons[exerciseIndex].translations = {
-            [syllabus.courseInfo.language || "en"]: startedJob(completionId),
+            [language]: startedJob(completionId),
           }
           console.log("Lesson", syllabus.lessons[exerciseIndex])
           if (syllabus.feedback && typeof syllabus.feedback === "string") {
@@ -2343,7 +2378,11 @@ class ServeCommand extends SessionCommand {
             syllabus.feedback = feedback
           }
 
-          syllabus.generationMode = mode
+          syllabus.generationMode = resolveGenerationMode(
+            previousStatus,
+            syllabus.generationMode,
+            mode
+          )
 
           await saveSyllabus(courseSlug, syllabus, bucket)
         })
@@ -5479,11 +5518,19 @@ class ServeCommand extends SessionCommand {
         syllabus.courseInfo.language
       )
 
+      const language = syllabus.courseInfo.language || "en"
       const initialSyllabus = {
         ...syllabus,
         lessons: syllabus.lessons.map((lesson, index) => {
           if (index < 1) {
-            return { ...lesson, generated: false, status: "GENERATING" }
+            // Its start time, so a retry right away is refused even before
+            // the job id is saved below.
+            return {
+              ...lesson,
+              generated: false,
+              status: "GENERATING",
+              translations: { [language]: startedJob(0) },
+            }
           }
 
           return {
@@ -5508,25 +5555,25 @@ class ServeCommand extends SessionCommand {
 
       const lastResult = "---"
 
-      // Use new two-phase generation workflow
-      const completionId = await startInitialContentGeneration(
-        rigoToken,
-        syllabus.lessons,
-        syllabus.courseInfo,
-        firstLesson,
-        courseSlug,
-        syllabus.courseInfo.purpose,
-        lastResult
-      )
-      if (firstLesson) {
-        firstLesson.translations = {
-          [syllabus.courseInfo.language || "en"]: {
-            completionId,
-            startedAt: Date.now(),
-            completedAt: 0,
-          },
-        }
-      }
+      // In the course queue, so the webhook of this job waits until its id
+      // is saved. The id was never saved before, so the first lesson had no
+      // protection against a second job.
+      await serializeByCourse(courseSlug, async () => {
+        // Use new two-phase generation workflow
+        const completionId = await startInitialContentGeneration(
+          rigoToken,
+          syllabus.lessons,
+          syllabus.courseInfo,
+          firstLesson,
+          courseSlug,
+          syllabus.courseInfo.purpose,
+          lastResult
+        )
+
+        await patchLesson(courseSlug, firstLesson.uid, bucket, lesson => {
+          lesson.translations = { [language]: startedJob(completionId) }
+        })
+      })
 
       await createInitialReadme(
         JSON.stringify(syllabus.courseInfo),

package/src/managers/config/index.ts

@@ -17,6 +17,7 @@ import defaults from "./defaults"
 import { exercise } from "./exercise"
 
 import { rmSync } from "../file"
+import { getLatestEditorVersion } from "../ideReleases"
 import { IConfigObj, TMode, TAgent } from "../../models/config"
 import {
   IConfigManagerAttributes,
@@ -29,9 +30,7 @@ import { IExercise } from "../../models/exercise-obj"
 const execAsync = promisify(exec)
 
 // eslint-disable-next-line
-const fetch = require("node-fetch")
-// eslint-disable-next-line
-const chalk = require("chalk")
+const chalk = require("chalk");
 
 /* exercise folder name standard */
 
@@ -167,8 +166,7 @@ export default async ({
       configPath: confPath.config,
     },
   })
-  if (!configObj.config) 
-throw InternalError("Config object not found")
+  if (!configObj.config) throw InternalError("Config object not found")
 
   configObj.config.outputPath = confPath.base + "/dist"
 
@@ -263,15 +261,10 @@ throw InternalError("Config object not found")
   configObj.config.editor.mode =
     process.env.TERM_PROGRAM === "vscode" ? "extension" : "preview"
 
-  if (version) 
-configObj.config.editor.version = version
+  if (version) configObj.config.editor.version = version
   else if (configObj.config.editor.version === null) {
     Console.debug("Config version not found, downloading default.")
-    const resp = await fetch(
-      "https://raw.githubusercontent.com/learnpack/ide/master/package.json"
-    )
-    const packageJSON = await resp.json()
-    configObj.config.editor.version = packageJSON.version || "5.0.0"
+    configObj.config.editor.version = await getLatestEditorVersion()
   }
 
   configObj.config.dirPath = "./" + confPath.base
@@ -303,8 +296,7 @@ configObj.config.editor.version = version
 
               // Execute the code and replace the value
               const { stdout } = await execAsync(variable.source)
-              if (!configObj.config || !configObj.config.variables) 
-return
+              if (!configObj.config || !configObj.config.variables) return
 
               configObj.config.variables[v] = stdout.trim()
             } catch (error) {
@@ -332,16 +324,14 @@ return
         const map: any = {
           python3: "python",
         }
-        if (map[_l]) 
-return map[_l]
+        if (map[_l]) return map[_l]
         return _l
       }
 
       // decode aliases
       language = alias(language)
 
-      if (this.validLanguages[language]) 
-return true
+      if (this.validLanguages[language]) return true
 
       Console.debug(`Validating engine for ${language} compilation`)
       let result = shell.exec("learnpack plugins", { silent: true })
@@ -430,8 +420,7 @@ return true
       const exercise = (configObj.exercises || []).find(
         ex => ex.slug === slug
       )
-      if (!exercise) 
-throw ValidationError(`Exercise ${slug} not found`)
+      if (!exercise) throw ValidationError(`Exercise ${slug} not found`)
 
       return exercise
     },
@@ -450,8 +439,7 @@ throw ValidationError(`Exercise ${slug} not found`)
       exercise.files.forEach((f: IFile) => {
         if (configObj.config) {
           const _path = configObj.config.outputPath + "/" + f.name
-          if (f.hidden === false && fs.existsSync(_path)) 
-fs.unlinkSync(_path)
+          if (f.hidden === false && fs.existsSync(_path)) fs.unlinkSync(_path)
         }
       })
 
@@ -472,7 +460,9 @@ fs.unlinkSync(_path)
         ex => ex.slug === slug
       )
       if (!exercise)
-        throw ValidationError(`Exercise ${slug} not found on the configuration`)
+        throw ValidationError(
+          `Exercise ${slug} not found on the configuration`
+        )
 
       if (configObj.config) {
         for (const fileName of fs.readdirSync(
@@ -494,8 +484,7 @@ fs.unlinkSync(_path)
         if (name === path.basename(configObj?.config?.dirPath || ""))
           return false
         // ignore folders that start with a dot
-        if (name.charAt(0) === "." || name.charAt(0) === "_") 
-return false
+        if (name.charAt(0) === "." || name.charAt(0) === "_") return false
 
         return fs.lstatSync(source).isDirectory()
       }
@@ -506,8 +495,7 @@ return false
           .map(name => path.join(source, name))
           .filter(isDirectory)
       // add the .learn folder
-      if (!fs.existsSync(confPath.base)) 
-fs.mkdirSync(confPath.base)
+      if (!fs.existsSync(confPath.base)) fs.mkdirSync(confPath.base)
       // add the outout folder where webpack will publish the the html/css/js files
       if (
         configObj.config &&
@@ -537,8 +525,7 @@ fs.mkdirSync(confPath.base)
     createExercise: (slug: string, content: string, language: string) => {
       try {
         const dirPath = `${configObj.config?.exercisesPath}/${slug}`
-        if (!fs.existsSync(dirPath)) 
-fs.mkdirSync(dirPath, { recursive: true })
+        if (!fs.existsSync(dirPath)) fs.mkdirSync(dirPath, { recursive: true })
         const isEnglish = language === "us" || language === "en"
         const fileName = isEnglish ? "README.md" : `README.${language}.md`
         fs.writeFileSync(`${dirPath}/${fileName}`, content)
@@ -623,8 +610,7 @@ function deepMerge(...sources: any): any {
           value = deepMerge(acc[key], value)
         }
 
-        if (value !== undefined) 
-acc = { ...acc, [key]: value }
+        if (value !== undefined) acc = { ...acc, [key]: value }
       }
     }
   }

package/src/managers/file.ts

@@ -6,9 +6,18 @@ import * as targz from "targz"
 import Console from "../utils/console"
 import * as https from "https"
 import { InternalError } from "../utils/errors"
+import {
+  FETCH_TIMEOUT_MS,
+  IDE_CLOUDFRONT_MANIFEST,
+  IDE_GITHUB_MANIFEST,
+  editorTarballUrl,
+  fetchTagList,
+  getLatestEditorVersion,
+  resolveEditorRelease,
+} from "./ideReleases"
 
 // eslint-disable-next-line
-const fetch = require("node-fetch")
+const fetch = require("node-fetch");
 
 export const decompress = (sourcePath: string, destinationPath: string) =>
   new Promise((resolve, reject) => {
@@ -34,81 +43,33 @@ export const downloadEditor = async (
   version: string | undefined,
   destination: string
 ) => {
-  Console.info("Downloading the LearnPack coding UI, this may take a minute...")
+  Console.info(
+    "Downloading the LearnPack coding UI, this may take a minute..."
+  )
 
-  if (!version) {
-    const res = await fetch(
-      "https://raw.githubusercontent.com/learnpack/ide/master/package.json"
-    )
-    const json = await res.json()
-    version = json.version || "4.0.0"
+  if (!version) version = await getLatestEditorVersion()
 
-    if (!version)
-      throw InternalError(
-        `Coding Editor version was not found on learnpack repository, check the config.editor.version property on learn.json`
-      )
-  }
+  const [cloudTags, githubTags] = await Promise.all([
+    fetchTagList(IDE_CLOUDFRONT_MANIFEST),
+    fetchTagList(IDE_GITHUB_MANIFEST),
+  ])
 
-  const [major, minor] = version.split(".")
-  const versionPrefix = `${major}.${minor}`
-
-  let tags
-  try {
-    const tagsRes = await fetch(
-      "https://raw.githubusercontent.com/learnpack/ide/master/versions.json"
-    )
-    tags = await tagsRes.json()
-
-    if (!Array.isArray(tags)) {
-      throw new TypeError(`Invalid versions.json format`)
-    }
-  } catch (error) {
+  let release = resolveEditorRelease(version, cloudTags, githubTags)
+  if (!release) {
     Console.debug(
-      "Error reading versions.json, defaulting to version 4.0.2",
-      error
+      `No IDE version found for ${version} in the versions manifests, defaulting to version 4.0.2`
     )
-    version = "4.0.2"
-    tags = [`learnpack-${version}.tar.gz`]
+    release = { version: "4.0.2", source: "github" }
   }
 
-  let matchingTags
-  try {
-    if (tags.length === 0 || !tags)
-      throw InternalError(`No found tags in versions.json ${tags}`)
-
-    matchingTags = tags
-      .filter(tag => tag.includes(`learnpack-${versionPrefix}`))
-      .sort((a, b) => b.localeCompare(a, undefined, { numeric: true }))
-
-    if (matchingTags.length === 0)
-      throw InternalError(
-        `No matching version found for prefix ${versionPrefix} in the versions.json`
-      )
-  } catch (error) {
-    Console.debug("Error processing tags, defaulting to version 4.0.2", error)
-    version = "4.0.2"
-    matchingTags = [`learnpack-${version}.tar.gz`]
-  }
-
-  const latestVersion = matchingTags[0]
-    .replace("learnpack-", "")
-    .replace(".tar.gz", "")
-
-  const versionNumber = parseInt(latestVersion.split(".")[0])
-
-  let url = `https://github.com/learnpack/coding-ide/blob/${latestVersion}/dist`
-
-  if (versionNumber > 2)
-    url = `https://github.com/learnpack/ide/blob/master/bin/learnpack-${latestVersion}.tar.gz`
-
-  const resp2 = await fetch(url, { method: "HEAD" })
-  if (!resp2.ok)
+  const url = editorTarballUrl(release.version, release.source)
+  const resp = await fetch(url, { method: "HEAD", timeout: FETCH_TIMEOUT_MS })
+  if (!resp.ok)
     throw InternalError(
-      `Coding Editor ${latestVersion} was not found on learnpack repository, check the config.editor.version property on learn.json`
+      `Coding Editor ${release.version} was not found on ${release.source}, check the config.editor.version property on learn.json`
     )
 
-  url += versionNumber > 2 ? `?raw=true` : `/app.tar.gz?raw=true`
-
+  Console.debug(`Using IDE ${release.version} from ${release.source}`)
   return download(url, destination)
 }
 
@@ -123,7 +84,7 @@ export const download = (url: string, dest: string) => {
         })
         file.on("error", err => {
           file.close()
-          if (err.name === "EEXIST") {
+          if ((err as NodeJS.ErrnoException).code === "EEXIST") {
             Console.debug("File already exists")
             resolve("File already exists")
           } else {
@@ -132,11 +93,10 @@ export const download = (url: string, dest: string) => {
           }
         })
         response.pipe(file)
-      } else if (response.statusCode === 302 || response.statusCode === 301) {
-        // Console.debug("Servers redirected to "+response.headers.location)
+      } else if ([301, 302, 303, 307, 308].includes(response.statusCode || 0)) {
         // Recursively follow redirects, only a 200 will resolve.
         if (response.headers.location) {
-          download(response.headers.location, dest)
+          download(new URL(response.headers.location, url).toString(), dest)
             .then(() => resolve(/* */ ""))
             .catch(error => {
               Console.error(error)

package/src/managers/ideReleases.ts

@@ -0,0 +1,142 @@
+import Console from "../utils/console"
+
+// eslint-disable-next-line
+const fetch = require("node-fetch");
+
+// IDE tarballs published from 5.0.417 on live on CloudFront; older ones stay in the
+// learnpack/ide GitHub repo, whose versions.json is frozen. Both lists are merged.
+export const IDE_CLOUDFRONT_BASE =
+  "https://d3a7w3e3g8ydll.cloudfront.net/learnpack/ide"
+export const IDE_CLOUDFRONT_MANIFEST = `${IDE_CLOUDFRONT_BASE}/versions.json`
+export const IDE_GITHUB_MANIFEST =
+  "https://raw.githubusercontent.com/learnpack/ide/master/versions.json"
+export const IDE_GITHUB_PACKAGE =
+  "https://raw.githubusercontent.com/learnpack/ide/master/package.json"
+
+export const FETCH_TIMEOUT_MS = 10_000
+const DEFAULT_EDITOR_VERSION = "5.0.0"
+
+export type TEditorSource = "cloudfront" | "github";
+
+export interface IEditorRelease {
+  version: string;
+  source: TEditorSource;
+}
+
+interface IParsedVersion {
+  version: string;
+  major: number;
+  minor: number;
+  patch: number;
+}
+
+export const parseVersion = (version: string): IParsedVersion | null => {
+  const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(version.trim())
+  if (!match) return null
+  return {
+    version: match.slice(1, 4).join("."),
+    major: Number(match[1]),
+    minor: Number(match[2]),
+    patch: Number(match[3]),
+  }
+}
+
+// "learnpack-5.0.417.tar.gz" -> parsed 5.0.417; anything else -> null
+export const parseTag = (tag: string): IParsedVersion | null => {
+  const match = /^learnpack-(\d+\.\d+\.\d+)\.tar\.gz$/.exec(tag)
+  return match ? parseVersion(match[1]) : null
+}
+
+const compareVersions = (a: IParsedVersion, b: IParsedVersion): number =>
+  a.major - b.major || a.minor - b.minor || a.patch - b.patch
+
+const parseTags = (tags: string[] | null): IParsedVersion[] =>
+  (tags || []).map(parseTag).filter((v): v is IParsedVersion => v !== null)
+
+// Highest version listed in a manifest, or null when there is none.
+export const highestVersion = (tags: string[] | null): string | null => {
+  const sorted = parseTags(tags).sort(compareVersions)
+  return sorted.length > 0 ? sorted[sorted.length - 1].version : null
+}
+
+// Picks the tarball to download for the requested version. Only major.minor of the
+// request is used and the highest patch available wins, as the CLI always did.
+// The source is CloudFront when the version is in its manifest, GitHub otherwise.
+export const resolveEditorRelease = (
+  requested: string,
+  cloudTags: string[] | null,
+  githubTags: string[] | null
+): IEditorRelease | null => {
+  const [major, minor] = requested.split(".").map(part => Number(part))
+  if (!Number.isInteger(major) || !Number.isInteger(minor)) return null
+
+  const cloud = parseTags(cloudTags)
+  const candidates = [...cloud, ...parseTags(githubTags)]
+    .filter(v => v.major === major && v.minor === minor)
+    .sort(compareVersions)
+
+  if (candidates.length === 0) return null
+
+  const { version } = candidates[candidates.length - 1]
+  const source: TEditorSource = cloud.some(v => v.version === version) ?
+    "cloudfront" :
+    "github"
+
+  return { version, source }
+}
+
+export const editorTarballUrl = (
+  version: string,
+  source: TEditorSource
+): string => {
+  if (source === "cloudfront") {
+    return `${IDE_CLOUDFRONT_BASE}/learnpack-${version}.tar.gz`
+  }
+
+  const major = Number.parseInt(version.split(".")[0], 10)
+  return major > 2 ?
+    `https://github.com/learnpack/ide/blob/master/bin/learnpack-${version}.tar.gz?raw=true` :
+    `https://github.com/learnpack/coding-ide/blob/${version}/dist/app.tar.gz?raw=true`
+}
+
+// Fetches a versions manifest. Never throws: any failure returns null.
+export const fetchTagList = async (url: string): Promise<string[] | null> => {
+  try {
+    const res = await fetch(url, { timeout: FETCH_TIMEOUT_MS })
+    if (!res.ok) {
+      Console.debug(`Could not read ${url}: ${res.status} ${res.statusText}`)
+      return null
+    }
+
+    const json = await res.json()
+    if (!Array.isArray(json) || !json.every(tag => typeof tag === "string")) {
+      Console.debug(`Invalid versions manifest format at ${url}`)
+      return null
+    }
+
+    return json
+  } catch (error) {
+    Console.debug(`Could not read ${url}`, error)
+    return null
+  }
+}
+
+// Latest published IDE version: CloudFront manifest, then GitHub package.json, then a default.
+export const getLatestEditorVersion = async (): Promise<string> => {
+  const fromCloud = highestVersion(await fetchTagList(IDE_CLOUDFRONT_MANIFEST))
+  if (fromCloud) return fromCloud
+
+  try {
+    const res = await fetch(IDE_GITHUB_PACKAGE, { timeout: FETCH_TIMEOUT_MS })
+    if (res.ok) {
+      const json = await res.json()
+      if (typeof json?.version === "string" && parseVersion(json.version)) {
+        return json.version
+      }
+    }
+  } catch (error) {
+    Console.debug("Could not read the IDE package.json from GitHub", error)
+  }
+
+  return DEFAULT_EDITOR_VERSION
+}