@learnpack/learnpack 5.0.352 → 5.0.354

package/src/utils/repair/legacyPackageRepair.ts

@@ -0,0 +1,731 @@
+import { Lesson, Syllabus, TDifficulty } from "../../models/creator"
+import { Exercise } from "../configBuilder"
+import { sortExerciseFiles } from "../exerciseFileOrder"
+import {
+  LessonType,
+  deriveLessonId,
+  parseReadmeContent,
+} from "../packageManifest"
+import { RepairStorage, RepairTarget } from "./repairStorage"
+
+/**
+ * Reconstruct the manifest inputs missing from legacy packages.
+ *
+ * Some packages predate the creator — a few were uploaded straight from a GitHub
+ * repository — and never got a `.learn/initialSyllabus.json`, a
+ * `.learn/sidebar.json`, or a usable `config.json`. The manifest tolerates the
+ * first two and comes out degraded: every lesson typed `READ`, and descriptions
+ * pinned to `null` forever, because `descriptionsS3Backfill` builds its work
+ * list from the syllabus keys and never even sees a course without one.
+ *
+ * This module writes those files back, derived from what the package *does*
+ * have: its exercise folders and their READMEs. It never invents content — a
+ * title comes from the README's own `# H1`, never from an LLM.
+ *
+ * It runs once per bucket (see `repairStorage.ts` for why the draft is repaired
+ * from the draft rather than copied from the snapshot). Ordering afterwards:
+ *
+ *   1. this script, on both targets                       (structure)
+ *   2. `descriptionsS3Backfill --reproject-manifest --mirror-draft`
+ *   3. `backfill:package-manifest`, unless step 2 projected it
+ *
+ * Safety: with one documented exception (a `config.json` whose `exercises` array
+ * is empty, which is merged rather than replaced) it only ever writes keys that
+ * do not exist. It never touches a file the real pipeline produced.
+ */
+
+export type RepairFile = "config" | "syllabus" | "sidebar";
+
+export const ALL_REPAIR_FILES: RepairFile[] = ["config", "syllabus", "sidebar"]
+
+export type FileOutcome =
+  | "created"
+  | "merged"
+  | "present"
+  | "not-requested"
+  | "impossible";
+
+export type FileResult = {
+  outcome: FileOutcome;
+  key?: string;
+  reason?: string;
+};
+
+export type RepairStats = {
+  exercises: number;
+  languages: string[];
+  primaryLanguage: string;
+  lessonTypes: Record<LessonType, number>;
+  titlesFromReadme: number;
+  titlesFromFolderName: number;
+};
+
+export type RepairResult = {
+  slug: string;
+  target: RepairTarget;
+  /**
+   * `absent` is not a failure: half the legacy packages published in S3 have no
+   * draft in GCS at all, so a run over both buckets is expected to find nothing
+   * to repair on one side. Only a package that *is* in the bucket but yields no
+   * exercises counts as `failed`.
+   */
+  status: "repaired" | "unchanged" | "absent" | "failed";
+  durationMs: number;
+  files: Record<RepairFile, FileResult>;
+  stats?: RepairStats;
+  error?: string;
+};
+
+export type RepairOptions = {
+  dryRun: boolean;
+  files: RepairFile[];
+};
+
+const README_FILENAME_PATTERN = /^readme(?:\.([a-z]{2}))?\.md$/i
+/**
+ * A markdown task list is how LearnPack renders a quiz, so two or more options
+ * in one README is the only signal a legacy package gives us about its type.
+ */
+const QUIZ_OPTION_PATTERN = /^\s*[*-]\s+\[[\sXx]]\s+\S/gm
+const CODE_FILE_EXTENSIONS = new Set([
+  "c",
+  "cpp",
+  "cs",
+  "css",
+  "go",
+  "html",
+  "java",
+  "js",
+  "json",
+  "jsx",
+  "php",
+  "py",
+  "rb",
+  "rs",
+  "scss",
+  "sh",
+  "sql",
+  "ts",
+  "tsx",
+  "vue",
+])
+
+type ConfigJsonShape = {
+  config?: Record<string, unknown>;
+  exercises?: Exercise[];
+};
+
+/** README body + parsed title, by exercise slug and language. */
+export type ReadmeIndex = Record<
+  string,
+  Record<string, { title: string | null; body: string }>
+>;
+
+function humanizeFolderName(folderSlug: string): string {
+  const match = folderSlug.match(/^(\d+(?:\.\d+)?)(?:-(.*))?$/)
+  const titlePart = match && match[2] ? match[2] : folderSlug
+  return titlePart.replace(/-/g, " ").trim() || folderSlug
+}
+
+function naturalCompare(a: string, b: string): number {
+  const regex = /(\d+|\D+)/g
+  const ax = a.match(regex) || []
+  const bx = b.match(regex) || []
+  for (let i = 0; i < Math.max(ax.length, bx.length); i++) {
+    const an = Number.parseInt(ax[i], 10)
+    const bn = Number.parseInt(bx[i], 10)
+    if (!Number.isNaN(an) && !Number.isNaN(bn)) {
+      if (an !== bn) {
+        return an - bn
+      }
+    } else if (ax[i] !== bx[i]) {
+      return (ax[i] || "").localeCompare(bx[i] || "")
+    }
+  }
+
+  return 0
+}
+
+/**
+ * Exercises from the bucket listing alone, mirroring `configBuilder.buildConfig`
+ * key for key: same README language rule, same natural ordering, same
+ * `graded: false` default.
+ */
+export function buildExercisesFromKeys(
+  exercisesPrefix: string,
+  keys: string[]
+): Exercise[] {
+  const map = new Map<string, Omit<Exercise, "position">>()
+
+  for (const key of keys) {
+    if (!key.startsWith(exercisesPrefix)) {
+      continue
+    }
+
+    const parts = key.slice(exercisesPrefix.length).split("/")
+    if (parts.length < 2) {
+      continue
+    }
+
+    const exerciseSlug = parts[0]
+    const filename = parts[parts.length - 1]
+    if (!exerciseSlug || !filename) {
+      continue
+    }
+
+    let exercise = map.get(exerciseSlug)
+    if (!exercise) {
+      exercise = {
+        title: exerciseSlug,
+        slug: exerciseSlug,
+        graded: false,
+        files: [],
+        translations: {},
+      }
+      map.set(exerciseSlug, exercise)
+    }
+
+    const readmeMatch = filename.match(README_FILENAME_PATTERN)
+    // Nested files keep their folder-relative name, as the IDE expects.
+    if (readmeMatch && parts.length === 2) {
+      exercise.translations[readmeMatch[1] || "en"] = filename
+    } else {
+      exercise.files.push({
+        name: parts.slice(1).join("/"),
+        slug: parts.slice(1).join("/"),
+        hidden: false,
+      })
+    }
+  }
+
+  return [...map.values()]
+    .sort((a, b) => naturalCompare(a.slug, b.slug))
+    .map((exercise, index) => ({
+      ...exercise,
+      files: sortExerciseFiles(exercise.files),
+      position: index,
+    }))
+}
+
+/**
+ * Language the course is really written in.
+ *
+ * Same rule as `workList.inferCourseBaseLanguage`: a declared language is
+ * trusted only when some exercise actually has a README in it, otherwise the
+ * most frequent translation wins. Defaulting to "en" would label a Spanish
+ * course English and anchor every later description to a translation.
+ *
+ * A bilingual course ties on count — most of the legacy ones are — so the tie is
+ * broken on the file names rather than arbitrarily: `getReadmeExtension` gives
+ * the *original* README no language suffix and suffixes every translation, so
+ * the language mapped to a bare `README.md` is the one the course was written
+ * in. This matters beyond the title: the descriptions backfill generates from
+ * `courseInfo.language` and would otherwise work from a translation.
+ */
+export function inferPrimaryLanguage(
+  exercises: Exercise[],
+  declared?: string
+): string {
+  const counts = new Map<string, number>()
+  const unsuffixed = new Set<string>()
+  for (const exercise of exercises) {
+    for (const [lang, filename] of Object.entries(
+      exercise.translations || {}
+    )) {
+      counts.set(lang, (counts.get(lang) || 0) + 1)
+      if (/^readme\.md$/i.test(filename)) {
+        unsuffixed.add(lang)
+      }
+    }
+  }
+
+  if (counts.size === 0) {
+    return declared || "en"
+  }
+
+  const normalized = declared?.trim().toLowerCase()
+  if (normalized) {
+    for (const lang of counts.keys()) {
+      if (lang.toLowerCase() === normalized) {
+        return lang
+      }
+    }
+  }
+
+  const ranked = [...counts.entries()].sort((a, b) => {
+    if (a[1] !== b[1]) {
+      return b[1] - a[1]
+    }
+
+    const aBase = unsuffixed.has(a[0]) ? 0 : 1
+    const bBase = unsuffixed.has(b[0]) ? 0 : 1
+    return aBase === bBase ? a[0].localeCompare(b[0]) : aBase - bBase
+  })
+
+  return ranked[0][0]
+}
+
+/**
+ * Lesson type, inferred — the one field of a synthesized syllabus that is a
+ * guess rather than a reading. Callers should treat it as such.
+ */
+export function inferLessonType(input: {
+  graded: boolean;
+  files: Array<{ name: string }>;
+  readmeBody: string;
+}): LessonType {
+  const quizOptions = input.readmeBody.match(QUIZ_OPTION_PATTERN) || []
+  if (quizOptions.length >= 2) {
+    return "QUIZ"
+  }
+
+  if (input.graded) {
+    return "CODE"
+  }
+
+  const hasCodeFile = input.files.some(file => {
+    const extension = file.name.split(".").pop()?.toLowerCase() || ""
+    return CODE_FILE_EXTENSIONS.has(extension)
+  })
+
+  return hasCodeFile ? "CODE" : "READ"
+}
+
+export type BuildSyllabusInput = {
+  slug: string;
+  learnJson: Record<string, unknown> | null;
+  exercises: Exercise[];
+  readmes: ReadmeIndex;
+  primaryLanguage: string;
+};
+
+function pickLocalized(
+  value: unknown,
+  primaryLanguage: string
+): string | undefined {
+  if (typeof value === "string") {
+    return value
+  }
+
+  if (!value || typeof value !== "object") {
+    return undefined
+  }
+
+  const map = value as Record<string, string>
+  const candidate = map[primaryLanguage] || Object.values(map)[0]
+  return typeof candidate === "string" ? candidate : undefined
+}
+
+function buildCourseInfo(input: BuildSyllabusInput): Syllabus["courseInfo"] {
+  const learnJson = input.learnJson || {}
+  const difficulty = learnJson.difficulty
+  const duration = learnJson.duration
+
+  return {
+    title: pickLocalized(learnJson.title, input.primaryLanguage) || input.slug,
+    description:
+      pickLocalized(learnJson.description, input.primaryLanguage) || "",
+    duration: typeof duration === "number" ? duration : 0,
+    difficulty: (typeof difficulty === "string" ?
+      difficulty :
+      "beginner") as TDifficulty,
+    language: input.primaryLanguage,
+    technologies: Array.isArray(learnJson.technologies) ?
+      (learnJson.technologies as string[]) :
+      [],
+    slug: input.slug,
+    // Fields the creator wizard owns. A repaired package was never authored
+    // there, so they carry the "nothing pending" value rather than a guess.
+    hasContentIndex: false,
+    contentIndex: "",
+    isCompleted: true,
+    variables: [],
+    currentStep: "",
+    purpose: "",
+  }
+}
+
+export type BuildSyllabusResult = {
+  syllabus: Syllabus;
+  stats: Omit<RepairStats, "exercises" | "languages" | "primaryLanguage">;
+};
+
+/**
+ * A syllabus equivalent to what the creator would have written, derived from the
+ * exercise folders and their READMEs.
+ *
+ * `uid` is set to the exercise folder slug on purpose. `findSyllabusLesson`
+ * matches on `slugify(id + "-" + title)` *or* `uid`, and only the second is
+ * guaranteed to hold once a title comes from an H1 the folder name never had —
+ * it is also what lets the S3 and GCS repairs, run independently, still line up
+ * for the descriptions mirror.
+ */
+export function buildSyllabusFromSources(
+  input: BuildSyllabusInput
+): BuildSyllabusResult {
+  const now = Date.now()
+  const lessonTypes: Record<LessonType, number> = { READ: 0, CODE: 0, QUIZ: 0 }
+  let titlesFromReadme = 0
+  let titlesFromFolderName = 0
+
+  const lessons: Lesson[] = [...input.exercises]
+    .sort((a, b) => a.position - b.position)
+    .map(exercise => {
+      const readmesByLang = input.readmes[exercise.slug] || {}
+      const primaryReadme = readmesByLang[input.primaryLanguage] ||
+        Object.values(readmesByLang)[0] || { title: null, body: "" }
+
+      const title = primaryReadme.title || humanizeFolderName(exercise.slug)
+      if (primaryReadme.title) {
+        titlesFromReadme += 1
+      } else {
+        titlesFromFolderName += 1
+      }
+
+      const type = inferLessonType({
+        graded: exercise.graded,
+        files: exercise.files,
+        readmeBody: primaryReadme.body,
+      })
+      lessonTypes[type] += 1
+
+      const translations: NonNullable<Lesson["translations"]> = {}
+      for (const lang of Object.keys(exercise.translations || {})) {
+        // Same shape `syllabusSync` repairs a missing slot with. Deliberately
+        // left without a fingerprint or prompt version so that
+        // `workList.needsGeneration` returns true and the descriptions backfill
+        // picks the course up on its next run.
+        translations[lang] = {
+          completionId: 0,
+          startedAt: now,
+          completedAt: now,
+        }
+      }
+
+      return {
+        id: deriveLessonId(exercise.slug),
+        uid: exercise.slug,
+        title,
+        type,
+        // `Lesson.description` is the generation topic, not the step
+        // description shown to students — that one lives per language in
+        // `translations[lang].description`, which the backfill fills in.
+        description: title,
+        duration: 2,
+        generated: true,
+        status: "DONE" as const,
+        translations,
+      }
+    })
+
+  return {
+    syllabus: {
+      lessons,
+      courseInfo: buildCourseInfo(input),
+    },
+    stats: { lessonTypes, titlesFromReadme, titlesFromFolderName },
+  }
+}
+
+/** `sidebar[exerciseSlug][lang] = title`, from the same READMEs. */
+export function buildSidebarFromReadmes(
+  exercises: Exercise[],
+  readmes: ReadmeIndex
+): Record<string, Record<string, string>> {
+  const sidebar: Record<string, Record<string, string>> = {}
+
+  for (const exercise of exercises) {
+    const readmesByLang = readmes[exercise.slug] || {}
+    const entry: Record<string, string> = {}
+
+    for (const lang of Object.keys(exercise.translations || {})) {
+      entry[lang] =
+        readmesByLang[lang]?.title || humanizeFolderName(exercise.slug)
+    }
+
+    if (Object.keys(entry).length > 0) {
+      sidebar[exercise.slug] = entry
+    }
+  }
+
+  return sidebar
+}
+
+async function readReadmes(
+  storage: RepairStorage,
+  slug: string,
+  exercises: Exercise[]
+): Promise<ReadmeIndex> {
+  const index: ReadmeIndex = {}
+  const prefix = storage.layout.exercisesPrefix(slug)
+
+  await Promise.all(
+    exercises.flatMap(exercise =>
+      Object.entries(exercise.translations || {}).map(
+        async ([lang, filename]) => {
+          const content = await storage.readText(
+            `${prefix}${exercise.slug}/${filename}`
+          )
+          if (content === null) {
+            return
+          }
+
+          const { title } = parseReadmeContent(content)
+          if (!index[exercise.slug]) {
+            index[exercise.slug] = {}
+          }
+
+          index[exercise.slug][lang] = { title: title || null, body: content }
+        }
+      )
+    )
+  )
+
+  return index
+}
+
+/** First key that resolves, with the parsed value. */
+async function readFirst<T>(
+  storage: RepairStorage,
+  keys: string[]
+): Promise<{ value: T; key: string } | null> {
+  for (const key of keys) {
+    // eslint-disable-next-line no-await-in-loop -- later keys are fallbacks
+    const value = await storage.readJson<T>(key)
+    if (value) {
+      return { value, key }
+    }
+  }
+
+  return null
+}
+
+function initialFiles(files: RepairFile[]): Record<RepairFile, FileResult> {
+  const requested = new Set(files)
+  const base: FileOutcome = "not-requested"
+  return {
+    config: { outcome: requested.has("config") ? "present" : base },
+    syllabus: { outcome: requested.has("syllabus") ? "present" : base },
+    sidebar: { outcome: requested.has("sidebar") ? "present" : base },
+  }
+}
+
+/**
+ * Repair one package in one bucket. Returns what it did (or would do, under
+ * `dryRun`) without throwing: a batch reports failures per slug.
+ */
+export async function repairPackage(
+  storage: RepairStorage,
+  slug: string,
+  options: RepairOptions
+): Promise<RepairResult> {
+  const started = Date.now()
+  const { layout } = storage
+  const wants = new Set(options.files)
+  const files = initialFiles(options.files)
+
+  const result = (
+    status: RepairResult["status"],
+    extra: Partial<RepairResult> = {}
+  ): RepairResult => ({
+    slug,
+    target: storage.target,
+    status,
+    durationMs: Date.now() - started,
+    files,
+    ...extra,
+  })
+
+  try {
+    const [learnJson, existingConfig, existingSyllabus, existingSidebar] =
+      await Promise.all([
+        storage.readJson<Record<string, unknown>>(layout.learnJsonKey(slug)),
+        readFirst<ConfigJsonShape>(storage, layout.configKeys(slug)),
+        storage.readJson<Syllabus>(layout.syllabusKey(slug)),
+        readFirst<Record<string, Record<string, string>>>(
+          storage,
+          layout.sidebarKeys(slug)
+        ),
+      ])
+
+    const configuredExercises = Array.isArray(existingConfig?.value.exercises) ?
+      (existingConfig!.value.exercises as Exercise[]) :
+      []
+
+    const needsConfig = wants.has("config") && configuredExercises.length === 0
+    const needsSyllabus = wants.has("syllabus") && !existingSyllabus
+    const needsSidebar = wants.has("sidebar") && !existingSidebar
+
+    if (wants.has("config") && !needsConfig) {
+      files.config = { outcome: "present", key: existingConfig!.key }
+    }
+
+    if (wants.has("syllabus") && !needsSyllabus) {
+      files.syllabus = { outcome: "present", key: layout.syllabusKey(slug) }
+    }
+
+    if (wants.has("sidebar") && !needsSidebar) {
+      files.sidebar = { outcome: "present", key: existingSidebar!.key }
+    }
+
+    // Nothing to build: stop before the listing and the READMEs, which is what
+    // makes a catalogue-wide pass over healthy packages cheap.
+    if (!needsConfig && !needsSyllabus && !needsSidebar) {
+      return result("unchanged")
+    }
+
+    const useConfigExercises =
+      layout.exercisesFromConfig && configuredExercises.length > 0
+
+    const derivedExercises = useConfigExercises ?
+      [] :
+      buildExercisesFromKeys(
+          layout.exercisesPrefix(slug),
+          await storage.listKeys(layout.exercisesPrefix(slug))
+        )
+
+    // Even where the listing is the authority on the exercise list, `graded` is
+    // worth keeping: the listing cannot express it (`buildConfig` hardcodes
+    // false) and it is a signal for typing a lesson CODE.
+    const gradedBySlug = new Map(
+      configuredExercises.map(exercise => [exercise.slug, exercise.graded])
+    )
+
+    const exercises = useConfigExercises ?
+      configuredExercises :
+      derivedExercises.map(exercise => ({
+          ...exercise,
+          graded: gradedBySlug.get(exercise.slug) ?? exercise.graded,
+        }))
+
+    if (exercises.length === 0) {
+      const anythingAtAll = Boolean(
+        learnJson || existingConfig || existingSyllabus || existingSidebar
+      )
+
+      if (anythingAtAll) {
+        return result("failed", {
+          error: `No exercises found for "${slug}" in ${storage.target}: neither its config.json nor its exercises/ listing yields any`,
+        })
+      }
+
+      for (const file of options.files) {
+        files[file] = {
+          outcome: "impossible",
+          reason: `the package is not in ${storage.target}`,
+        }
+      }
+
+      return result("absent")
+    }
+
+    const readmes = await readReadmes(storage, slug, exercises)
+    const primaryLanguage = inferPrimaryLanguage(
+      exercises,
+      (learnJson?.language as string | undefined) ||
+        existingSyllabus?.courseInfo?.language
+    )
+
+    let wrote = false
+
+    // --- config.json ----------------------------------------------------
+    if (needsConfig) {
+      // `config` is not decoration: the publish route destructures
+      // `{ config, exercises }` and reads `config.title` right after, so a
+      // config carrying only exercises would break the next publication.
+      const configField = existingConfig?.value.config || learnJson
+
+      if (configField) {
+        const key = existingConfig?.key || layout.configKeys(slug)[0]
+        const merged: ConfigJsonShape = {
+          ...existingConfig?.value,
+          config: configField,
+          exercises: derivedExercises,
+        }
+
+        files.config = {
+          outcome: existingConfig ? "merged" : "created",
+          key,
+          reason: existingConfig ?
+            "existing config.json had no exercises; other fields preserved" :
+            "no config.json in the package",
+        }
+        if (!options.dryRun) {
+          await storage.writeJson(key, merged)
+        }
+
+        wrote = true
+      } else {
+        files.config = {
+          outcome: "impossible",
+          reason: "no learn.json and no config.config to build it from",
+        }
+      }
+    }
+
+    // --- initialSyllabus.json -------------------------------------------
+    let syllabusStats: BuildSyllabusResult["stats"] | undefined
+
+    if (needsSyllabus) {
+      const built = buildSyllabusFromSources({
+        slug,
+        learnJson,
+        exercises,
+        readmes,
+        primaryLanguage,
+      })
+      syllabusStats = built.stats
+
+      files.syllabus = { outcome: "created", key: layout.syllabusKey(slug) }
+      if (!options.dryRun) {
+        await storage.writeJson(layout.syllabusKey(slug), built.syllabus)
+      }
+
+      wrote = true
+    }
+
+    // --- sidebar.json ----------------------------------------------------
+    if (needsSidebar) {
+      const sidebar = buildSidebarFromReadmes(exercises, readmes)
+      if (Object.keys(sidebar).length === 0) {
+        files.sidebar = {
+          outcome: "impossible",
+          reason: "no README titles to build it from",
+        }
+      } else {
+        const key = layout.sidebarKeys(slug)[0]
+        files.sidebar = { outcome: "created", key }
+        if (!options.dryRun) {
+          await storage.writeJson(key, sidebar)
+        }
+
+        wrote = true
+      }
+    }
+
+    const languages = [
+      ...new Set(
+        exercises.flatMap(exercise =>
+          Object.keys(exercise.translations || {})
+        )
+      ),
+    ].sort()
+
+    return result(wrote ? "repaired" : "unchanged", {
+      stats: {
+        exercises: exercises.length,
+        languages,
+        primaryLanguage,
+        lessonTypes: syllabusStats?.lessonTypes || {
+          READ: 0,
+          CODE: 0,
+          QUIZ: 0,
+        },
+        titlesFromReadme: syllabusStats?.titlesFromReadme || 0,
+        titlesFromFolderName: syllabusStats?.titlesFromFolderName || 0,
+      },
+    })
+  } catch (error) {
+    return result("failed", { error: (error as Error).message })
+  }
+}