@learnpack/learnpack 5.0.351 → 5.0.353

package/src/utils/syllabusSync.ts

@@ -0,0 +1,390 @@
+import { Bucket } from "@google-cloud/storage"
+import { Lesson, Syllabus } from "../models/creator"
+import { buildConfig } from "./configBuilder"
+import { slugify } from "./creatorUtilities"
+
+/**
+ * Provisional syllabus ↔ bucket reconciliation.
+ *
+ * The `initialSyllabus.json` occasionally drifts from what actually lives in the
+ * bucket (lessons/translations present as files but missing from the syllabus,
+ * or vice versa). Because step descriptions are stored per (step, lang) in the
+ * syllabus, that drift leaves missing slots and causes skipped/spurious
+ * generations. This reconciles the two.
+ *
+ * Extracted from the inline `/actions/synchronize-syllabus` handler so it can be
+ * reused by the sweep/backfill, and generalized over a minimal storage adapter
+ * so it works for both GCS and S3.
+ *
+ * NOTE: this is a temporary patch until the root cause of the drift is fixed.
+ * Track its removal via the `DESCRIPTIONS_RECONCILE_SYLLABUS` flag on callers.
+ */
+
+export interface SyllabusSyncStorage {
+  /**
+   * Map of `folderSlug -> fileCount` for every folder directly under
+   * `exercises/`. A single authoritative listing: if it throws, callers must
+   * abort any destructive action (never delete on an errored/unknown listing).
+   */
+  listExerciseFolderFileCounts(
+    courseSlug: string
+  ): Promise<Map<string, number>>;
+  /** README language codes (lowercased) present per exercise folder slug. */
+  translationLangsBySlug(courseSlug: string): Promise<Map<string, string[]>>;
+}
+
+export type SyllabusSyncOptions = {
+  /**
+   * When false (default, "additive" mode) only non-destructive passes run:
+   * add-missing, repair-translations, unstick-status. Removal/dedupe are
+   * skipped. When true (manual dev button / root-cause runs) removals apply.
+   */
+  prune?: boolean;
+};
+
+export type SyllabusSyncCounters = {
+  totalLessons: number;
+  keptLessons: number;
+  removedLessons: number;
+  duplicatesResolved: number;
+  addedLessons: number;
+  fixedLessons: number;
+  repairedTranslationsInLessons: number;
+  repairedTranslationEntries: number;
+};
+
+type LessonRef = { id: string; title: string; slug: string };
+type LessonRefWithCount = LessonRef & { fileCount: number };
+
+export type SyllabusSyncResult = {
+  syllabus: Syllabus;
+  changed: boolean;
+  counters: SyllabusSyncCounters;
+  details: {
+    removed: LessonRef[];
+    duplicates: LessonRefWithCount[];
+    kept: Array<LessonRef & { fileCount?: number }>;
+    added: LessonRef[];
+  };
+};
+
+function isInFlight(lesson: Lesson): boolean {
+  // A lesson still being generated may legitimately have no files yet; never
+  // treat it as "missing" (E4 hardening against deleting something real).
+  return (
+    lesson.generated === false ||
+    lesson.status === "GENERATING" ||
+    lesson.status === "PENDING"
+  )
+}
+
+export async function synchronizeSyllabusWithBucket(
+  storage: SyllabusSyncStorage,
+  courseSlug: string,
+  syllabus: Syllabus,
+  options: SyllabusSyncOptions = {}
+): Promise<SyllabusSyncResult> {
+  const prune = options.prune ?? false
+
+  const removed: LessonRef[] = []
+  const duplicates: LessonRefWithCount[] = []
+  const kept: Array<LessonRef & { fileCount?: number }> = []
+  const added: LessonRef[] = []
+  let repairedTranslationsInLessons = 0
+  let repairedTranslationEntries = 0
+  let fixedLessons = 0
+  let changed = false
+
+  // Single authoritative listing. If it fails it throws (callers must not
+  // delete on an unknown listing). The adapter is expected to retry internally.
+  const folderCounts = await storage.listExerciseFolderFileCounts(courseSlug)
+
+  // First pass: resolve existence per lesson, confirming absence under both the
+  // slugified id-title and the uid folder-name variants.
+  const existingLessons: Array<{
+    lesson: Lesson;
+    slug: string;
+    fileCount: number;
+  }> = []
+
+  for (const lesson of syllabus.lessons) {
+    const lessonSlug = slugify(lesson.id + "-" + lesson.title)
+    let resolvedSlug = lessonSlug
+    let fileCount = folderCounts.get(lessonSlug) ?? 0
+
+    if (
+      fileCount === 0 &&
+      lesson.uid &&
+      (folderCounts.get(lesson.uid) ?? 0) > 0
+    ) {
+      resolvedSlug = lesson.uid
+      fileCount = folderCounts.get(lesson.uid) ?? 0
+    }
+
+    if (fileCount === 0) {
+      if (isInFlight(lesson)) {
+        // Keep untouched: do not mark for removal nor as existing.
+        continue
+      }
+
+      removed.push({ id: lesson.id, title: lesson.title, slug: lessonSlug })
+    } else {
+      existingLessons.push({ lesson, slug: resolvedSlug, fileCount })
+    }
+  }
+
+  // Second pass: detect duplicates (multiple lessons resolving to one folder).
+  const slugMap = new Map<
+    string,
+    Array<{ lesson: Lesson; slug: string; fileCount: number }>
+  >()
+  for (const item of existingLessons) {
+    if (!slugMap.has(item.slug)) {
+      slugMap.set(item.slug, [])
+    }
+
+    slugMap.get(item.slug)!.push(item)
+  }
+
+  for (const items of slugMap.values()) {
+    if (items.length > 1) {
+      items.sort((a, b) => b.fileCount - a.fileCount)
+      const winner = items[0]
+      kept.push({
+        id: winner.lesson.id,
+        title: winner.lesson.title,
+        slug: winner.slug,
+        fileCount: winner.fileCount,
+      })
+      for (const loser of items.slice(1)) {
+        duplicates.push({
+          id: loser.lesson.id,
+          title: loser.lesson.title,
+          slug: loser.slug,
+          fileCount: loser.fileCount,
+        })
+      }
+    } else {
+      kept.push({
+        id: items[0].lesson.id,
+        title: items[0].lesson.title,
+        slug: items[0].slug,
+        fileCount: items[0].fileCount,
+      })
+    }
+  }
+
+  // Apply removals only in prune mode (additive mode never deletes).
+  if (prune && (removed.length > 0 || duplicates.length > 0)) {
+    syllabus.lessons = syllabus.lessons.filter(lesson => {
+      const lessonSlug = slugify(lesson.id + "-" + lesson.title)
+      const isRemoved = removed.some(r => r.slug === lessonSlug)
+      const isDuplicate = duplicates.some(
+        d => d.id === lesson.id && d.title === lesson.title
+      )
+      return !isRemoved && !isDuplicate
+    })
+    changed = true
+  }
+
+  // Third pass: add lessons that exist in the bucket but are missing from the
+  // syllabus, then re-sort by numeric id.
+  const existingSlugs = new Set(existingLessons.map(e => e.slug))
+  for (const [folderSlug, count] of folderCounts) {
+    if (count <= 0 || existingSlugs.has(folderSlug)) {
+      continue
+    }
+
+    const idMatch = folderSlug.match(/^(\d+(?:\.\d+)?)(?:-(.*))?$/)
+    const id = idMatch ? idMatch[1] : folderSlug
+    const titlePart = idMatch && idMatch[2] ? idMatch[2] : folderSlug
+    const title = titlePart.replace(/-/g, " ").trim() || folderSlug
+    const newLesson: Lesson = {
+      id,
+      uid: folderSlug,
+      title,
+      type: "READ",
+      description: title,
+      duration: 2,
+      generated: true,
+      status: "DONE",
+    }
+    syllabus.lessons.push(newLesson)
+    added.push({ id, title, slug: folderSlug })
+    changed = true
+  }
+
+  if (added.length > 0) {
+    syllabus.lessons.sort((a, b) => {
+      const na = Number.parseFloat(a.id)
+      const nb = Number.parseFloat(b.id)
+      if (!Number.isNaN(na) && !Number.isNaN(nb)) {
+        return na - nb
+      }
+
+      return String(a.id).localeCompare(String(b.id))
+    })
+  }
+
+  // Fourth + fifth pass: reconcile translations metadata from the actual README
+  // files, and unstick lessons stuck in GENERATING/ERROR whose primary-language
+  // README already exists.
+  try {
+    const translationsBySlug = await storage.translationLangsBySlug(courseSlug)
+
+    for (const lesson of syllabus.lessons) {
+      const candidateSlugs = [
+        slugify(lesson.id + "-" + lesson.title),
+        lesson.uid,
+      ].filter(Boolean) as string[]
+      const matchedSlug = candidateSlugs.find(s => translationsBySlug.has(s))
+      if (!matchedSlug) {
+        continue
+      }
+
+      const languageCodes = translationsBySlug.get(matchedSlug) || []
+      if (languageCodes.length === 0) {
+        continue
+      }
+
+      const currentTranslations = lesson.translations || {}
+      let lessonChanged = false
+
+      for (const lang of languageCodes) {
+        const now = Date.now()
+        if (!currentTranslations[lang]) {
+          currentTranslations[lang] = {
+            completionId: 0,
+            startedAt: now,
+            completedAt: now,
+          }
+          repairedTranslationEntries += 1
+          lessonChanged = true
+        } else {
+          if (!currentTranslations[lang].startedAt) {
+            currentTranslations[lang].startedAt = now
+            lessonChanged = true
+          }
+
+          if (!currentTranslations[lang].completedAt) {
+            currentTranslations[lang].completedAt = now
+            repairedTranslationEntries += 1
+            lessonChanged = true
+          }
+        }
+      }
+
+      if (lessonChanged) {
+        lesson.translations = currentTranslations
+        repairedTranslationsInLessons += 1
+        changed = true
+      }
+    }
+
+    const primaryLanguage = syllabus.courseInfo.language || "en"
+    for (const lesson of syllabus.lessons) {
+      if (lesson.generated !== false) {
+        continue
+      }
+
+      if (lesson.status !== "GENERATING" && lesson.status !== "ERROR") {
+        continue
+      }
+
+      const candidateSlugs = [
+        slugify(lesson.id + "-" + lesson.title),
+        lesson.uid,
+      ].filter(Boolean) as string[]
+      const matchedSlug = candidateSlugs.find(s => translationsBySlug.has(s))
+      if (!matchedSlug) {
+        continue
+      }
+
+      const langs = translationsBySlug.get(matchedSlug) || []
+      if (!langs.includes(primaryLanguage)) {
+        continue
+      }
+
+      lesson.generated = true
+      lesson.status = "DONE"
+      fixedLessons += 1
+      changed = true
+    }
+  } catch (error) {
+    console.error(
+      "⚠️ Could not reconcile lesson translations during syllabus sync:",
+      error
+    )
+  }
+
+  const counters: SyllabusSyncCounters = {
+    totalLessons: syllabus.lessons.length,
+    keptLessons: kept.length,
+    removedLessons: prune ? removed.length : 0,
+    duplicatesResolved: prune ? duplicates.length : 0,
+    addedLessons: added.length,
+    fixedLessons,
+    repairedTranslationsInLessons,
+    repairedTranslationEntries,
+  }
+
+  return {
+    syllabus,
+    changed,
+    counters,
+    details: { removed, duplicates, kept, added },
+  }
+}
+
+async function withRetry<T>(fn: () => Promise<T>, attempts = 3): Promise<T> {
+  let lastError: unknown
+  for (let attempt = 0; attempt < attempts; attempt++) {
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      return await fn()
+    } catch (error) {
+      lastError = error
+    }
+  }
+
+  throw lastError
+}
+
+/** GCS-backed storage adapter (uses the bucket listing + buildConfig). */
+export function createGcsSyllabusSyncStorage(
+  bucket: Bucket
+): SyllabusSyncStorage {
+  return {
+    async listExerciseFolderFileCounts(courseSlug) {
+      const prefix = `courses/${courseSlug}/exercises/`
+      const [files] = await withRetry(() => bucket.getFiles({ prefix }))
+      const counts = new Map<string, number>()
+      for (const file of files) {
+        if (!file.name.startsWith(prefix)) {
+          continue
+        }
+
+        const segment = file.name.slice(prefix.length).split("/")[0]
+        if (segment) {
+          counts.set(segment, (counts.get(segment) || 0) + 1)
+        }
+      }
+
+      return counts
+    },
+    async translationLangsBySlug(courseSlug) {
+      const { exercises } = await buildConfig(bucket, courseSlug)
+      const map = new Map<string, string[]>()
+      for (const exercise of exercises) {
+        const langs = Object.keys(exercise.translations || {})
+          .map(lang => lang.toLowerCase())
+          .filter(Boolean)
+        if (langs.length > 0) {
+          map.set(exercise.slug, [...new Set(langs)])
+        }
+      }
+
+      return map
+    },
+  }
+}