@learnpack/learnpack 5.0.352 → 5.0.354

package/src/utils/s3/packageSourcesAudit.ts

@@ -0,0 +1,311 @@
+import { ListObjectsV2Command } from "@aws-sdk/client-s3"
+import { AwsClient, withRetry } from "./packageManifestBackfill"
+
+/**
+ * Inventory of the files a published package needs for its
+ * `package-manifest.json` to be generated at full fidelity.
+ *
+ * This reads **key names only** — one paginated `ListObjectsV2` over the bucket,
+ * zero `GetObject`. Cheap enough to run over the whole catalogue on demand, and
+ * safe at any time: it never writes.
+ *
+ * What it answers: which packages would *fail* the manifest backfill (no
+ * `config.json`, no exercises), and which ones only produce a *degraded*
+ * manifest (no `initialSyllabus.json` → every lesson typed `READ`, ids derived
+ * from the folder name, and descriptions permanently `null`, because the
+ * descriptions pipeline builds its work list from the syllabus keys and cannot
+ * see a course that has none).
+ *
+ * Companion of `legacyPackageRepair.ts`, which writes what this reports missing.
+ */
+
+export type PackageSourcesAuditEntry = {
+  slug: string;
+  /** A published package is a prefix holding an `index.html` at its root. */
+  hasIndexHtml: boolean;
+  /** Required: without it the manifest backfill reports `failed`. */
+  hasConfigJson: boolean;
+  configJsonKey: string | null;
+  /**
+   * Optional *as a key*: `config.json` may carry the same metadata under
+   * `config`. Only a `GetObject` could tell, so `false` here means "resolves
+   * through the fallback, or not at all" — not "broken".
+   */
+  hasLearnJson: boolean;
+  hasSyllabus: boolean;
+  hasSidebar: boolean;
+  sidebarKey: string | null;
+  hasManifest: boolean;
+  /** Folders directly under `exercises/` holding at least one object. */
+  exerciseFolders: number;
+  /** `README.md` / `README.xx.md` files found in those folders. */
+  readmeCount: number;
+  /** Missing inputs, named as in the docs, for quick reporting. */
+  missing: string[];
+  /** True when the manifest backfill cannot produce a manifest at all. */
+  blocking: boolean;
+};
+
+export type PackageSourcesAuditSummary = {
+  missingConfigJson: number;
+  missingLearnJson: number;
+  missingSyllabus: number;
+  missingSidebar: number;
+  missingManifest: number;
+  noExerciseFolders: number;
+  blocking: number;
+};
+
+export type PackageSourcesAuditReport = {
+  generatedAt: string;
+  bucket: string;
+  scannedKeys: number;
+  totalPackages: number;
+  summary: PackageSourcesAuditSummary;
+  packages: PackageSourcesAuditEntry[];
+};
+
+export type AuditOptions = {
+  /** Restrict the scan to a single package (uses a prefixed listing). */
+  slug?: string;
+  /** Keep prefixes with no `index.html` (drafts, leftovers). Default false. */
+  includeUnpublished?: boolean;
+};
+
+const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i
+
+const ROOT_FILES = new Set([
+  "index.html",
+  "learn.json",
+  "config.json",
+  ".learn/config.json",
+  ".learn/initialSyllabus.json",
+  ".learn/sidebar.json",
+  "sidebar.json",
+  ".learn/package-manifest.json",
+])
+
+type PackageAccumulator = {
+  slug: string;
+  hasIndexHtml: boolean;
+  configJsonKey: string | null;
+  hasLearnJson: boolean;
+  hasSyllabus: boolean;
+  sidebarKey: string | null;
+  hasManifest: boolean;
+  exerciseFolders: Set<string>;
+  readmeCount: number;
+};
+
+function emptyAccumulator(slug: string): PackageAccumulator {
+  return {
+    slug,
+    hasIndexHtml: false,
+    configJsonKey: null,
+    hasLearnJson: false,
+    hasSyllabus: false,
+    sidebarKey: null,
+    hasManifest: false,
+    exerciseFolders: new Set<string>(),
+    readmeCount: 0,
+  }
+}
+
+/** Fold one object key into the accumulator map. */
+function classifyKey(
+  entries: Map<string, PackageAccumulator>,
+  key: string
+): void {
+  const separatorIndex = key.indexOf("/")
+  if (separatorIndex <= 0) {
+    // Object at the bucket root: not part of any package.
+    return
+  }
+
+  const slug = key.slice(0, separatorIndex)
+  const rest = key.slice(separatorIndex + 1)
+  if (!rest) {
+    return
+  }
+
+  const exerciseParts = rest.startsWith("exercises/") ? rest.split("/") : []
+  // `exercises/<slug>/<file>`: shorter paths are the folder marker itself.
+  const exerciseSlug =
+    exerciseParts.length >= 3 && exerciseParts[1] ? exerciseParts[1] : null
+
+  if (!ROOT_FILES.has(rest) && !exerciseSlug) {
+    return
+  }
+
+  let entry = entries.get(slug)
+  if (!entry) {
+    entry = emptyAccumulator(slug)
+    entries.set(slug, entry)
+  }
+
+  if (exerciseSlug) {
+    entry.exerciseFolders.add(exerciseSlug)
+    const filename = exerciseParts[exerciseParts.length - 1]
+    if (exerciseParts.length === 3 && README_FILENAME_PATTERN.test(filename)) {
+      entry.readmeCount += 1
+    }
+
+    return
+  }
+
+  switch (rest) {
+    case "index.html":
+      entry.hasIndexHtml = true
+      break
+    case "learn.json":
+      entry.hasLearnJson = true
+      break
+    // The root copy wins over `.learn/config.json`, matching the order
+    // `fetchPackageSources` resolves them in. Same for the sidebar, inverted.
+    case "config.json":
+      entry.configJsonKey = key
+      break
+    case ".learn/config.json":
+      entry.configJsonKey = entry.configJsonKey || key
+      break
+    case ".learn/sidebar.json":
+      entry.sidebarKey = key
+      break
+    case "sidebar.json":
+      entry.sidebarKey = entry.sidebarKey || key
+      break
+    case ".learn/initialSyllabus.json":
+      entry.hasSyllabus = true
+      break
+    case ".learn/package-manifest.json":
+      entry.hasManifest = true
+      break
+    default:
+      break
+  }
+}
+
+function finalizeEntry(entry: PackageAccumulator): PackageSourcesAuditEntry {
+  const missing: string[] = []
+
+  if (!entry.configJsonKey) {
+    missing.push("config.json")
+  }
+
+  if (!entry.hasLearnJson) {
+    missing.push("learn.json")
+  }
+
+  if (!entry.hasSyllabus) {
+    missing.push("initialSyllabus.json")
+  }
+
+  if (!entry.sidebarKey) {
+    missing.push("sidebar.json")
+  }
+
+  if (entry.exerciseFolders.size === 0) {
+    missing.push("exercises/")
+  }
+
+  if (!entry.hasManifest) {
+    missing.push("package-manifest.json")
+  }
+
+  return {
+    slug: entry.slug,
+    hasIndexHtml: entry.hasIndexHtml,
+    hasConfigJson: Boolean(entry.configJsonKey),
+    configJsonKey: entry.configJsonKey,
+    hasLearnJson: entry.hasLearnJson,
+    hasSyllabus: entry.hasSyllabus,
+    hasSidebar: Boolean(entry.sidebarKey),
+    sidebarKey: entry.sidebarKey,
+    hasManifest: entry.hasManifest,
+    exerciseFolders: entry.exerciseFolders.size,
+    readmeCount: entry.readmeCount,
+    missing,
+    // A missing `learn.json` is not blocking on its own: `config.config` can
+    // stand in for it, and only a download would tell.
+    blocking: !entry.configJsonKey || entry.exerciseFolders.size === 0,
+  }
+}
+
+function count(
+  packages: PackageSourcesAuditEntry[],
+  predicate: (entry: PackageSourcesAuditEntry) => boolean
+): number {
+  return packages.filter(entry => predicate(entry)).length
+}
+
+export function summarize(
+  packages: PackageSourcesAuditEntry[]
+): PackageSourcesAuditSummary {
+  return {
+    missingConfigJson: count(packages, entry => !entry.hasConfigJson),
+    missingLearnJson: count(packages, entry => !entry.hasLearnJson),
+    missingSyllabus: count(packages, entry => !entry.hasSyllabus),
+    missingSidebar: count(packages, entry => !entry.hasSidebar),
+    missingManifest: count(packages, entry => !entry.hasManifest),
+    noExerciseFolders: count(packages, entry => entry.exerciseFolders === 0),
+    blocking: count(packages, entry => entry.blocking),
+  }
+}
+
+/**
+ * One paginated pass over the bucket, folding each page into the accumulator.
+ *
+ * Deliberately not built on `listObjectKeys`: that one materializes every key in
+ * the catalogue before the caller sees the first one.
+ */
+export async function auditPackageSources(
+  s3: AwsClient,
+  bucket: string,
+  options: AuditOptions = {}
+): Promise<PackageSourcesAuditReport> {
+  const entries = new Map<string, PackageAccumulator>()
+  const prefix = options.slug ? `${options.slug}/` : undefined
+  let continuationToken: string | undefined
+  let scannedKeys = 0
+
+  do {
+    // eslint-disable-next-line no-await-in-loop -- S3 pagination is sequential
+    const response = (await withRetry(() =>
+      s3.send(
+        new ListObjectsV2Command({
+          Bucket: bucket,
+          Prefix: prefix,
+          ContinuationToken: continuationToken,
+        })
+      )
+    )) as {
+      Contents?: Array<{ Key?: string }>;
+      NextContinuationToken?: string;
+    }
+
+    for (const item of response.Contents || []) {
+      if (!item.Key) {
+        continue
+      }
+
+      scannedKeys += 1
+      classifyKey(entries, item.Key)
+    }
+
+    continuationToken = response.NextContinuationToken
+  } while (continuationToken)
+
+  const packages = [...entries.values()]
+    .filter(entry => options.includeUnpublished || entry.hasIndexHtml)
+    .map(entry => finalizeEntry(entry))
+    .sort((a, b) => a.slug.localeCompare(b.slug))
+
+  return {
+    generatedAt: new Date().toISOString(),
+    bucket,
+    scannedKeys,
+    totalPackages: packages.length,
+    summary: summarize(packages),
+    packages,
+  }
+}

package/src/utils/syllabusSync.ts

@@ -0,0 +1,390 @@
+import { Bucket } from "@google-cloud/storage"
+import { Lesson, Syllabus } from "../models/creator"
+import { buildConfig } from "./configBuilder"
+import { slugify } from "./creatorUtilities"
+
+/**
+ * Provisional syllabus ↔ bucket reconciliation.
+ *
+ * The `initialSyllabus.json` occasionally drifts from what actually lives in the
+ * bucket (lessons/translations present as files but missing from the syllabus,
+ * or vice versa). Because step descriptions are stored per (step, lang) in the
+ * syllabus, that drift leaves missing slots and causes skipped/spurious
+ * generations. This reconciles the two.
+ *
+ * Extracted from the inline `/actions/synchronize-syllabus` handler so it can be
+ * reused by the sweep/backfill, and generalized over a minimal storage adapter
+ * so it works for both GCS and S3.
+ *
+ * NOTE: this is a temporary patch until the root cause of the drift is fixed.
+ * Track its removal via the `DESCRIPTIONS_RECONCILE_SYLLABUS` flag on callers.
+ */
+
+export interface SyllabusSyncStorage {
+  /**
+   * Map of `folderSlug -> fileCount` for every folder directly under
+   * `exercises/`. A single authoritative listing: if it throws, callers must
+   * abort any destructive action (never delete on an errored/unknown listing).
+   */
+  listExerciseFolderFileCounts(
+    courseSlug: string
+  ): Promise<Map<string, number>>;
+  /** README language codes (lowercased) present per exercise folder slug. */
+  translationLangsBySlug(courseSlug: string): Promise<Map<string, string[]>>;
+}
+
+export type SyllabusSyncOptions = {
+  /**
+   * When false (default, "additive" mode) only non-destructive passes run:
+   * add-missing, repair-translations, unstick-status. Removal/dedupe are
+   * skipped. When true (manual dev button / root-cause runs) removals apply.
+   */
+  prune?: boolean;
+};
+
+export type SyllabusSyncCounters = {
+  totalLessons: number;
+  keptLessons: number;
+  removedLessons: number;
+  duplicatesResolved: number;
+  addedLessons: number;
+  fixedLessons: number;
+  repairedTranslationsInLessons: number;
+  repairedTranslationEntries: number;
+};
+
+type LessonRef = { id: string; title: string; slug: string };
+type LessonRefWithCount = LessonRef & { fileCount: number };
+
+export type SyllabusSyncResult = {
+  syllabus: Syllabus;
+  changed: boolean;
+  counters: SyllabusSyncCounters;
+  details: {
+    removed: LessonRef[];
+    duplicates: LessonRefWithCount[];
+    kept: Array<LessonRef & { fileCount?: number }>;
+    added: LessonRef[];
+  };
+};
+
+function isInFlight(lesson: Lesson): boolean {
+  // A lesson still being generated may legitimately have no files yet; never
+  // treat it as "missing" (E4 hardening against deleting something real).
+  return (
+    lesson.generated === false ||
+    lesson.status === "GENERATING" ||
+    lesson.status === "PENDING"
+  )
+}
+
+export async function synchronizeSyllabusWithBucket(
+  storage: SyllabusSyncStorage,
+  courseSlug: string,
+  syllabus: Syllabus,
+  options: SyllabusSyncOptions = {}
+): Promise<SyllabusSyncResult> {
+  const prune = options.prune ?? false
+
+  const removed: LessonRef[] = []
+  const duplicates: LessonRefWithCount[] = []
+  const kept: Array<LessonRef & { fileCount?: number }> = []
+  const added: LessonRef[] = []
+  let repairedTranslationsInLessons = 0
+  let repairedTranslationEntries = 0
+  let fixedLessons = 0
+  let changed = false
+
+  // Single authoritative listing. If it fails it throws (callers must not
+  // delete on an unknown listing). The adapter is expected to retry internally.
+  const folderCounts = await storage.listExerciseFolderFileCounts(courseSlug)
+
+  // First pass: resolve existence per lesson, confirming absence under both the
+  // slugified id-title and the uid folder-name variants.
+  const existingLessons: Array<{
+    lesson: Lesson;
+    slug: string;
+    fileCount: number;
+  }> = []
+
+  for (const lesson of syllabus.lessons) {
+    const lessonSlug = slugify(lesson.id + "-" + lesson.title)
+    let resolvedSlug = lessonSlug
+    let fileCount = folderCounts.get(lessonSlug) ?? 0
+
+    if (
+      fileCount === 0 &&
+      lesson.uid &&
+      (folderCounts.get(lesson.uid) ?? 0) > 0
+    ) {
+      resolvedSlug = lesson.uid
+      fileCount = folderCounts.get(lesson.uid) ?? 0
+    }
+
+    if (fileCount === 0) {
+      if (isInFlight(lesson)) {
+        // Keep untouched: do not mark for removal nor as existing.
+        continue
+      }
+
+      removed.push({ id: lesson.id, title: lesson.title, slug: lessonSlug })
+    } else {
+      existingLessons.push({ lesson, slug: resolvedSlug, fileCount })
+    }
+  }
+
+  // Second pass: detect duplicates (multiple lessons resolving to one folder).
+  const slugMap = new Map<
+    string,
+    Array<{ lesson: Lesson; slug: string; fileCount: number }>
+  >()
+  for (const item of existingLessons) {
+    if (!slugMap.has(item.slug)) {
+      slugMap.set(item.slug, [])
+    }
+
+    slugMap.get(item.slug)!.push(item)
+  }
+
+  for (const items of slugMap.values()) {
+    if (items.length > 1) {
+      items.sort((a, b) => b.fileCount - a.fileCount)
+      const winner = items[0]
+      kept.push({
+        id: winner.lesson.id,
+        title: winner.lesson.title,
+        slug: winner.slug,
+        fileCount: winner.fileCount,
+      })
+      for (const loser of items.slice(1)) {
+        duplicates.push({
+          id: loser.lesson.id,
+          title: loser.lesson.title,
+          slug: loser.slug,
+          fileCount: loser.fileCount,
+        })
+      }
+    } else {
+      kept.push({
+        id: items[0].lesson.id,
+        title: items[0].lesson.title,
+        slug: items[0].slug,
+        fileCount: items[0].fileCount,
+      })
+    }
+  }
+
+  // Apply removals only in prune mode (additive mode never deletes).
+  if (prune && (removed.length > 0 || duplicates.length > 0)) {
+    syllabus.lessons = syllabus.lessons.filter(lesson => {
+      const lessonSlug = slugify(lesson.id + "-" + lesson.title)
+      const isRemoved = removed.some(r => r.slug === lessonSlug)
+      const isDuplicate = duplicates.some(
+        d => d.id === lesson.id && d.title === lesson.title
+      )
+      return !isRemoved && !isDuplicate
+    })
+    changed = true
+  }
+
+  // Third pass: add lessons that exist in the bucket but are missing from the
+  // syllabus, then re-sort by numeric id.
+  const existingSlugs = new Set(existingLessons.map(e => e.slug))
+  for (const [folderSlug, count] of folderCounts) {
+    if (count <= 0 || existingSlugs.has(folderSlug)) {
+      continue
+    }
+
+    const idMatch = folderSlug.match(/^(\d+(?:\.\d+)?)(?:-(.*))?$/)
+    const id = idMatch ? idMatch[1] : folderSlug
+    const titlePart = idMatch && idMatch[2] ? idMatch[2] : folderSlug
+    const title = titlePart.replace(/-/g, " ").trim() || folderSlug
+    const newLesson: Lesson = {
+      id,
+      uid: folderSlug,
+      title,
+      type: "READ",
+      description: title,
+      duration: 2,
+      generated: true,
+      status: "DONE",
+    }
+    syllabus.lessons.push(newLesson)
+    added.push({ id, title, slug: folderSlug })
+    changed = true
+  }
+
+  if (added.length > 0) {
+    syllabus.lessons.sort((a, b) => {
+      const na = Number.parseFloat(a.id)
+      const nb = Number.parseFloat(b.id)
+      if (!Number.isNaN(na) && !Number.isNaN(nb)) {
+        return na - nb
+      }
+
+      return String(a.id).localeCompare(String(b.id))
+    })
+  }
+
+  // Fourth + fifth pass: reconcile translations metadata from the actual README
+  // files, and unstick lessons stuck in GENERATING/ERROR whose primary-language
+  // README already exists.
+  try {
+    const translationsBySlug = await storage.translationLangsBySlug(courseSlug)
+
+    for (const lesson of syllabus.lessons) {
+      const candidateSlugs = [
+        slugify(lesson.id + "-" + lesson.title),
+        lesson.uid,
+      ].filter(Boolean) as string[]
+      const matchedSlug = candidateSlugs.find(s => translationsBySlug.has(s))
+      if (!matchedSlug) {
+        continue
+      }
+
+      const languageCodes = translationsBySlug.get(matchedSlug) || []
+      if (languageCodes.length === 0) {
+        continue
+      }
+
+      const currentTranslations = lesson.translations || {}
+      let lessonChanged = false
+
+      for (const lang of languageCodes) {
+        const now = Date.now()
+        if (!currentTranslations[lang]) {
+          currentTranslations[lang] = {
+            completionId: 0,
+            startedAt: now,
+            completedAt: now,
+          }
+          repairedTranslationEntries += 1
+          lessonChanged = true
+        } else {
+          if (!currentTranslations[lang].startedAt) {
+            currentTranslations[lang].startedAt = now
+            lessonChanged = true
+          }
+
+          if (!currentTranslations[lang].completedAt) {
+            currentTranslations[lang].completedAt = now
+            repairedTranslationEntries += 1
+            lessonChanged = true
+          }
+        }
+      }
+
+      if (lessonChanged) {
+        lesson.translations = currentTranslations
+        repairedTranslationsInLessons += 1
+        changed = true
+      }
+    }
+
+    const primaryLanguage = syllabus.courseInfo.language || "en"
+    for (const lesson of syllabus.lessons) {
+      if (lesson.generated !== false) {
+        continue
+      }
+
+      if (lesson.status !== "GENERATING" && lesson.status !== "ERROR") {
+        continue
+      }
+
+      const candidateSlugs = [
+        slugify(lesson.id + "-" + lesson.title),
+        lesson.uid,
+      ].filter(Boolean) as string[]
+      const matchedSlug = candidateSlugs.find(s => translationsBySlug.has(s))
+      if (!matchedSlug) {
+        continue
+      }
+
+      const langs = translationsBySlug.get(matchedSlug) || []
+      if (!langs.includes(primaryLanguage)) {
+        continue
+      }
+
+      lesson.generated = true
+      lesson.status = "DONE"
+      fixedLessons += 1
+      changed = true
+    }
+  } catch (error) {
+    console.error(
+      "⚠️ Could not reconcile lesson translations during syllabus sync:",
+      error
+    )
+  }
+
+  const counters: SyllabusSyncCounters = {
+    totalLessons: syllabus.lessons.length,
+    keptLessons: kept.length,
+    removedLessons: prune ? removed.length : 0,
+    duplicatesResolved: prune ? duplicates.length : 0,
+    addedLessons: added.length,
+    fixedLessons,
+    repairedTranslationsInLessons,
+    repairedTranslationEntries,
+  }
+
+  return {
+    syllabus,
+    changed,
+    counters,
+    details: { removed, duplicates, kept, added },
+  }
+}
+
+async function withRetry<T>(fn: () => Promise<T>, attempts = 3): Promise<T> {
+  let lastError: unknown
+  for (let attempt = 0; attempt < attempts; attempt++) {
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      return await fn()
+    } catch (error) {
+      lastError = error
+    }
+  }
+
+  throw lastError
+}
+
+/** GCS-backed storage adapter (uses the bucket listing + buildConfig). */
+export function createGcsSyllabusSyncStorage(
+  bucket: Bucket
+): SyllabusSyncStorage {
+  return {
+    async listExerciseFolderFileCounts(courseSlug) {
+      const prefix = `courses/${courseSlug}/exercises/`
+      const [files] = await withRetry(() => bucket.getFiles({ prefix }))
+      const counts = new Map<string, number>()
+      for (const file of files) {
+        if (!file.name.startsWith(prefix)) {
+          continue
+        }
+
+        const segment = file.name.slice(prefix.length).split("/")[0]
+        if (segment) {
+          counts.set(segment, (counts.get(segment) || 0) + 1)
+        }
+      }
+
+      return counts
+    },
+    async translationLangsBySlug(courseSlug) {
+      const { exercises } = await buildConfig(bucket, courseSlug)
+      const map = new Map<string, string[]>()
+      for (const exercise of exercises) {
+        const langs = Object.keys(exercise.translations || {})
+          .map(lang => lang.toLowerCase())
+          .filter(Boolean)
+        if (langs.length > 0) {
+          map.set(exercise.slug, [...new Set(langs)])
+        }
+      }
+
+      return map
+    },
+  }
+}