@learnpack/learnpack 5.0.351 → 5.0.353

package/lib/utils/syllabusSync.js

@@ -0,0 +1,273 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.synchronizeSyllabusWithBucket = synchronizeSyllabusWithBucket;
+exports.createGcsSyllabusSyncStorage = createGcsSyllabusSyncStorage;
+const configBuilder_1 = require("./configBuilder");
+const creatorUtilities_1 = require("./creatorUtilities");
+function isInFlight(lesson) {
+    // A lesson still being generated may legitimately have no files yet; never
+    // treat it as "missing" (E4 hardening against deleting something real).
+    return (lesson.generated === false ||
+        lesson.status === "GENERATING" ||
+        lesson.status === "PENDING");
+}
+async function synchronizeSyllabusWithBucket(storage, courseSlug, syllabus, options = {}) {
+    var _a, _b, _c, _d;
+    const prune = (_a = options.prune) !== null && _a !== void 0 ? _a : false;
+    const removed = [];
+    const duplicates = [];
+    const kept = [];
+    const added = [];
+    let repairedTranslationsInLessons = 0;
+    let repairedTranslationEntries = 0;
+    let fixedLessons = 0;
+    let changed = false;
+    // Single authoritative listing. If it fails it throws (callers must not
+    // delete on an unknown listing). The adapter is expected to retry internally.
+    const folderCounts = await storage.listExerciseFolderFileCounts(courseSlug);
+    // First pass: resolve existence per lesson, confirming absence under both the
+    // slugified id-title and the uid folder-name variants.
+    const existingLessons = [];
+    for (const lesson of syllabus.lessons) {
+        const lessonSlug = (0, creatorUtilities_1.slugify)(lesson.id + "-" + lesson.title);
+        let resolvedSlug = lessonSlug;
+        let fileCount = (_b = folderCounts.get(lessonSlug)) !== null && _b !== void 0 ? _b : 0;
+        if (fileCount === 0 &&
+            lesson.uid &&
+            ((_c = folderCounts.get(lesson.uid)) !== null && _c !== void 0 ? _c : 0) > 0) {
+            resolvedSlug = lesson.uid;
+            fileCount = (_d = folderCounts.get(lesson.uid)) !== null && _d !== void 0 ? _d : 0;
+        }
+        if (fileCount === 0) {
+            if (isInFlight(lesson)) {
+                // Keep untouched: do not mark for removal nor as existing.
+                continue;
+            }
+            removed.push({ id: lesson.id, title: lesson.title, slug: lessonSlug });
+        }
+        else {
+            existingLessons.push({ lesson, slug: resolvedSlug, fileCount });
+        }
+    }
+    // Second pass: detect duplicates (multiple lessons resolving to one folder).
+    const slugMap = new Map();
+    for (const item of existingLessons) {
+        if (!slugMap.has(item.slug)) {
+            slugMap.set(item.slug, []);
+        }
+        slugMap.get(item.slug).push(item);
+    }
+    for (const items of slugMap.values()) {
+        if (items.length > 1) {
+            items.sort((a, b) => b.fileCount - a.fileCount);
+            const winner = items[0];
+            kept.push({
+                id: winner.lesson.id,
+                title: winner.lesson.title,
+                slug: winner.slug,
+                fileCount: winner.fileCount,
+            });
+            for (const loser of items.slice(1)) {
+                duplicates.push({
+                    id: loser.lesson.id,
+                    title: loser.lesson.title,
+                    slug: loser.slug,
+                    fileCount: loser.fileCount,
+                });
+            }
+        }
+        else {
+            kept.push({
+                id: items[0].lesson.id,
+                title: items[0].lesson.title,
+                slug: items[0].slug,
+                fileCount: items[0].fileCount,
+            });
+        }
+    }
+    // Apply removals only in prune mode (additive mode never deletes).
+    if (prune && (removed.length > 0 || duplicates.length > 0)) {
+        syllabus.lessons = syllabus.lessons.filter(lesson => {
+            const lessonSlug = (0, creatorUtilities_1.slugify)(lesson.id + "-" + lesson.title);
+            const isRemoved = removed.some(r => r.slug === lessonSlug);
+            const isDuplicate = duplicates.some(d => d.id === lesson.id && d.title === lesson.title);
+            return !isRemoved && !isDuplicate;
+        });
+        changed = true;
+    }
+    // Third pass: add lessons that exist in the bucket but are missing from the
+    // syllabus, then re-sort by numeric id.
+    const existingSlugs = new Set(existingLessons.map(e => e.slug));
+    for (const [folderSlug, count] of folderCounts) {
+        if (count <= 0 || existingSlugs.has(folderSlug)) {
+            continue;
+        }
+        const idMatch = folderSlug.match(/^(\d+(?:\.\d+)?)(?:-(.*))?$/);
+        const id = idMatch ? idMatch[1] : folderSlug;
+        const titlePart = idMatch && idMatch[2] ? idMatch[2] : folderSlug;
+        const title = titlePart.replace(/-/g, " ").trim() || folderSlug;
+        const newLesson = {
+            id,
+            uid: folderSlug,
+            title,
+            type: "READ",
+            description: title,
+            duration: 2,
+            generated: true,
+            status: "DONE",
+        };
+        syllabus.lessons.push(newLesson);
+        added.push({ id, title, slug: folderSlug });
+        changed = true;
+    }
+    if (added.length > 0) {
+        syllabus.lessons.sort((a, b) => {
+            const na = Number.parseFloat(a.id);
+            const nb = Number.parseFloat(b.id);
+            if (!Number.isNaN(na) && !Number.isNaN(nb)) {
+                return na - nb;
+            }
+            return String(a.id).localeCompare(String(b.id));
+        });
+    }
+    // Fourth + fifth pass: reconcile translations metadata from the actual README
+    // files, and unstick lessons stuck in GENERATING/ERROR whose primary-language
+    // README already exists.
+    try {
+        const translationsBySlug = await storage.translationLangsBySlug(courseSlug);
+        for (const lesson of syllabus.lessons) {
+            const candidateSlugs = [
+                (0, creatorUtilities_1.slugify)(lesson.id + "-" + lesson.title),
+                lesson.uid,
+            ].filter(Boolean);
+            const matchedSlug = candidateSlugs.find(s => translationsBySlug.has(s));
+            if (!matchedSlug) {
+                continue;
+            }
+            const languageCodes = translationsBySlug.get(matchedSlug) || [];
+            if (languageCodes.length === 0) {
+                continue;
+            }
+            const currentTranslations = lesson.translations || {};
+            let lessonChanged = false;
+            for (const lang of languageCodes) {
+                const now = Date.now();
+                if (!currentTranslations[lang]) {
+                    currentTranslations[lang] = {
+                        completionId: 0,
+                        startedAt: now,
+                        completedAt: now,
+                    };
+                    repairedTranslationEntries += 1;
+                    lessonChanged = true;
+                }
+                else {
+                    if (!currentTranslations[lang].startedAt) {
+                        currentTranslations[lang].startedAt = now;
+                        lessonChanged = true;
+                    }
+                    if (!currentTranslations[lang].completedAt) {
+                        currentTranslations[lang].completedAt = now;
+                        repairedTranslationEntries += 1;
+                        lessonChanged = true;
+                    }
+                }
+            }
+            if (lessonChanged) {
+                lesson.translations = currentTranslations;
+                repairedTranslationsInLessons += 1;
+                changed = true;
+            }
+        }
+        const primaryLanguage = syllabus.courseInfo.language || "en";
+        for (const lesson of syllabus.lessons) {
+            if (lesson.generated !== false) {
+                continue;
+            }
+            if (lesson.status !== "GENERATING" && lesson.status !== "ERROR") {
+                continue;
+            }
+            const candidateSlugs = [
+                (0, creatorUtilities_1.slugify)(lesson.id + "-" + lesson.title),
+                lesson.uid,
+            ].filter(Boolean);
+            const matchedSlug = candidateSlugs.find(s => translationsBySlug.has(s));
+            if (!matchedSlug) {
+                continue;
+            }
+            const langs = translationsBySlug.get(matchedSlug) || [];
+            if (!langs.includes(primaryLanguage)) {
+                continue;
+            }
+            lesson.generated = true;
+            lesson.status = "DONE";
+            fixedLessons += 1;
+            changed = true;
+        }
+    }
+    catch (error) {
+        console.error("⚠️ Could not reconcile lesson translations during syllabus sync:", error);
+    }
+    const counters = {
+        totalLessons: syllabus.lessons.length,
+        keptLessons: kept.length,
+        removedLessons: prune ? removed.length : 0,
+        duplicatesResolved: prune ? duplicates.length : 0,
+        addedLessons: added.length,
+        fixedLessons,
+        repairedTranslationsInLessons,
+        repairedTranslationEntries,
+    };
+    return {
+        syllabus,
+        changed,
+        counters,
+        details: { removed, duplicates, kept, added },
+    };
+}
+async function withRetry(fn, attempts = 3) {
+    let lastError;
+    for (let attempt = 0; attempt < attempts; attempt++) {
+        try {
+            // eslint-disable-next-line no-await-in-loop
+            return await fn();
+        }
+        catch (error) {
+            lastError = error;
+        }
+    }
+    throw lastError;
+}
+/** GCS-backed storage adapter (uses the bucket listing + buildConfig). */
+function createGcsSyllabusSyncStorage(bucket) {
+    return {
+        async listExerciseFolderFileCounts(courseSlug) {
+            const prefix = `courses/${courseSlug}/exercises/`;
+            const [files] = await withRetry(() => bucket.getFiles({ prefix }));
+            const counts = new Map();
+            for (const file of files) {
+                if (!file.name.startsWith(prefix)) {
+                    continue;
+                }
+                const segment = file.name.slice(prefix.length).split("/")[0];
+                if (segment) {
+                    counts.set(segment, (counts.get(segment) || 0) + 1);
+                }
+            }
+            return counts;
+        },
+        async translationLangsBySlug(courseSlug) {
+            const { exercises } = await (0, configBuilder_1.buildConfig)(bucket, courseSlug);
+            const map = new Map();
+            for (const exercise of exercises) {
+                const langs = Object.keys(exercise.translations || {})
+                    .map(lang => lang.toLowerCase())
+                    .filter(Boolean);
+                if (langs.length > 0) {
+                    map.set(exercise.slug, [...new Set(langs)]);
+                }
+            }
+            return map;
+        },
+    };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@learnpack/learnpack",
   "description": "Seamlessly build, sell and/or take interactive & auto-graded tutorials, start learning now or build a new tutorial to your audience.",
-  "version": "5.0.351",
+  "version": "5.0.353",
   "author": "Alejandro Sanchez @alesanchezr",
   "contributors": [
     {
@@ -21,6 +21,8 @@
     "url": "https://github.com/learnpack/learnpack-cli/issues"
   },
   "dependencies": {
+    "@aws-sdk/client-cloudfront": "^3.1092.0",
+    "@aws-sdk/client-s3": "^3.1092.0",
     "@google-cloud/storage": "^7.16.0",
     "@oclif/command": "^1.6.1",
     "@oclif/config": "^1.15.1",
@@ -161,6 +163,7 @@
     ]
   },
   "scripts": {
+    "backfill:package-manifest": "ts-node --project ./tsconfig.json ./scripts/backfill-package-manifest-s3.ts",
     "copy-assets": "npx cpy src/creatorDist/**/* lib/creatorDist --parents --verbose && npx cpy src/utils/templates/**/* lib/utils/templates --parents --verbose && npx cpy src/lua/**/* lib/lua --parents --verbose",
     "tsc": "tsc -b",
     "postpack": "rm -f oclif.manifest.json && eslint . --ext .ts --config .eslintrc",

package/src/commands/publish.ts

@@ -666,6 +666,13 @@ class BuildCommand extends SessionCommand {
 
       Console.debug("Zip file saved in project root")
 
+      // TODO(#2103): this CLI path publishes without the guarantees the
+      // creator flow has. It builds no package-manifest.json (pre-existing), it
+      // does not run the descriptions stage, and it emits neither
+      // package_published nor package_manifest_updated, so breathecode never
+      // hears about these publications. Decide between parity, deferring the
+      // work to the sweep, or documenting this path as degraded.
+
       const formData = new FormData()
       formData.append("file", fs.createReadStream(zipFilePath))
       formData.append("config", JSON.stringify(learnJson))