@learnpack/learnpack 5.0.350 → 5.0.352

package/lib/commands/serve.js

@@ -33,6 +33,7 @@ const creatorUtilities_2 = require("../utils/creatorUtilities");
 const sidebarGenerator_1 = require("../utils/sidebarGenerator");
 const publish_1 = require("./publish");
 const export_1 = require("../utils/export");
+const packageManifest_1 = require("../utils/packageManifest");
 const errorHandler_1 = require("../utils/errorHandler");
 const jsdom_1 = require("jsdom");
 const redis_1 = require("redis");
@@ -3956,6 +3957,10 @@ class ServeCommand extends SessionCommand_1.default {
                     const [buf] = await file.download();
                     fs.writeFileSync(out, buf);
                 }));
+                await (0, packageManifest_1.generateAndPersistPackageManifest)(bucket, slug, {
+                    publishedAt: new Date().toISOString(),
+                    localDir: buildRoot,
+                });
                 // 8) Crear el config.json en buildRoot con lo que retorna buildConfig
                 const fullConfig = { config, exercises };
                 fullConfig.config.slug = slug;
@@ -4576,6 +4581,7 @@ class ServeCommand extends SessionCommand_1.default {
             ".learn/config.json",
             ".learn/initialSyllabus.json",
             ".learn/sidebar.json",
+            ".learn/package-manifest.json",
             ".learn/memory_bank.txt",
         ]);
         const isStructuralFile = (relativePath) => GITHUB_STRUCTURAL_FILES.has(relativePath) ||

package/lib/utils/export/scorm.js

@@ -8,6 +8,7 @@ const mkdirp = require("mkdirp");
 const rimraf = require("rimraf");
 const uuid_1 = require("uuid");
 const shared_1 = require("./shared");
+const packageManifest_1 = require("../packageManifest");
 async function exportToScorm(options) {
     var _a;
     const { courseSlug, bucket, outDir } = options;
@@ -45,6 +46,10 @@ async function exportToScorm(options) {
     // 3. Download exercises and .learn from bucket
     await (0, shared_1.downloadS3Folder)(bucket, `courses/${courseSlug}/exercises/`, path.join(scormOutDir, "exercises"));
     await (0, shared_1.downloadS3Folder)(bucket, `courses/${courseSlug}/.learn/`, path.join(scormOutDir, ".learn"));
+    await (0, packageManifest_1.generateAndPersistPackageManifest)(bucket, courseSlug, {
+        publishedAt: null,
+        localDir: scormOutDir,
+    });
     // 4. Read learn.json for course info
     const learnJson = await (0, shared_1.getCourseMetadata)(bucket, courseSlug);
     // Resolve the course title once, escaped for safe XML/HTML embedding

package/lib/utils/export/zip.js

@@ -8,6 +8,7 @@ const mkdirp = require("mkdirp");
 const rimraf = require("rimraf");
 const uuid_1 = require("uuid");
 const shared_1 = require("./shared");
+const packageManifest_1 = require("../packageManifest");
 async function exportToZip(options) {
     const { courseSlug, bucket, outDir } = options;
     // 1. Create temporary folder
@@ -15,6 +16,9 @@ async function exportToZip(options) {
     const zipOutDir = path.join(outDir, tmpName);
     rimraf.sync(zipOutDir);
     mkdirp.sync(zipOutDir);
+    await (0, packageManifest_1.generateAndPersistPackageManifest)(bucket, courseSlug, {
+        publishedAt: null,
+    });
     // 2. Download all course files from bucket to temporary directory
     await (0, shared_1.downloadS3Folder)(bucket, `courses/${courseSlug}/`, zipOutDir);
     // 3. Create ZIP file

package/lib/utils/packageManifest.d.ts

@@ -0,0 +1,58 @@
+import { Bucket } from "@google-cloud/storage";
+import { Syllabus } from "../models/creator";
+import { Exercise } from "./configBuilder";
+export declare const PACKAGE_MANIFEST_FILENAME = "package-manifest.json";
+export declare const PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json";
+export declare const SCHEMA_VERSION = 1;
+export type LessonType = "READ" | "CODE" | "QUIZ";
+export type PackageManifestLesson = {
+    id: string;
+    slug: string;
+    position: number;
+    type: LessonType;
+    title: Record<string, string>;
+    description: Record<string, string | null>;
+    video: Record<string, string | null>;
+};
+export type PackageManifest = {
+    schemaVersion: number;
+    generatedAt: string;
+    publishedAt: string | null;
+    slug: string;
+    title: Record<string, string>;
+    description: Record<string, string>;
+    preview: string;
+    technologies?: string[];
+    difficulty?: string;
+    duration?: number;
+    lessons: PackageManifestLesson[];
+};
+export type ReadmeFrontmatter = {
+    tutorial?: string;
+    intro?: string;
+    title?: string;
+};
+export type PackageManifestBuildInput = {
+    learnJson: Record<string, unknown>;
+    syllabus: Syllabus | null;
+    sidebar: Record<string, Record<string, string>>;
+    exercises: Exercise[];
+    readmeFrontmatters: Record<string, Record<string, ReadmeFrontmatter>>;
+    options: {
+        slug: string;
+        generatedAt: string;
+        publishedAt: string | null;
+    };
+};
+export declare function packageManifestGcsPath(courseSlug: string): string;
+export declare function resolvePreviewUrl(slug: string, preview?: string): string;
+export declare function extractReadmeH1(body: string): string | null;
+export declare function normalizeReadmeTitle(raw: string): string | null;
+export declare function buildPackageManifestFromSources(input: PackageManifestBuildInput): PackageManifest;
+export declare function parseReadmeContent(content: string): ReadmeFrontmatter;
+export declare function comparePackageManifests(existing: PackageManifest | null, next: PackageManifest): "created" | "updated" | "skipped";
+export declare function serializePackageManifest(manifest: PackageManifest): string;
+export declare function generateAndPersistPackageManifest(bucket: Bucket, courseSlug: string, options: {
+    publishedAt: string | null;
+    localDir?: string;
+}): Promise<PackageManifest>;

package/lib/utils/packageManifest.js

@@ -0,0 +1,250 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SCHEMA_VERSION = exports.PACKAGE_MANIFEST_REL_PATH = exports.PACKAGE_MANIFEST_FILENAME = void 0;
+exports.packageManifestGcsPath = packageManifestGcsPath;
+exports.resolvePreviewUrl = resolvePreviewUrl;
+exports.extractReadmeH1 = extractReadmeH1;
+exports.normalizeReadmeTitle = normalizeReadmeTitle;
+exports.buildPackageManifestFromSources = buildPackageManifestFromSources;
+exports.parseReadmeContent = parseReadmeContent;
+exports.comparePackageManifests = comparePackageManifests;
+exports.serializePackageManifest = serializePackageManifest;
+exports.generateAndPersistPackageManifest = generateAndPersistPackageManifest;
+const tslib_1 = require("tslib");
+const fs = require("fs");
+const mkdirp = require("mkdirp");
+const path = require("path");
+const configBuilder_1 = require("./configBuilder");
+const creatorUtilities_1 = require("./creatorUtilities");
+const titlefy_1 = require("./titlefy");
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const frontMatter = require("front-matter");
+exports.PACKAGE_MANIFEST_FILENAME = "package-manifest.json";
+exports.PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json";
+exports.SCHEMA_VERSION = 1;
+const VALID_LESSON_TYPES = new Set(["READ", "CODE", "QUIZ"]);
+function packageManifestGcsPath(courseSlug) {
+    return `courses/${courseSlug}/${exports.PACKAGE_MANIFEST_REL_PATH}`;
+}
+function resolvePreviewUrl(slug, preview) {
+    if (preview && preview.startsWith("http")) {
+        return preview;
+    }
+    return `https://${slug}.learn-pack.com/preview.png`;
+}
+function findSyllabusLesson(exerciseSlug, lessons) {
+    return lessons.find(lesson => {
+        const candidates = [
+            (0, creatorUtilities_1.slugify)(lesson.id + "-" + lesson.title),
+            lesson.uid,
+        ].filter(Boolean);
+        return candidates.includes(exerciseSlug);
+    });
+}
+function deriveLessonId(exerciseSlug) {
+    const match = exerciseSlug.match(/^(\d+(?:\.\d+)?)/);
+    return match ? match[1] : exerciseSlug;
+}
+function coerceLessonType(type) {
+    if (type && VALID_LESSON_TYPES.has(type)) {
+        return type;
+    }
+    if (type) {
+        console.warn(`[package-manifest] Invalid lesson type "${type}", defaulting to READ`);
+    }
+    return "READ";
+}
+function extractReadmeH1(body) {
+    const match = body.match(/^#\s+(.+)$/m);
+    if (!match) {
+        return null;
+    }
+    const title = match[1].replace(/\r/g, "").trim();
+    return title || null;
+}
+function normalizeReadmeTitle(raw) {
+    const normalized = raw
+        .trim()
+        .replace(/^\d+(?:\.\d+)?\s*[.:-]?\s*/, "")
+        .trim();
+    return normalized || null;
+}
+function firstNonEmpty(...candidates) {
+    for (const candidate of candidates) {
+        const value = candidate === null || candidate === void 0 ? void 0 : candidate.trim();
+        if (value) {
+            return value;
+        }
+    }
+    return null;
+}
+function resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, readmeByLang) {
+    var _a, _b;
+    const sidebarEntry = (_a = sidebar[exercise.slug]) !== null && _a !== void 0 ? _a : {};
+    const languages = new Set([
+        ...Object.keys(exercise.translations),
+        ...Object.keys(sidebarEntry),
+    ]);
+    const titles = {};
+    for (const lang of languages) {
+        const title = firstNonEmpty((_b = readmeByLang[lang]) === null || _b === void 0 ? void 0 : _b.title, sidebarEntry[lang] ? (0, titlefy_1.formatSidebarTitle)(sidebarEntry[lang]) : null, lang === courseLang ? syllabusLesson === null || syllabusLesson === void 0 ? void 0 : syllabusLesson.title : null);
+        if (title) {
+            titles[lang] = title;
+        }
+    }
+    return titles;
+}
+function resolveLessonDescriptions(exercise) {
+    const description = {};
+    for (const lang of Object.keys(exercise.translations)) {
+        // TODO: add description content
+        description[lang] = null;
+    }
+    return description;
+}
+function resolveLessonVideos(exercise, readmeFrontmatters) {
+    const video = {};
+    const frontmatters = readmeFrontmatters[exercise.slug] || {};
+    for (const lang of Object.keys(exercise.translations)) {
+        const attrs = frontmatters[lang] || {};
+        const url = attrs.tutorial || attrs.intro || null;
+        video[lang] = url && url.trim().length > 0 ? url.trim() : null;
+    }
+    return video;
+}
+function buildPackageManifestFromSources(input) {
+    var _a, _b;
+    const { learnJson, syllabus, sidebar, exercises, readmeFrontmatters, options, } = input;
+    const syllabusLessons = (_a = syllabus === null || syllabus === void 0 ? void 0 : syllabus.lessons) !== null && _a !== void 0 ? _a : [];
+    const courseLang = ((_b = syllabus === null || syllabus === void 0 ? void 0 : syllabus.courseInfo) === null || _b === void 0 ? void 0 : _b.language) || "en";
+    const sortedExercises = [...exercises].sort((a, b) => a.position - b.position);
+    const lessons = sortedExercises.map(exercise => {
+        var _a, _b;
+        const syllabusLesson = findSyllabusLesson(exercise.slug, syllabusLessons);
+        const id = (_a = syllabusLesson === null || syllabusLesson === void 0 ? void 0 : syllabusLesson.id) !== null && _a !== void 0 ? _a : deriveLessonId(exercise.slug);
+        const type = syllabusLesson ?
+            coerceLessonType(syllabusLesson.type) :
+            "READ";
+        if (!syllabusLesson) {
+            console.warn(`[package-manifest] No syllabus match for exercise "${exercise.slug}", using derived id "${id}"`);
+        }
+        return {
+            id,
+            slug: exercise.slug,
+            position: exercise.position,
+            type,
+            title: resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, (_b = readmeFrontmatters[exercise.slug]) !== null && _b !== void 0 ? _b : {}),
+            description: resolveLessonDescriptions(exercise),
+            video: resolveLessonVideos(exercise, readmeFrontmatters),
+        };
+    });
+    const title = learnJson.title || {};
+    const description = learnJson.description || {};
+    return {
+        schemaVersion: exports.SCHEMA_VERSION,
+        generatedAt: options.generatedAt,
+        publishedAt: options.publishedAt,
+        slug: options.slug,
+        title,
+        description,
+        preview: resolvePreviewUrl(options.slug, learnJson.preview),
+        technologies: learnJson.technologies,
+        difficulty: learnJson.difficulty,
+        duration: learnJson.duration,
+        lessons,
+    };
+}
+async function downloadJsonFile(bucket, filePath) {
+    try {
+        const [buf] = await bucket.file(filePath).download();
+        return JSON.parse(buf.toString());
+    }
+    catch (_a) {
+        return null;
+    }
+}
+function parseReadmeContent(content) {
+    const parsed = frontMatter(content);
+    const attributes = parsed.attributes || {};
+    const h1 = extractReadmeH1(parsed.body || "");
+    const title = h1 ? normalizeReadmeTitle(h1) : null;
+    return Object.assign(Object.assign(Object.assign({}, (attributes.tutorial ? { tutorial: attributes.tutorial } : {})), (attributes.intro ? { intro: attributes.intro } : {})), (title ? { title } : {}));
+}
+function comparePackageManifests(existing, next) {
+    if (existing === null) {
+        return "created";
+    }
+    const { generatedAt: _existingGeneratedAt } = existing, existingRest = tslib_1.__rest(existing, ["generatedAt"]);
+    const { generatedAt: _nextGeneratedAt } = next, nextRest = tslib_1.__rest(next, ["generatedAt"]);
+    if (JSON.stringify(existingRest) === JSON.stringify(nextRest)) {
+        return "skipped";
+    }
+    return "updated";
+}
+function serializePackageManifest(manifest) {
+    return JSON.stringify(manifest, null, 2);
+}
+async function readReadmeFrontmatter(bucket, exerciseSlug, lang, readmePath) {
+    try {
+        const [buf] = await bucket.file(readmePath).download();
+        return {
+            exerciseSlug,
+            lang,
+            frontmatter: parseReadmeContent(buf.toString()),
+        };
+    }
+    catch (error) {
+        console.warn(`[package-manifest] Could not read README for ${exerciseSlug} (${lang}):`, error.message);
+        return { exerciseSlug, lang, frontmatter: {} };
+    }
+}
+async function collectReadmeFrontmatters(bucket, courseSlug, exercises) {
+    const readmeTasks = exercises.flatMap(exercise => Object.keys(exercise.translations).map(lang => {
+        const readmeFilename = `README${(0, creatorUtilities_1.getReadmeExtension)(lang)}`;
+        const readmePath = `courses/${courseSlug}/exercises/${exercise.slug}/${readmeFilename}`;
+        return readReadmeFrontmatter(bucket, exercise.slug, lang, readmePath);
+    }));
+    const readmeEntries = await Promise.all(readmeTasks);
+    const result = {};
+    for (const { exerciseSlug, lang, frontmatter } of readmeEntries) {
+        if (!result[exerciseSlug]) {
+            result[exerciseSlug] = {};
+        }
+        result[exerciseSlug][lang] = frontmatter;
+    }
+    return result;
+}
+function writeManifestJson(manifest, localDir) {
+    const json = serializePackageManifest(manifest);
+    if (localDir) {
+        const learnDir = path.join(localDir, ".learn");
+        mkdirp.sync(learnDir);
+        const localPath = path.join(learnDir, exports.PACKAGE_MANIFEST_FILENAME);
+        fs.writeFileSync(localPath, json, "utf-8");
+    }
+    return json;
+}
+async function generateAndPersistPackageManifest(bucket, courseSlug, options) {
+    const generatedAt = new Date().toISOString();
+    const { config: learnJson, exercises } = await (0, configBuilder_1.buildConfig)(bucket, courseSlug);
+    const syllabus = await downloadJsonFile(bucket, `courses/${courseSlug}/.learn/initialSyllabus.json`);
+    const sidebar = (await downloadJsonFile(bucket, `courses/${courseSlug}/.learn/sidebar.json`)) || {};
+    const readmeFrontmatters = await collectReadmeFrontmatters(bucket, courseSlug, exercises);
+    const manifest = buildPackageManifestFromSources({
+        learnJson,
+        syllabus,
+        sidebar,
+        exercises,
+        readmeFrontmatters,
+        options: {
+            slug: courseSlug,
+            generatedAt,
+            publishedAt: options.publishedAt,
+        },
+    });
+    const json = writeManifestJson(manifest, options.localDir);
+    await bucket
+        .file(packageManifestGcsPath(courseSlug))
+        .save(Buffer.from(json, "utf8"));
+    return manifest;
+}

package/lib/utils/s3/packageManifestBackfill.d.ts

@@ -0,0 +1,82 @@
+import { Exercise } from "../configBuilder";
+import { Syllabus } from "../../models/creator";
+import { PackageManifest, ReadmeFrontmatter } from "../packageManifest";
+export type PackageResultStatus = "created" | "updated" | "skipped" | "failed";
+export type PackageResult = {
+    slug: string;
+    status: PackageResultStatus;
+    durationMs: number;
+    error?: string;
+    step?: string;
+    sources?: {
+        learnJson: boolean;
+        configJson: boolean;
+        sidebar: boolean;
+        syllabus: boolean;
+        readmes: number;
+    };
+};
+export type BackfillOptions = {
+    dryRun: boolean;
+    force: boolean;
+    skipExisting: boolean;
+    noInvalidate: boolean;
+    concurrency: number;
+    publishedAt: string | null;
+    resumeFrom?: string;
+    reportOut: string;
+    slug?: string;
+    bucket: string;
+    region: string;
+    cloudFrontDistributionId?: string;
+};
+export type BatchReport = {
+    total: number;
+    created: number;
+    updated: number;
+    skipped: number;
+    failed: number;
+    startedAt: string;
+    finishedAt: string;
+    options: BackfillOptions;
+    results: PackageResult[];
+    invalidation?: {
+        attempted: boolean;
+        skippedReason?: string;
+        id?: string;
+        paths?: string[];
+        error?: string;
+    };
+};
+export type AwsClient = {
+    send(command: unknown): Promise<unknown>;
+};
+type PackageSources = {
+    learnJson: Record<string, unknown>;
+    exercises: Exercise[];
+    sidebar: Record<string, Record<string, string>>;
+    syllabus: Syllabus | null;
+    readmeFrontmatters: Record<string, Record<string, ReadmeFrontmatter>>;
+    meta: PackageResult["sources"];
+};
+export declare function streamToString(body: unknown): Promise<string>;
+export declare function withRetry<T>(fn: () => Promise<T>, attempts?: number, attempt?: number): Promise<T>;
+export declare function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
+export declare function listPublishedSlugs(s3: AwsClient, bucket: string): Promise<string[]>;
+export declare function fetchJsonObject<T>(s3: AwsClient, bucket: string, key: string): Promise<T | null>;
+export declare function fetchPackageSources(s3: AwsClient, bucket: string, slug: string): Promise<PackageSources>;
+export declare function buildManifestFromS3Sources(slug: string, sources: PackageSources, options: {
+    publishedAt: string | null;
+    generatedAt?: string;
+}): PackageManifest;
+export declare function processPackage(s3: AwsClient, bucket: string, slug: string, options: Pick<BackfillOptions, "dryRun" | "force" | "skipExisting" | "publishedAt">): Promise<PackageResult>;
+export declare function invalidatePackageManifestPaths(cf: AwsClient, distributionId: string, paths: string[]): Promise<{
+    id: string;
+}>;
+export declare function runBatch(options: BackfillOptions, clients?: {
+    s3?: AwsClient;
+    cf?: AwsClient;
+    onProgress?: (index: number, total: number, result: PackageResult) => void;
+}): Promise<BatchReport>;
+export declare function getBatchExitCode(report: BatchReport): number;
+export {};