@learnpack/learnpack 5.0.350 → 5.0.352

package/lib/utils/s3/packageManifestBackfill.js

@@ -0,0 +1,485 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.streamToString = streamToString;
+exports.withRetry = withRetry;
+exports.mapWithConcurrency = mapWithConcurrency;
+exports.listPublishedSlugs = listPublishedSlugs;
+exports.fetchJsonObject = fetchJsonObject;
+exports.fetchPackageSources = fetchPackageSources;
+exports.buildManifestFromS3Sources = buildManifestFromS3Sources;
+exports.processPackage = processPackage;
+exports.invalidatePackageManifestPaths = invalidatePackageManifestPaths;
+exports.runBatch = runBatch;
+exports.getBatchExitCode = getBatchExitCode;
+const tslib_1 = require("tslib");
+const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const packageManifest_1 = require("../packageManifest");
+const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i;
+const INDEX_HTML_PATTERN = /^[^/]+\/index\.html$/;
+const TRANSIENT_ERROR_CODES = new Set([
+    "SlowDown",
+    "ServiceUnavailable",
+    "RequestTimeout",
+    "Throttling",
+    "ThrottlingException",
+    "TooManyRequestsException",
+]);
+const CLOUDFRONT_INVALIDATION_CHUNK_SIZE = 3000;
+async function streamToString(body) {
+    var _a, e_1, _b, _c;
+    if (!body) {
+        return "";
+    }
+    if (typeof body === "string") {
+        return body;
+    }
+    if (Buffer.isBuffer(body)) {
+        return body.toString("utf-8");
+    }
+    if (body instanceof Uint8Array) {
+        return Buffer.from(body).toString("utf-8");
+    }
+    const chunks = [];
+    try {
+        for (var _d = true, _e = tslib_1.__asyncValues(body), _f; _f = await _e.next(), _a = _f.done, !_a; _d = true) {
+            _c = _f.value;
+            _d = false;
+            const chunk = _c;
+            chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+        }
+    }
+    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+    finally {
+        try {
+            if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
+        }
+        finally { if (e_1) throw e_1.error; }
+    }
+    return Buffer.concat(chunks).toString("utf-8");
+}
+function isTransientError(error) {
+    var _a;
+    const err = error;
+    const code = err.name || err.Code || "";
+    if (TRANSIENT_ERROR_CODES.has(code)) {
+        return true;
+    }
+    const status = (_a = err.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode;
+    return status === 429 || status === 503;
+}
+function sleep(ms) {
+    return new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+}
+async function withRetry(fn, attempts = 3, attempt = 1) {
+    try {
+        return await fn();
+    }
+    catch (error) {
+        if (!isTransientError(error) || attempt === attempts) {
+            throw error;
+        }
+        const multiplier = 2 ** (attempt - 1);
+        const delayMs = 250 * multiplier;
+        await sleep(delayMs);
+        return withRetry(fn, attempts, attempt + 1);
+    }
+}
+async function mapWithConcurrency(items, limit, fn) {
+    const results = Array.from({ length: items.length });
+    let nextIndex = 0;
+    async function worker() {
+        while (nextIndex < items.length) {
+            const currentIndex = nextIndex;
+            nextIndex += 1;
+            // eslint-disable-next-line no-await-in-loop -- each worker processes items sequentially
+            results[currentIndex] = await fn(items[currentIndex]);
+        }
+    }
+    const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+    await Promise.all(workers);
+    return results;
+}
+function manifestKey(slug) {
+    return `${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
+}
+function cloudFrontManifestPath(slug) {
+    return `/${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
+}
+function isNotFoundError(error) {
+    const err = error;
+    const code = err.name || err.Code || "";
+    return code === "NoSuchKey" || code === "NotFound" || code === "NoSuchBucket";
+}
+async function listPublishedSlugs(s3, bucket) {
+    const slugs = new Set();
+    let continuationToken;
+    do {
+        // eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
+        const response = (await withRetry(() => s3.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: bucket,
+            ContinuationToken: continuationToken,
+        }))));
+        for (const item of response.Contents || []) {
+            const key = item.Key;
+            if (key && INDEX_HTML_PATTERN.test(key)) {
+                slugs.add(key.split("/")[0]);
+            }
+        }
+        continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+    return [...slugs].sort();
+}
+async function fetchJsonObject(s3, bucket, key) {
+    try {
+        const response = (await withRetry(() => s3.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))));
+        const content = await streamToString(response.Body);
+        return JSON.parse(content);
+    }
+    catch (error) {
+        if (isNotFoundError(error)) {
+            return null;
+        }
+        throw error;
+    }
+}
+async function fetchTextObject(s3, bucket, key) {
+    try {
+        const response = (await withRetry(() => s3.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))));
+        return await streamToString(response.Body);
+    }
+    catch (error) {
+        if (isNotFoundError(error)) {
+            return null;
+        }
+        throw error;
+    }
+}
+async function listObjectKeys(s3, bucket, prefix) {
+    const keys = [];
+    let continuationToken;
+    do {
+        // eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
+        const response = (await withRetry(() => s3.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: bucket,
+            Prefix: prefix,
+            ContinuationToken: continuationToken,
+        }))));
+        for (const item of response.Contents || []) {
+            if (item.Key) {
+                keys.push(item.Key);
+            }
+        }
+        continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+    return keys;
+}
+function resolveReadmeLang(filename, exercise) {
+    const lower = filename.toLowerCase();
+    for (const [lang, readmeFile] of Object.entries(exercise.translations)) {
+        if (readmeFile.toLowerCase() === lower) {
+            return lang;
+        }
+    }
+    return null;
+}
+function normalizeLearnAndExercises(slug, learnJson, configJson) {
+    const exercises = configJson === null || configJson === void 0 ? void 0 : configJson.exercises;
+    if (!Array.isArray(exercises) || exercises.length === 0) {
+        throw new Error(`Missing or empty exercises array in ${slug}/config.json`);
+    }
+    const resolvedLearn = learnJson ||
+        (configJson === null || configJson === void 0 ? void 0 : configJson.config) ||
+        null;
+    if (!resolvedLearn) {
+        throw new Error(`Missing learn metadata: no ${slug}/learn.json and no config.config`);
+    }
+    return { learnJson: resolvedLearn, exercises };
+}
+async function fetchPackageSources(s3, bucket, slug) {
+    const [learnJsonRaw, configJsonRoot, configJsonLearn, sidebarLearn, sidebarRoot, syllabus,] = await Promise.all([
+        fetchJsonObject(s3, bucket, `${slug}/learn.json`),
+        fetchJsonObject(s3, bucket, `${slug}/config.json`),
+        fetchJsonObject(s3, bucket, `${slug}/.learn/config.json`),
+        fetchJsonObject(s3, bucket, `${slug}/.learn/sidebar.json`),
+        fetchJsonObject(s3, bucket, `${slug}/sidebar.json`),
+        fetchJsonObject(s3, bucket, `${slug}/.learn/initialSyllabus.json`),
+    ]);
+    const configJson = configJsonRoot || configJsonLearn;
+    if (!configJson) {
+        throw new Error(`Missing required file: ${slug}/config.json or ${slug}/.learn/config.json`);
+    }
+    const { learnJson, exercises } = normalizeLearnAndExercises(slug, learnJsonRaw, configJson);
+    const exerciseSlugs = new Set(exercises.map(exercise => exercise.slug));
+    const readmeKeys = (await listObjectKeys(s3, bucket, `${slug}/exercises/`))
+        .filter(key => {
+        const filename = key.split("/").pop() || "";
+        return README_FILENAME_PATTERN.test(filename);
+    })
+        .filter(key => {
+        const parts = key.split("/");
+        const exerciseSlug = parts[2];
+        return Boolean(exerciseSlug && exerciseSlugs.has(exerciseSlug));
+    });
+    const readmeFrontmatters = {};
+    await Promise.all(readmeKeys.map(async (key) => {
+        const parts = key.split("/");
+        const exerciseSlug = parts[2];
+        const filename = parts[parts.length - 1];
+        const exercise = exercises.find(item => item.slug === exerciseSlug);
+        if (!exercise) {
+            return;
+        }
+        const lang = resolveReadmeLang(filename, exercise);
+        if (!lang) {
+            return;
+        }
+        const content = await fetchTextObject(s3, bucket, key);
+        if (!content) {
+            return;
+        }
+        if (!readmeFrontmatters[exerciseSlug]) {
+            readmeFrontmatters[exerciseSlug] = {};
+        }
+        readmeFrontmatters[exerciseSlug][lang] = (0, packageManifest_1.parseReadmeContent)(content);
+    }));
+    const readmeCount = Object.values(readmeFrontmatters).reduce((total, byLang) => total + Object.keys(byLang).length, 0);
+    return {
+        learnJson,
+        exercises,
+        sidebar: sidebarLearn || sidebarRoot || {},
+        syllabus,
+        readmeFrontmatters,
+        meta: {
+            learnJson: Boolean(learnJsonRaw || configJson.config),
+            configJson: true,
+            sidebar: Boolean(sidebarLearn || sidebarRoot),
+            syllabus: Boolean(syllabus),
+            readmes: readmeCount,
+        },
+    };
+}
+function buildManifestFromS3Sources(slug, sources, options) {
+    return (0, packageManifest_1.buildPackageManifestFromSources)({
+        learnJson: sources.learnJson,
+        syllabus: sources.syllabus,
+        sidebar: sources.sidebar,
+        exercises: sources.exercises,
+        readmeFrontmatters: sources.readmeFrontmatters,
+        options: {
+            slug,
+            generatedAt: options.generatedAt || new Date().toISOString(),
+            publishedAt: options.publishedAt,
+        },
+    });
+}
+async function processPackage(s3, bucket, slug, options) {
+    const started = Date.now();
+    try {
+        const existing = await fetchJsonObject(s3, bucket, manifestKey(slug));
+        if (options.skipExisting && existing) {
+            return {
+                slug,
+                status: "skipped",
+                durationMs: Date.now() - started,
+                sources: {
+                    learnJson: false,
+                    configJson: false,
+                    sidebar: false,
+                    syllabus: false,
+                    readmes: 0,
+                },
+            };
+        }
+        const sources = await fetchPackageSources(s3, bucket, slug);
+        const next = buildManifestFromS3Sources(slug, sources, {
+            publishedAt: options.publishedAt,
+        });
+        let status = (0, packageManifest_1.comparePackageManifests)(existing, next);
+        if (options.force && status === "skipped") {
+            status = "updated";
+        }
+        if (!options.dryRun && (status === "created" || status === "updated")) {
+            await withRetry(() => s3.send(new client_s3_1.PutObjectCommand({
+                Bucket: bucket,
+                Key: manifestKey(slug),
+                Body: (0, packageManifest_1.serializePackageManifest)(next),
+                ContentType: "application/json",
+            })));
+        }
+        return {
+            slug,
+            status,
+            durationMs: Date.now() - started,
+            sources: sources.meta,
+        };
+    }
+    catch (error) {
+        const err = error;
+        return {
+            slug,
+            status: "failed",
+            durationMs: Date.now() - started,
+            error: err.message,
+            step: err.step || inferFailureStep(err.message),
+        };
+    }
+}
+function inferFailureStep(message) {
+    if (message.includes("config.json") || message.includes("learn")) {
+        return "fetch_sources";
+    }
+    if (message.includes("PutObject")) {
+        return "put_object";
+    }
+    return "build";
+}
+async function invalidatePackageManifestPaths(cf, distributionId, paths) {
+    var _a;
+    if (paths.length === 0) {
+        throw new Error("No paths to invalidate");
+    }
+    const callerReference = `package-manifest-backfill-${Date.now()}`;
+    let lastId = "";
+    for (let index = 0; index < paths.length; index += CLOUDFRONT_INVALIDATION_CHUNK_SIZE) {
+        const chunk = paths.slice(index, index + CLOUDFRONT_INVALIDATION_CHUNK_SIZE);
+        // eslint-disable-next-line no-await-in-loop -- sequential invalidation chunks to avoid rate limits
+        const response = (await withRetry(() => cf.send(new client_cloudfront_1.CreateInvalidationCommand({
+            DistributionId: distributionId,
+            InvalidationBatch: {
+                CallerReference: `${callerReference}-${index}`,
+                Paths: {
+                    Quantity: chunk.length,
+                    Items: chunk,
+                },
+            },
+        }))));
+        lastId = ((_a = response.Invalidation) === null || _a === void 0 ? void 0 : _a.Id) || lastId;
+    }
+    return { id: lastId };
+}
+function loadResumeSlugs(reportPath) {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const fs = require("fs");
+    const raw = fs.readFileSync(reportPath, "utf-8");
+    const report = JSON.parse(raw);
+    const skipStatuses = new Set([
+        "created",
+        "updated",
+        "skipped",
+    ]);
+    return new Set(report.results
+        .filter(result => skipStatuses.has(result.status))
+        .map(result => result.slug));
+}
+function summarizeResults(results) {
+    return {
+        created: results.filter(result => result.status === "created").length,
+        updated: results.filter(result => result.status === "updated").length,
+        skipped: results.filter(result => result.status === "skipped").length,
+        failed: results.filter(result => result.status === "failed").length,
+    };
+}
+async function runBatch(options, clients) {
+    const startedAt = new Date().toISOString();
+    const s3 = (clients === null || clients === void 0 ? void 0 : clients.s3) ||
+        new client_s3_1.S3Client({
+            region: options.region,
+            credentials: {
+                accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
+                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
+            },
+        });
+    const cf = (clients === null || clients === void 0 ? void 0 : clients.cf) ||
+        new client_cloudfront_1.CloudFrontClient({
+            region: "us-east-1",
+            credentials: {
+                accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
+                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
+            },
+        });
+    let slugs = options.slug ?
+        [options.slug] :
+        await listPublishedSlugs(s3, options.bucket);
+    if (options.resumeFrom) {
+        const resumeSlugs = loadResumeSlugs(options.resumeFrom);
+        slugs = slugs.filter(slug => !resumeSlugs.has(slug));
+    }
+    let completed = 0;
+    const results = await mapWithConcurrency(slugs, options.concurrency, async (slug) => {
+        var _a;
+        const result = await processPackage(s3, options.bucket, slug, options);
+        completed += 1;
+        (_a = clients === null || clients === void 0 ? void 0 : clients.onProgress) === null || _a === void 0 ? void 0 : _a.call(clients, completed, slugs.length, result);
+        return result;
+    });
+    const summary = summarizeResults(results);
+    const invalidationPaths = results
+        .filter(result => result.status === "created" || result.status === "updated")
+        .map(result => cloudFrontManifestPath(result.slug));
+    let invalidation;
+    if (options.dryRun) {
+        invalidation = {
+            attempted: false,
+            skippedReason: "dry-run",
+        };
+    }
+    else if (options.noInvalidate) {
+        invalidation = {
+            attempted: false,
+            skippedReason: "no-invalidate",
+        };
+    }
+    else if (invalidationPaths.length === 0) {
+        invalidation = {
+            attempted: false,
+            skippedReason: "no-paths",
+        };
+    }
+    else if (!options.cloudFrontDistributionId) {
+        invalidation = {
+            attempted: true,
+            paths: invalidationPaths,
+            error: "Missing CLOUDFRONT_DISTRIBUTION_ID",
+        };
+    }
+    else {
+        try {
+            const result = await invalidatePackageManifestPaths(cf, options.cloudFrontDistributionId, invalidationPaths);
+            invalidation = {
+                attempted: true,
+                id: result.id,
+                paths: invalidationPaths,
+            };
+        }
+        catch (error) {
+            invalidation = {
+                attempted: true,
+                paths: invalidationPaths,
+                error: error.message,
+            };
+        }
+    }
+    const report = Object.assign(Object.assign({ total: slugs.length }, summary), { startedAt, finishedAt: new Date().toISOString(), options,
+        results,
+        invalidation });
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const fs = require("fs");
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const path = require("path");
+    fs.mkdirSync(path.dirname(options.reportOut), { recursive: true });
+    fs.writeFileSync(options.reportOut, JSON.stringify(report, null, 2), "utf-8");
+    return report;
+}
+function getBatchExitCode(report) {
+    var _a;
+    if (report.failed > 0) {
+        return 1;
+    }
+    if (((_a = report.invalidation) === null || _a === void 0 ? void 0 : _a.attempted) && report.invalidation.error) {
+        return 1;
+    }
+    return 0;
+}

package/lib/utils/titlefy.d.ts

@@ -0,0 +1,2 @@
+export declare function titlefy(str: string): string;
+export declare function formatSidebarTitle(value: string): string | null;

package/lib/utils/titlefy.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.titlefy = titlefy;
+exports.formatSidebarTitle = formatSidebarTitle;
+const SLUG_PATTERN = /^\d+(?:\.\d+)?-/;
+function titlefy(str) {
+    const arr = str.split("-");
+    arr.shift();
+    const result = arr.join(" ");
+    if (!result) {
+        return "";
+    }
+    return result.charAt(0).toUpperCase() + result.slice(1);
+}
+function formatSidebarTitle(value) {
+    const trimmed = value.trim();
+    if (!trimmed) {
+        return null;
+    }
+    if (/^\d+(?:\.\d+)?$/.test(trimmed)) {
+        return null;
+    }
+    const formatted = SLUG_PATTERN.test(trimmed) ? titlefy(trimmed) : trimmed;
+    return formatted.trim() || null;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@learnpack/learnpack",
   "description": "Seamlessly build, sell and/or take interactive & auto-graded tutorials, start learning now or build a new tutorial to your audience.",
-  "version": "5.0.350",
+  "version": "5.0.352",
   "author": "Alejandro Sanchez @alesanchezr",
   "contributors": [
     {
@@ -21,6 +21,8 @@
     "url": "https://github.com/learnpack/learnpack-cli/issues"
   },
   "dependencies": {
+    "@aws-sdk/client-cloudfront": "^3.1092.0",
+    "@aws-sdk/client-s3": "^3.1092.0",
     "@google-cloud/storage": "^7.16.0",
     "@oclif/command": "^1.6.1",
     "@oclif/config": "^1.15.1",
@@ -161,6 +163,7 @@
     ]
   },
   "scripts": {
+    "backfill:package-manifest": "ts-node --project ./tsconfig.json ./scripts/backfill-package-manifest-s3.ts",
     "copy-assets": "npx cpy src/creatorDist/**/* lib/creatorDist --parents --verbose && npx cpy src/utils/templates/**/* lib/utils/templates --parents --verbose && npx cpy src/lua/**/* lib/lua --parents --verbose",
     "tsc": "tsc -b",
     "postpack": "rm -f oclif.manifest.json && eslint . --ext .ts --config .eslintrc",

package/src/commands/serve.ts

@@ -73,6 +73,7 @@ import {
   SyncNotification,
 } from "../models/creator"
 import { exportToScorm, exportToEpub, exportToZip } from "../utils/export"
+import { generateAndPersistPackageManifest } from "../utils/packageManifest"
 import {
   errorHandler,
   notFoundHandler,
@@ -5665,6 +5666,11 @@ class ServeCommand extends SessionCommand {
           })
         )
 
+        await generateAndPersistPackageManifest(bucket, slug, {
+          publishedAt: new Date().toISOString(),
+          localDir: buildRoot,
+        })
+
         // 8) Crear el config.json en buildRoot con lo que retorna buildConfig
         const fullConfig = { config, exercises }
         fullConfig.config.slug = slug
@@ -6500,6 +6506,7 @@ class ServeCommand extends SessionCommand {
       ".learn/config.json",
       ".learn/initialSyllabus.json",
       ".learn/sidebar.json",
+      ".learn/package-manifest.json",
       ".learn/memory_bank.txt",
     ])