@learnpack/learnpack 5.0.353 → 5.0.355

package/lib/utils/repair/repairStorage.d.ts

@@ -0,0 +1,68 @@
+import { Bucket } from "@google-cloud/storage";
+import { AwsClient } from "../s3/packageManifestBackfill";
+/**
+ * The two buckets a package lives in, behind one interface.
+ *
+ * A legacy package is repaired **twice, independently** — once against the
+ * published snapshot in S3 and once against the draft in GCS — never by copying
+ * one into the other. The two hold different content (the draft keeps moving
+ * after a publication) and the synthesized structure is a projection of that
+ * content: a syllabus derived from the snapshot could name steps the draft does
+ * not have. This is the same rule `mirrorDescriptions` enforces with its content
+ * hash; deriving from each bucket needs no guard at all.
+ *
+ * What crosses between them is only the descriptions, through
+ * `descriptionsS3Backfill --mirror-draft`, which is hash-guarded per step.
+ */
+export type RepairTarget = "s3" | "gcs";
+export declare const ALL_REPAIR_TARGETS: RepairTarget[];
+/**
+ * Where each input lives. The two buckets do not agree on this, and the
+ * disagreement is load-bearing:
+ *
+ * - S3 reads `{slug}/config.json` first (`fetchPackageSources`), which is where
+ *   the publish writes it into the zip.
+ * - GCS reads `courses/{slug}/.learn/config.json` — the publish route downloads
+ *   that one before anything else, so a config written anywhere else in the
+ *   draft would not make the package publishable.
+ */
+export type PackageLayout = {
+    learnJsonKey(slug: string): string;
+    /** Config locations in resolution order; the first is the write target. */
+    configKeys(slug: string): string[];
+    syllabusKey(slug: string): string;
+    /** Sidebar locations in resolution order; the first is the write target. */
+    sidebarKeys(slug: string): string[];
+    exercisesPrefix(slug: string): string;
+    /**
+     * Whether `config.json` is the authority on the exercise list, or the folder
+     * listing is.
+     *
+     * This decides the **language keys** of the synthesized syllabus, and the two
+     * buckets genuinely disagree:
+     *
+     * - S3 reads the published `config.json` (`fetchPackageSources`), where these
+     *   legacy packages carry `us` for the unsuffixed README.
+     * - GCS has no such reader. Both the publish-time manifest and the
+     *   descriptions pipeline go through `configBuilder.buildConfig`, which
+     *   derives exercises from the listing and maps an unsuffixed `README.md` to
+     *   `en` — regardless of what the draft's `.learn/config.json` says.
+     *
+     * A syllabus keyed the other way is not read at all: the manifest would look
+     * up `translations["en"]`, find nothing, and publish `description: null` for
+     * every step. So each side is built the way its own reader will read it.
+     */
+    exercisesFromConfig: boolean;
+};
+export type RepairStorage = {
+    target: RepairTarget;
+    layout: PackageLayout;
+    readJson<T>(key: string): Promise<T | null>;
+    readText(key: string): Promise<string | null>;
+    listKeys(prefix: string): Promise<string[]>;
+    writeJson(key: string, value: unknown): Promise<void>;
+};
+export declare const s3Layout: PackageLayout;
+export declare const gcsLayout: PackageLayout;
+export declare function createS3RepairStorage(s3: AwsClient, bucket: string): RepairStorage;
+export declare function createGcsRepairStorage(bucket: Bucket): RepairStorage;

package/lib/utils/repair/repairStorage.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.gcsLayout = exports.s3Layout = exports.ALL_REPAIR_TARGETS = void 0;
+exports.createS3RepairStorage = createS3RepairStorage;
+exports.createGcsRepairStorage = createGcsRepairStorage;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const packageManifestBackfill_1 = require("../s3/packageManifestBackfill");
+exports.ALL_REPAIR_TARGETS = ["s3", "gcs"];
+exports.s3Layout = {
+    learnJsonKey: slug => `${slug}/learn.json`,
+    configKeys: slug => [`${slug}/config.json`, `${slug}/.learn/config.json`],
+    syllabusKey: slug => `${slug}/.learn/initialSyllabus.json`,
+    // 25 of the published packages keep their sidebar at the root. It has to be
+    // resolved before writing: `fetchPackageSources` prefers the `.learn/` copy,
+    // so a synthesized one would shadow the published titles.
+    sidebarKeys: slug => [
+        `${slug}/.learn/sidebar.json`,
+        `${slug}/sidebar.json`,
+    ],
+    exercisesPrefix: slug => `${slug}/exercises/`,
+    exercisesFromConfig: true,
+};
+exports.gcsLayout = {
+    learnJsonKey: slug => `courses/${slug}/learn.json`,
+    configKeys: slug => [
+        `courses/${slug}/.learn/config.json`,
+        `courses/${slug}/config.json`,
+    ],
+    syllabusKey: slug => `courses/${slug}/.learn/initialSyllabus.json`,
+    sidebarKeys: slug => [`courses/${slug}/.learn/sidebar.json`],
+    exercisesPrefix: slug => `courses/${slug}/exercises/`,
+    exercisesFromConfig: false,
+};
+function createS3RepairStorage(s3, bucket) {
+    return {
+        target: "s3",
+        layout: exports.s3Layout,
+        readJson: key => (0, packageManifestBackfill_1.fetchJsonObject)(s3, bucket, key),
+        readText: key => (0, packageManifestBackfill_1.fetchTextObject)(s3, bucket, key),
+        listKeys: prefix => (0, packageManifestBackfill_1.listObjectKeys)(s3, bucket, prefix),
+        async writeJson(key, value) {
+            await (0, packageManifestBackfill_1.withRetry)(() => s3.send(new client_s3_1.PutObjectCommand({
+                Bucket: bucket,
+                Key: key,
+                Body: JSON.stringify(value, null, 2),
+                ContentType: "application/json",
+            })));
+        },
+    };
+}
+function createGcsRepairStorage(bucket) {
+    const download = async (key) => {
+        try {
+            const [buf] = await bucket.file(key).download();
+            return buf.toString();
+        }
+        catch (_a) {
+            return null;
+        }
+    };
+    return {
+        target: "gcs",
+        layout: exports.gcsLayout,
+        async readJson(key) {
+            const content = await download(key);
+            if (content === null) {
+                return null;
+            }
+            try {
+                return JSON.parse(content);
+            }
+            catch (_a) {
+                return null;
+            }
+        },
+        readText: download,
+        async listKeys(prefix) {
+            const [files] = await bucket.getFiles({ prefix });
+            return files.map(file => file.name);
+        },
+        async writeJson(key, value) {
+            await bucket
+                .file(key)
+                .save(Buffer.from(JSON.stringify(value, null, 2), "utf8"), {
+                contentType: "application/json",
+            });
+        },
+    };
+}

package/lib/utils/s3/packageManifestBackfill.js

@@ -16,6 +16,7 @@ exports.getBatchExitCode = getBatchExitCode;
 const tslib_1 = require("tslib");
 const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
 const client_s3_1 = require("@aws-sdk/client-s3");
+const awsCredentials_1 = require("../awsCredentials");
 const packageManifest_1 = require("../packageManifest");
 const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i;
 const INDEX_HTML_PATTERN = /^[^/]+\/index\.html$/;
@@ -389,18 +390,12 @@ async function runBatch(options, clients) {
     const s3 = (clients === null || clients === void 0 ? void 0 : clients.s3) ||
         new client_s3_1.S3Client({
             region: options.region,
-            credentials: {
-                accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
-                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
-            },
+            credentials: (0, awsCredentials_1.requireAwsCredentials)(),
         });
     const cf = (clients === null || clients === void 0 ? void 0 : clients.cf) ||
         new client_cloudfront_1.CloudFrontClient({
             region: "us-east-1",
-            credentials: {
-                accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
-                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
-            },
+            credentials: (0, awsCredentials_1.requireAwsCredentials)(),
         });
     let slugs = options.slug ?
         [options.slug] :

package/lib/utils/s3/packageSourcesAudit.d.ts

@@ -0,0 +1,75 @@
+import { AwsClient } from "./packageManifestBackfill";
+/**
+ * Inventory of the files a published package needs for its
+ * `package-manifest.json` to be generated at full fidelity.
+ *
+ * This reads **key names only** — one paginated `ListObjectsV2` over the bucket,
+ * zero `GetObject`. Cheap enough to run over the whole catalogue on demand, and
+ * safe at any time: it never writes.
+ *
+ * What it answers: which packages would *fail* the manifest backfill (no
+ * `config.json`, no exercises), and which ones only produce a *degraded*
+ * manifest (no `initialSyllabus.json` → every lesson typed `READ`, ids derived
+ * from the folder name, and descriptions permanently `null`, because the
+ * descriptions pipeline builds its work list from the syllabus keys and cannot
+ * see a course that has none).
+ *
+ * Companion of `legacyPackageRepair.ts`, which writes what this reports missing.
+ */
+export type PackageSourcesAuditEntry = {
+    slug: string;
+    /** A published package is a prefix holding an `index.html` at its root. */
+    hasIndexHtml: boolean;
+    /** Required: without it the manifest backfill reports `failed`. */
+    hasConfigJson: boolean;
+    configJsonKey: string | null;
+    /**
+     * Optional *as a key*: `config.json` may carry the same metadata under
+     * `config`. Only a `GetObject` could tell, so `false` here means "resolves
+     * through the fallback, or not at all" — not "broken".
+     */
+    hasLearnJson: boolean;
+    hasSyllabus: boolean;
+    hasSidebar: boolean;
+    sidebarKey: string | null;
+    hasManifest: boolean;
+    /** Folders directly under `exercises/` holding at least one object. */
+    exerciseFolders: number;
+    /** `README.md` / `README.xx.md` files found in those folders. */
+    readmeCount: number;
+    /** Missing inputs, named as in the docs, for quick reporting. */
+    missing: string[];
+    /** True when the manifest backfill cannot produce a manifest at all. */
+    blocking: boolean;
+};
+export type PackageSourcesAuditSummary = {
+    missingConfigJson: number;
+    missingLearnJson: number;
+    missingSyllabus: number;
+    missingSidebar: number;
+    missingManifest: number;
+    noExerciseFolders: number;
+    blocking: number;
+};
+export type PackageSourcesAuditReport = {
+    generatedAt: string;
+    bucket: string;
+    scannedKeys: number;
+    totalPackages: number;
+    summary: PackageSourcesAuditSummary;
+    packages: PackageSourcesAuditEntry[];
+};
+export type AuditOptions = {
+    /** Restrict the scan to a single package (uses a prefixed listing). */
+    slug?: string;
+    /** Keep prefixes with no `index.html` (drafts, leftovers). Default false. */
+    includeUnpublished?: boolean;
+};
+export declare function summarize(packages: PackageSourcesAuditEntry[]): PackageSourcesAuditSummary;
+/**
+ * One paginated pass over the bucket, folding each page into the accumulator.
+ *
+ * Deliberately not built on `listObjectKeys`: that one materializes every key in
+ * the catalogue before the caller sees the first one.
+ */
+export declare function auditPackageSources(s3: AwsClient, bucket: string, options?: AuditOptions): Promise<PackageSourcesAuditReport>;

package/lib/utils/s3/packageSourcesAudit.js

@@ -0,0 +1,184 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarize = summarize;
+exports.auditPackageSources = auditPackageSources;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const packageManifestBackfill_1 = require("./packageManifestBackfill");
+const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i;
+const ROOT_FILES = new Set([
+    "index.html",
+    "learn.json",
+    "config.json",
+    ".learn/config.json",
+    ".learn/initialSyllabus.json",
+    ".learn/sidebar.json",
+    "sidebar.json",
+    ".learn/package-manifest.json",
+]);
+function emptyAccumulator(slug) {
+    return {
+        slug,
+        hasIndexHtml: false,
+        configJsonKey: null,
+        hasLearnJson: false,
+        hasSyllabus: false,
+        sidebarKey: null,
+        hasManifest: false,
+        exerciseFolders: new Set(),
+        readmeCount: 0,
+    };
+}
+/** Fold one object key into the accumulator map. */
+function classifyKey(entries, key) {
+    const separatorIndex = key.indexOf("/");
+    if (separatorIndex <= 0) {
+        // Object at the bucket root: not part of any package.
+        return;
+    }
+    const slug = key.slice(0, separatorIndex);
+    const rest = key.slice(separatorIndex + 1);
+    if (!rest) {
+        return;
+    }
+    const exerciseParts = rest.startsWith("exercises/") ? rest.split("/") : [];
+    // `exercises/<slug>/<file>`: shorter paths are the folder marker itself.
+    const exerciseSlug = exerciseParts.length >= 3 && exerciseParts[1] ? exerciseParts[1] : null;
+    if (!ROOT_FILES.has(rest) && !exerciseSlug) {
+        return;
+    }
+    let entry = entries.get(slug);
+    if (!entry) {
+        entry = emptyAccumulator(slug);
+        entries.set(slug, entry);
+    }
+    if (exerciseSlug) {
+        entry.exerciseFolders.add(exerciseSlug);
+        const filename = exerciseParts[exerciseParts.length - 1];
+        if (exerciseParts.length === 3 && README_FILENAME_PATTERN.test(filename)) {
+            entry.readmeCount += 1;
+        }
+        return;
+    }
+    switch (rest) {
+        case "index.html":
+            entry.hasIndexHtml = true;
+            break;
+        case "learn.json":
+            entry.hasLearnJson = true;
+            break;
+        // The root copy wins over `.learn/config.json`, matching the order
+        // `fetchPackageSources` resolves them in. Same for the sidebar, inverted.
+        case "config.json":
+            entry.configJsonKey = key;
+            break;
+        case ".learn/config.json":
+            entry.configJsonKey = entry.configJsonKey || key;
+            break;
+        case ".learn/sidebar.json":
+            entry.sidebarKey = key;
+            break;
+        case "sidebar.json":
+            entry.sidebarKey = entry.sidebarKey || key;
+            break;
+        case ".learn/initialSyllabus.json":
+            entry.hasSyllabus = true;
+            break;
+        case ".learn/package-manifest.json":
+            entry.hasManifest = true;
+            break;
+        default:
+            break;
+    }
+}
+function finalizeEntry(entry) {
+    const missing = [];
+    if (!entry.configJsonKey) {
+        missing.push("config.json");
+    }
+    if (!entry.hasLearnJson) {
+        missing.push("learn.json");
+    }
+    if (!entry.hasSyllabus) {
+        missing.push("initialSyllabus.json");
+    }
+    if (!entry.sidebarKey) {
+        missing.push("sidebar.json");
+    }
+    if (entry.exerciseFolders.size === 0) {
+        missing.push("exercises/");
+    }
+    if (!entry.hasManifest) {
+        missing.push("package-manifest.json");
+    }
+    return {
+        slug: entry.slug,
+        hasIndexHtml: entry.hasIndexHtml,
+        hasConfigJson: Boolean(entry.configJsonKey),
+        configJsonKey: entry.configJsonKey,
+        hasLearnJson: entry.hasLearnJson,
+        hasSyllabus: entry.hasSyllabus,
+        hasSidebar: Boolean(entry.sidebarKey),
+        sidebarKey: entry.sidebarKey,
+        hasManifest: entry.hasManifest,
+        exerciseFolders: entry.exerciseFolders.size,
+        readmeCount: entry.readmeCount,
+        missing,
+        // A missing `learn.json` is not blocking on its own: `config.config` can
+        // stand in for it, and only a download would tell.
+        blocking: !entry.configJsonKey || entry.exerciseFolders.size === 0,
+    };
+}
+function count(packages, predicate) {
+    return packages.filter(entry => predicate(entry)).length;
+}
+function summarize(packages) {
+    return {
+        missingConfigJson: count(packages, entry => !entry.hasConfigJson),
+        missingLearnJson: count(packages, entry => !entry.hasLearnJson),
+        missingSyllabus: count(packages, entry => !entry.hasSyllabus),
+        missingSidebar: count(packages, entry => !entry.hasSidebar),
+        missingManifest: count(packages, entry => !entry.hasManifest),
+        noExerciseFolders: count(packages, entry => entry.exerciseFolders === 0),
+        blocking: count(packages, entry => entry.blocking),
+    };
+}
+/**
+ * One paginated pass over the bucket, folding each page into the accumulator.
+ *
+ * Deliberately not built on `listObjectKeys`: that one materializes every key in
+ * the catalogue before the caller sees the first one.
+ */
+async function auditPackageSources(s3, bucket, options = {}) {
+    const entries = new Map();
+    const prefix = options.slug ? `${options.slug}/` : undefined;
+    let continuationToken;
+    let scannedKeys = 0;
+    do {
+        // eslint-disable-next-line no-await-in-loop -- S3 pagination is sequential
+        const response = (await (0, packageManifestBackfill_1.withRetry)(() => s3.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: bucket,
+            Prefix: prefix,
+            ContinuationToken: continuationToken,
+        }))));
+        for (const item of response.Contents || []) {
+            if (!item.Key) {
+                continue;
+            }
+            scannedKeys += 1;
+            classifyKey(entries, item.Key);
+        }
+        continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+    const packages = [...entries.values()]
+        .filter(entry => options.includeUnpublished || entry.hasIndexHtml)
+        .map(entry => finalizeEntry(entry))
+        .sort((a, b) => a.slug.localeCompare(b.slug));
+    return {
+        generatedAt: new Date().toISOString(),
+        bucket,
+        scannedKeys,
+        totalPackages: packages.length,
+        summary: summarize(packages),
+        packages,
+    };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@learnpack/learnpack",
   "description": "Seamlessly build, sell and/or take interactive & auto-graded tutorials, start learning now or build a new tutorial to your audience.",
-  "version": "5.0.353",
+  "version": "5.0.355",
   "author": "Alejandro Sanchez @alesanchezr",
   "contributors": [
     {
@@ -163,7 +163,9 @@
     ]
   },
   "scripts": {
+    "audit:package-sources": "ts-node --project ./tsconfig.json ./scripts/audit-package-sources.ts",
     "backfill:package-manifest": "ts-node --project ./tsconfig.json ./scripts/backfill-package-manifest-s3.ts",
+    "repair:legacy-packages": "ts-node --project ./tsconfig.json ./scripts/repair-legacy-packages.ts",
     "copy-assets": "npx cpy src/creatorDist/**/* lib/creatorDist --parents --verbose && npx cpy src/utils/templates/**/* lib/utils/templates --parents --verbose && npx cpy src/lua/**/* lib/lua --parents --verbose",
     "tsc": "tsc -b",
     "postpack": "rm -f oclif.manifest.json && eslint . --ext .ts --config .eslintrc",

package/src/commands/serve.ts

@@ -6731,7 +6731,7 @@ class ServeCommand extends SessionCommand {
 
             const buffer = Buffer.from(content, "base64")
 
-            if (isImageFile(item.relativePath)) {
+            if (item.isAsset || isImageFile(item.relativePath)) {
               // eslint-disable-next-line no-await-in-loop -- Sequential processing to avoid rate limits
               await uploadBinaryToBucket(
                 bucket,