@learnpack/learnpack 5.0.351 → 5.0.353

package/lib/utils/s3/packageManifestBackfill.js

@@ -0,0 +1,487 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.streamToString = streamToString;
+exports.withRetry = withRetry;
+exports.mapWithConcurrency = mapWithConcurrency;
+exports.listPublishedSlugs = listPublishedSlugs;
+exports.fetchJsonObject = fetchJsonObject;
+exports.fetchTextObject = fetchTextObject;
+exports.listObjectKeys = listObjectKeys;
+exports.fetchPackageSources = fetchPackageSources;
+exports.buildManifestFromS3Sources = buildManifestFromS3Sources;
+exports.processPackage = processPackage;
+exports.invalidatePackageManifestPaths = invalidatePackageManifestPaths;
+exports.runBatch = runBatch;
+exports.getBatchExitCode = getBatchExitCode;
+const tslib_1 = require("tslib");
+const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const packageManifest_1 = require("../packageManifest");
+const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i;
+const INDEX_HTML_PATTERN = /^[^/]+\/index\.html$/;
+const TRANSIENT_ERROR_CODES = new Set([
+    "SlowDown",
+    "ServiceUnavailable",
+    "RequestTimeout",
+    "Throttling",
+    "ThrottlingException",
+    "TooManyRequestsException",
+]);
+const CLOUDFRONT_INVALIDATION_CHUNK_SIZE = 3000;
+async function streamToString(body) {
+    var _a, e_1, _b, _c;
+    if (!body) {
+        return "";
+    }
+    if (typeof body === "string") {
+        return body;
+    }
+    if (Buffer.isBuffer(body)) {
+        return body.toString("utf-8");
+    }
+    if (body instanceof Uint8Array) {
+        return Buffer.from(body).toString("utf-8");
+    }
+    const chunks = [];
+    try {
+        for (var _d = true, _e = tslib_1.__asyncValues(body), _f; _f = await _e.next(), _a = _f.done, !_a; _d = true) {
+            _c = _f.value;
+            _d = false;
+            const chunk = _c;
+            chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+        }
+    }
+    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+    finally {
+        try {
+            if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
+        }
+        finally { if (e_1) throw e_1.error; }
+    }
+    return Buffer.concat(chunks).toString("utf-8");
+}
+function isTransientError(error) {
+    var _a;
+    const err = error;
+    const code = err.name || err.Code || "";
+    if (TRANSIENT_ERROR_CODES.has(code)) {
+        return true;
+    }
+    const status = (_a = err.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode;
+    return status === 429 || status === 503;
+}
+function sleep(ms) {
+    return new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+}
+async function withRetry(fn, attempts = 3, attempt = 1) {
+    try {
+        return await fn();
+    }
+    catch (error) {
+        if (!isTransientError(error) || attempt === attempts) {
+            throw error;
+        }
+        const multiplier = 2 ** (attempt - 1);
+        const delayMs = 250 * multiplier;
+        await sleep(delayMs);
+        return withRetry(fn, attempts, attempt + 1);
+    }
+}
+async function mapWithConcurrency(items, limit, fn) {
+    const results = Array.from({ length: items.length });
+    let nextIndex = 0;
+    async function worker() {
+        while (nextIndex < items.length) {
+            const currentIndex = nextIndex;
+            nextIndex += 1;
+            // eslint-disable-next-line no-await-in-loop -- each worker processes items sequentially
+            results[currentIndex] = await fn(items[currentIndex]);
+        }
+    }
+    const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+    await Promise.all(workers);
+    return results;
+}
+function manifestKey(slug) {
+    return `${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
+}
+function cloudFrontManifestPath(slug) {
+    return `/${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
+}
+function isNotFoundError(error) {
+    const err = error;
+    const code = err.name || err.Code || "";
+    return code === "NoSuchKey" || code === "NotFound" || code === "NoSuchBucket";
+}
+async function listPublishedSlugs(s3, bucket) {
+    const slugs = new Set();
+    let continuationToken;
+    do {
+        // eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
+        const response = (await withRetry(() => s3.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: bucket,
+            ContinuationToken: continuationToken,
+        }))));
+        for (const item of response.Contents || []) {
+            const key = item.Key;
+            if (key && INDEX_HTML_PATTERN.test(key)) {
+                slugs.add(key.split("/")[0]);
+            }
+        }
+        continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+    return [...slugs].sort();
+}
+async function fetchJsonObject(s3, bucket, key) {
+    try {
+        const response = (await withRetry(() => s3.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))));
+        const content = await streamToString(response.Body);
+        return JSON.parse(content);
+    }
+    catch (error) {
+        if (isNotFoundError(error)) {
+            return null;
+        }
+        throw error;
+    }
+}
+async function fetchTextObject(s3, bucket, key) {
+    try {
+        const response = (await withRetry(() => s3.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))));
+        return await streamToString(response.Body);
+    }
+    catch (error) {
+        if (isNotFoundError(error)) {
+            return null;
+        }
+        throw error;
+    }
+}
+async function listObjectKeys(s3, bucket, prefix) {
+    const keys = [];
+    let continuationToken;
+    do {
+        // eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
+        const response = (await withRetry(() => s3.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: bucket,
+            Prefix: prefix,
+            ContinuationToken: continuationToken,
+        }))));
+        for (const item of response.Contents || []) {
+            if (item.Key) {
+                keys.push(item.Key);
+            }
+        }
+        continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+    return keys;
+}
+function resolveReadmeLang(filename, exercise) {
+    const lower = filename.toLowerCase();
+    for (const [lang, readmeFile] of Object.entries(exercise.translations)) {
+        if (readmeFile.toLowerCase() === lower) {
+            return lang;
+        }
+    }
+    return null;
+}
+function normalizeLearnAndExercises(slug, learnJson, configJson) {
+    const exercises = configJson === null || configJson === void 0 ? void 0 : configJson.exercises;
+    if (!Array.isArray(exercises) || exercises.length === 0) {
+        throw new Error(`Missing or empty exercises array in ${slug}/config.json`);
+    }
+    const resolvedLearn = learnJson ||
+        (configJson === null || configJson === void 0 ? void 0 : configJson.config) ||
+        null;
+    if (!resolvedLearn) {
+        throw new Error(`Missing learn metadata: no ${slug}/learn.json and no config.config`);
+    }
+    return { learnJson: resolvedLearn, exercises };
+}
+async function fetchPackageSources(s3, bucket, slug) {
+    const [learnJsonRaw, configJsonRoot, configJsonLearn, sidebarLearn, sidebarRoot, syllabus,] = await Promise.all([
+        fetchJsonObject(s3, bucket, `${slug}/learn.json`),
+        fetchJsonObject(s3, bucket, `${slug}/config.json`),
+        fetchJsonObject(s3, bucket, `${slug}/.learn/config.json`),
+        fetchJsonObject(s3, bucket, `${slug}/.learn/sidebar.json`),
+        fetchJsonObject(s3, bucket, `${slug}/sidebar.json`),
+        fetchJsonObject(s3, bucket, `${slug}/.learn/initialSyllabus.json`),
+    ]);
+    const configJson = configJsonRoot || configJsonLearn;
+    if (!configJson) {
+        throw new Error(`Missing required file: ${slug}/config.json or ${slug}/.learn/config.json`);
+    }
+    const { learnJson, exercises } = normalizeLearnAndExercises(slug, learnJsonRaw, configJson);
+    const exerciseSlugs = new Set(exercises.map(exercise => exercise.slug));
+    const readmeKeys = (await listObjectKeys(s3, bucket, `${slug}/exercises/`))
+        .filter(key => {
+        const filename = key.split("/").pop() || "";
+        return README_FILENAME_PATTERN.test(filename);
+    })
+        .filter(key => {
+        const parts = key.split("/");
+        const exerciseSlug = parts[2];
+        return Boolean(exerciseSlug && exerciseSlugs.has(exerciseSlug));
+    });
+    const readmeFrontmatters = {};
+    await Promise.all(readmeKeys.map(async (key) => {
+        const parts = key.split("/");
+        const exerciseSlug = parts[2];
+        const filename = parts[parts.length - 1];
+        const exercise = exercises.find(item => item.slug === exerciseSlug);
+        if (!exercise) {
+            return;
+        }
+        const lang = resolveReadmeLang(filename, exercise);
+        if (!lang) {
+            return;
+        }
+        const content = await fetchTextObject(s3, bucket, key);
+        if (!content) {
+            return;
+        }
+        if (!readmeFrontmatters[exerciseSlug]) {
+            readmeFrontmatters[exerciseSlug] = {};
+        }
+        readmeFrontmatters[exerciseSlug][lang] = (0, packageManifest_1.parseReadmeContent)(content);
+    }));
+    const readmeCount = Object.values(readmeFrontmatters).reduce((total, byLang) => total + Object.keys(byLang).length, 0);
+    return {
+        learnJson,
+        exercises,
+        sidebar: sidebarLearn || sidebarRoot || {},
+        syllabus,
+        readmeFrontmatters,
+        meta: {
+            learnJson: Boolean(learnJsonRaw || configJson.config),
+            configJson: true,
+            sidebar: Boolean(sidebarLearn || sidebarRoot),
+            syllabus: Boolean(syllabus),
+            readmes: readmeCount,
+        },
+    };
+}
+function buildManifestFromS3Sources(slug, sources, options) {
+    return (0, packageManifest_1.buildPackageManifestFromSources)({
+        learnJson: sources.learnJson,
+        syllabus: sources.syllabus,
+        sidebar: sources.sidebar,
+        exercises: sources.exercises,
+        readmeFrontmatters: sources.readmeFrontmatters,
+        options: {
+            slug,
+            generatedAt: options.generatedAt || new Date().toISOString(),
+            publishedAt: options.publishedAt,
+        },
+    });
+}
+async function processPackage(s3, bucket, slug, options) {
+    const started = Date.now();
+    try {
+        const existing = await fetchJsonObject(s3, bucket, manifestKey(slug));
+        if (options.skipExisting && existing) {
+            return {
+                slug,
+                status: "skipped",
+                durationMs: Date.now() - started,
+                sources: {
+                    learnJson: false,
+                    configJson: false,
+                    sidebar: false,
+                    syllabus: false,
+                    readmes: 0,
+                },
+            };
+        }
+        const sources = await fetchPackageSources(s3, bucket, slug);
+        const next = buildManifestFromS3Sources(slug, sources, {
+            publishedAt: options.publishedAt,
+        });
+        let status = (0, packageManifest_1.comparePackageManifests)(existing, next);
+        if (options.force && status === "skipped") {
+            status = "updated";
+        }
+        if (!options.dryRun && (status === "created" || status === "updated")) {
+            await withRetry(() => s3.send(new client_s3_1.PutObjectCommand({
+                Bucket: bucket,
+                Key: manifestKey(slug),
+                Body: (0, packageManifest_1.serializePackageManifest)(next),
+                ContentType: "application/json",
+            })));
+        }
+        return {
+            slug,
+            status,
+            durationMs: Date.now() - started,
+            sources: sources.meta,
+        };
+    }
+    catch (error) {
+        const err = error;
+        return {
+            slug,
+            status: "failed",
+            durationMs: Date.now() - started,
+            error: err.message,
+            step: err.step || inferFailureStep(err.message),
+        };
+    }
+}
+function inferFailureStep(message) {
+    if (message.includes("config.json") || message.includes("learn")) {
+        return "fetch_sources";
+    }
+    if (message.includes("PutObject")) {
+        return "put_object";
+    }
+    return "build";
+}
+async function invalidatePackageManifestPaths(cf, distributionId, paths) {
+    var _a;
+    if (paths.length === 0) {
+        throw new Error("No paths to invalidate");
+    }
+    const callerReference = `package-manifest-backfill-${Date.now()}`;
+    let lastId = "";
+    for (let index = 0; index < paths.length; index += CLOUDFRONT_INVALIDATION_CHUNK_SIZE) {
+        const chunk = paths.slice(index, index + CLOUDFRONT_INVALIDATION_CHUNK_SIZE);
+        // eslint-disable-next-line no-await-in-loop -- sequential invalidation chunks to avoid rate limits
+        const response = (await withRetry(() => cf.send(new client_cloudfront_1.CreateInvalidationCommand({
+            DistributionId: distributionId,
+            InvalidationBatch: {
+                CallerReference: `${callerReference}-${index}`,
+                Paths: {
+                    Quantity: chunk.length,
+                    Items: chunk,
+                },
+            },
+        }))));
+        lastId = ((_a = response.Invalidation) === null || _a === void 0 ? void 0 : _a.Id) || lastId;
+    }
+    return { id: lastId };
+}
+function loadResumeSlugs(reportPath) {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const fs = require("fs");
+    const raw = fs.readFileSync(reportPath, "utf-8");
+    const report = JSON.parse(raw);
+    const skipStatuses = new Set([
+        "created",
+        "updated",
+        "skipped",
+    ]);
+    return new Set(report.results
+        .filter(result => skipStatuses.has(result.status))
+        .map(result => result.slug));
+}
+function summarizeResults(results) {
+    return {
+        created: results.filter(result => result.status === "created").length,
+        updated: results.filter(result => result.status === "updated").length,
+        skipped: results.filter(result => result.status === "skipped").length,
+        failed: results.filter(result => result.status === "failed").length,
+    };
+}
+async function runBatch(options, clients) {
+    const startedAt = new Date().toISOString();
+    const s3 = (clients === null || clients === void 0 ? void 0 : clients.s3) ||
+        new client_s3_1.S3Client({
+            region: options.region,
+            credentials: {
+                accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
+                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
+            },
+        });
+    const cf = (clients === null || clients === void 0 ? void 0 : clients.cf) ||
+        new client_cloudfront_1.CloudFrontClient({
+            region: "us-east-1",
+            credentials: {
+                accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
+                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
+            },
+        });
+    let slugs = options.slug ?
+        [options.slug] :
+        await listPublishedSlugs(s3, options.bucket);
+    if (options.resumeFrom) {
+        const resumeSlugs = loadResumeSlugs(options.resumeFrom);
+        slugs = slugs.filter(slug => !resumeSlugs.has(slug));
+    }
+    let completed = 0;
+    const results = await mapWithConcurrency(slugs, options.concurrency, async (slug) => {
+        var _a;
+        const result = await processPackage(s3, options.bucket, slug, options);
+        completed += 1;
+        (_a = clients === null || clients === void 0 ? void 0 : clients.onProgress) === null || _a === void 0 ? void 0 : _a.call(clients, completed, slugs.length, result);
+        return result;
+    });
+    const summary = summarizeResults(results);
+    const invalidationPaths = results
+        .filter(result => result.status === "created" || result.status === "updated")
+        .map(result => cloudFrontManifestPath(result.slug));
+    let invalidation;
+    if (options.dryRun) {
+        invalidation = {
+            attempted: false,
+            skippedReason: "dry-run",
+        };
+    }
+    else if (options.noInvalidate) {
+        invalidation = {
+            attempted: false,
+            skippedReason: "no-invalidate",
+        };
+    }
+    else if (invalidationPaths.length === 0) {
+        invalidation = {
+            attempted: false,
+            skippedReason: "no-paths",
+        };
+    }
+    else if (!options.cloudFrontDistributionId) {
+        invalidation = {
+            attempted: true,
+            paths: invalidationPaths,
+            error: "Missing CLOUDFRONT_DISTRIBUTION_ID",
+        };
+    }
+    else {
+        try {
+            const result = await invalidatePackageManifestPaths(cf, options.cloudFrontDistributionId, invalidationPaths);
+            invalidation = {
+                attempted: true,
+                id: result.id,
+                paths: invalidationPaths,
+            };
+        }
+        catch (error) {
+            invalidation = {
+                attempted: true,
+                paths: invalidationPaths,
+                error: error.message,
+            };
+        }
+    }
+    const report = Object.assign(Object.assign({ total: slugs.length }, summary), { startedAt, finishedAt: new Date().toISOString(), options,
+        results,
+        invalidation });
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const fs = require("fs");
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const path = require("path");
+    fs.mkdirSync(path.dirname(options.reportOut), { recursive: true });
+    fs.writeFileSync(options.reportOut, JSON.stringify(report, null, 2), "utf-8");
+    return report;
+}
+function getBatchExitCode(report) {
+    var _a;
+    if (report.failed > 0) {
+        return 1;
+    }
+    if (((_a = report.invalidation) === null || _a === void 0 ? void 0 : _a.attempted) && report.invalidation.error) {
+        return 1;
+    }
+    return 0;
+}

package/lib/utils/syllabusSync.d.ts

@@ -0,0 +1,71 @@
+import { Bucket } from "@google-cloud/storage";
+import { Syllabus } from "../models/creator";
+/**
+ * Provisional syllabus ↔ bucket reconciliation.
+ *
+ * The `initialSyllabus.json` occasionally drifts from what actually lives in the
+ * bucket (lessons/translations present as files but missing from the syllabus,
+ * or vice versa). Because step descriptions are stored per (step, lang) in the
+ * syllabus, that drift leaves missing slots and causes skipped/spurious
+ * generations. This reconciles the two.
+ *
+ * Extracted from the inline `/actions/synchronize-syllabus` handler so it can be
+ * reused by the sweep/backfill, and generalized over a minimal storage adapter
+ * so it works for both GCS and S3.
+ *
+ * NOTE: this is a temporary patch until the root cause of the drift is fixed.
+ * Track its removal via the `DESCRIPTIONS_RECONCILE_SYLLABUS` flag on callers.
+ */
+export interface SyllabusSyncStorage {
+    /**
+     * Map of `folderSlug -> fileCount` for every folder directly under
+     * `exercises/`. A single authoritative listing: if it throws, callers must
+     * abort any destructive action (never delete on an errored/unknown listing).
+     */
+    listExerciseFolderFileCounts(courseSlug: string): Promise<Map<string, number>>;
+    /** README language codes (lowercased) present per exercise folder slug. */
+    translationLangsBySlug(courseSlug: string): Promise<Map<string, string[]>>;
+}
+export type SyllabusSyncOptions = {
+    /**
+     * When false (default, "additive" mode) only non-destructive passes run:
+     * add-missing, repair-translations, unstick-status. Removal/dedupe are
+     * skipped. When true (manual dev button / root-cause runs) removals apply.
+     */
+    prune?: boolean;
+};
+export type SyllabusSyncCounters = {
+    totalLessons: number;
+    keptLessons: number;
+    removedLessons: number;
+    duplicatesResolved: number;
+    addedLessons: number;
+    fixedLessons: number;
+    repairedTranslationsInLessons: number;
+    repairedTranslationEntries: number;
+};
+type LessonRef = {
+    id: string;
+    title: string;
+    slug: string;
+};
+type LessonRefWithCount = LessonRef & {
+    fileCount: number;
+};
+export type SyllabusSyncResult = {
+    syllabus: Syllabus;
+    changed: boolean;
+    counters: SyllabusSyncCounters;
+    details: {
+        removed: LessonRef[];
+        duplicates: LessonRefWithCount[];
+        kept: Array<LessonRef & {
+            fileCount?: number;
+        }>;
+        added: LessonRef[];
+    };
+};
+export declare function synchronizeSyllabusWithBucket(storage: SyllabusSyncStorage, courseSlug: string, syllabus: Syllabus, options?: SyllabusSyncOptions): Promise<SyllabusSyncResult>;
+/** GCS-backed storage adapter (uses the bucket listing + buildConfig). */
+export declare function createGcsSyllabusSyncStorage(bucket: Bucket): SyllabusSyncStorage;
+export {};