@learnpack/learnpack 5.0.352 → 5.0.354

package/lib/utils/rigoActions.d.ts

@@ -99,6 +99,50 @@ type TInitialContentGeneratorInputs = {
     topic_description?: string;
 };
 export declare const initialContentGenerator: (token: string, inputs: TInitialContentGeneratorInputs, webhookUrl?: string, endpointSlug?: string) => Promise<any>;
+export type TGenerateStepDescriptionsParams = {
+    /** README of the step, in `sourceLanguage`. Images are stripped before sending. */
+    readmeContent: string;
+    sourceLanguage: string;
+    /** Every language to produce; may or may not include `sourceLanguage`. */
+    outputLanguages: string[];
+    lessonTitle: string;
+    courseTitle?: string;
+    targetWordCount?: number;
+    promptVersion?: number;
+};
+export type TGenerateStepDescriptionsResult = {
+    /**
+     * Descriptions keyed by the `language_code` Rigobot answered with — building
+     * this record IS the reassociation, the response order is not contractual.
+     * A `null` value is the deliberate fallback for content too thin to describe.
+     */
+    descriptionsByLanguage: Record<string, string | null>;
+    /** Requested languages absent from the response (a protocol problem, not a
+     * "no description" answer: callers should retry these rather than storing a
+     * null that would look settled). */
+    missingLanguages: string[];
+    promptVersion: number;
+    /** Seconds the completion took; worth logging to watch the 30s Heroku edge. */
+    durationSeconds: number | null;
+    completionId: number | null;
+};
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+export declare const generateStepDescriptions: (token: string, params: TGenerateStepDescriptionsParams, endpointSlug?: string) => Promise<TGenerateStepDescriptionsResult | null>;
 type TAddInteractivityInputs = {
     components: string;
     prev_lesson: string;

package/lib/utils/rigoActions.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateStepSlug = exports.generateCodeChallenge = exports.addInteractivity = exports.initialContentGenerator = exports.getLanguageCodes = exports.isPackageAuthor = exports.fillSidebarJSON = exports.generateCourseShortName = exports.isValidRigoToken = exports.translateCourseMetadata = exports.createStructuredPreviewReadme = exports.readmeCreator = exports.createCodingReadme = exports.createCodeFile = exports.interactiveCreation = exports.generateCourseIntroduction = exports.translateExercise = exports.generateImage = exports.hasCreatorPermission = exports.createReadme = void 0;
+exports.generateStepSlug = exports.generateCodeChallenge = exports.addInteractivity = exports.generateStepDescriptions = exports.initialContentGenerator = exports.getLanguageCodes = exports.isPackageAuthor = exports.fillSidebarJSON = exports.generateCourseShortName = exports.isValidRigoToken = exports.translateCourseMetadata = exports.createStructuredPreviewReadme = exports.readmeCreator = exports.createCodingReadme = exports.createCodeFile = exports.interactiveCreation = exports.generateCourseIntroduction = exports.translateExercise = exports.generateImage = exports.hasCreatorPermission = exports.createReadme = void 0;
 exports.downloadImage = downloadImage;
 exports.createPreviewReadme = createPreviewReadme;
 exports.makeReadmeReadable = makeReadmeReadable;
@@ -10,6 +10,8 @@ const console_1 = require("../utils/console");
 const fs = require("fs");
 const path = require("path");
 const api_1 = require("./api");
+const descriptionHash_1 = require("./descriptionHash");
+const packageManifest_1 = require("./packageManifest");
 const createReadme = async (token, inputs, purpose, webhookUrl) => {
     try {
         const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/create-sequencial-readme/`, {
@@ -304,6 +306,78 @@ const initialContentGenerator = async (token, inputs, webhookUrl, endpointSlug =
     }
 };
 exports.initialContentGenerator = initialContentGenerator;
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+const generateStepDescriptions = async (token, params, endpointSlug = "generate-step-descriptions") => {
+    var _a, _b, _c, _d, _e;
+    const promptVersion = (_a = params.promptVersion) !== null && _a !== void 0 ? _a : packageManifest_1.DESCRIPTION_PROMPT_VERSION;
+    const targetWordCount = (_b = params.targetWordCount) !== null && _b !== void 0 ? _b : packageManifest_1.DESCRIPTION_TARGET_WORD_COUNT;
+    const inputs = {
+        // Images never help a description and their alt text may be a whole
+        // image-generation prompt; markdown structure is kept, it helps the model.
+        readme_content: (0, descriptionHash_1.stripMarkdownImages)(params.readmeContent),
+        source_language: params.sourceLanguage,
+        output_languages: params.outputLanguages.join(","),
+        lesson_title: params.lessonTitle,
+        course_title: (_c = params.courseTitle) !== null && _c !== void 0 ? _c : "",
+        target_word_count: String(targetWordCount),
+        prompt_version: String(promptVersion),
+    };
+    try {
+        const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/${endpointSlug}/`, { inputs, execute_async: false }, {
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: "Token " + token.trim(),
+            },
+        });
+        const data = response.data;
+        if (!data || data.status === "ERROR") {
+            console.error("Error in generateStepDescriptions:", (data === null || data === void 0 ? void 0 : data.status_text) || "completion returned ERROR");
+            return null;
+        }
+        const entries = Array.isArray((_d = data.parsed) === null || _d === void 0 ? void 0 : _d.descriptions) ?
+            data.parsed.descriptions :
+            [];
+        const descriptionsByLanguage = {};
+        for (const entry of entries) {
+            const code = entry === null || entry === void 0 ? void 0 : entry.language_code;
+            if (typeof code !== "string" || !code) {
+                continue;
+            }
+            descriptionsByLanguage[code] =
+                typeof entry.description === "string" ? entry.description : null;
+        }
+        const echoedVersion = Number.parseInt((_e = data.inputs) === null || _e === void 0 ? void 0 : _e.prompt_version, 10);
+        return {
+            descriptionsByLanguage,
+            missingLanguages: params.outputLanguages.filter(lang => !(lang in descriptionsByLanguage)),
+            promptVersion: Number.isFinite(echoedVersion) ?
+                echoedVersion :
+                promptVersion,
+            durationSeconds: typeof data.duration === "number" ? data.duration : null,
+            completionId: typeof data.id === "number" ? data.id : null,
+        };
+    }
+    catch (error) {
+        console.error("Error in generateStepDescriptions:", error);
+        return null;
+    }
+};
+exports.generateStepDescriptions = generateStepDescriptions;
 const addInteractivity = async (token, inputs, webhookUrl) => {
     try {
         const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/add-lesson-interactivy/`, {

package/lib/utils/s3/packageManifestBackfill.d.ts

@@ -64,6 +64,8 @@ export declare function withRetry<T>(fn: () => Promise<T>, attempts?: number, at
 export declare function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
 export declare function listPublishedSlugs(s3: AwsClient, bucket: string): Promise<string[]>;
 export declare function fetchJsonObject<T>(s3: AwsClient, bucket: string, key: string): Promise<T | null>;
+export declare function fetchTextObject(s3: AwsClient, bucket: string, key: string): Promise<string | null>;
+export declare function listObjectKeys(s3: AwsClient, bucket: string, prefix: string): Promise<string[]>;
 export declare function fetchPackageSources(s3: AwsClient, bucket: string, slug: string): Promise<PackageSources>;
 export declare function buildManifestFromS3Sources(slug: string, sources: PackageSources, options: {
     publishedAt: string | null;

package/lib/utils/s3/packageManifestBackfill.js

@@ -5,6 +5,8 @@ exports.withRetry = withRetry;
 exports.mapWithConcurrency = mapWithConcurrency;
 exports.listPublishedSlugs = listPublishedSlugs;
 exports.fetchJsonObject = fetchJsonObject;
+exports.fetchTextObject = fetchTextObject;
+exports.listObjectKeys = listObjectKeys;
 exports.fetchPackageSources = fetchPackageSources;
 exports.buildManifestFromS3Sources = buildManifestFromS3Sources;
 exports.processPackage = processPackage;
@@ -14,6 +16,7 @@ exports.getBatchExitCode = getBatchExitCode;
 const tslib_1 = require("tslib");
 const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
 const client_s3_1 = require("@aws-sdk/client-s3");
+const awsCredentials_1 = require("../awsCredentials");
 const packageManifest_1 = require("../packageManifest");
 const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i;
 const INDEX_HTML_PATTERN = /^[^/]+\/index\.html$/;
@@ -387,18 +390,12 @@ async function runBatch(options, clients) {
     const s3 = (clients === null || clients === void 0 ? void 0 : clients.s3) ||
         new client_s3_1.S3Client({
             region: options.region,
-            credentials: {
-                accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
-                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
-            },
+            credentials: (0, awsCredentials_1.requireAwsCredentials)(),
         });
     const cf = (clients === null || clients === void 0 ? void 0 : clients.cf) ||
         new client_cloudfront_1.CloudFrontClient({
             region: "us-east-1",
-            credentials: {
-                accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
-                secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
-            },
+            credentials: (0, awsCredentials_1.requireAwsCredentials)(),
         });
     let slugs = options.slug ?
         [options.slug] :

package/lib/utils/s3/packageSourcesAudit.d.ts

@@ -0,0 +1,75 @@
+import { AwsClient } from "./packageManifestBackfill";
+/**
+ * Inventory of the files a published package needs for its
+ * `package-manifest.json` to be generated at full fidelity.
+ *
+ * This reads **key names only** — one paginated `ListObjectsV2` over the bucket,
+ * zero `GetObject`. Cheap enough to run over the whole catalogue on demand, and
+ * safe at any time: it never writes.
+ *
+ * What it answers: which packages would *fail* the manifest backfill (no
+ * `config.json`, no exercises), and which ones only produce a *degraded*
+ * manifest (no `initialSyllabus.json` → every lesson typed `READ`, ids derived
+ * from the folder name, and descriptions permanently `null`, because the
+ * descriptions pipeline builds its work list from the syllabus keys and cannot
+ * see a course that has none).
+ *
+ * Companion of `legacyPackageRepair.ts`, which writes what this reports missing.
+ */
+export type PackageSourcesAuditEntry = {
+    slug: string;
+    /** A published package is a prefix holding an `index.html` at its root. */
+    hasIndexHtml: boolean;
+    /** Required: without it the manifest backfill reports `failed`. */
+    hasConfigJson: boolean;
+    configJsonKey: string | null;
+    /**
+     * Optional *as a key*: `config.json` may carry the same metadata under
+     * `config`. Only a `GetObject` could tell, so `false` here means "resolves
+     * through the fallback, or not at all" — not "broken".
+     */
+    hasLearnJson: boolean;
+    hasSyllabus: boolean;
+    hasSidebar: boolean;
+    sidebarKey: string | null;
+    hasManifest: boolean;
+    /** Folders directly under `exercises/` holding at least one object. */
+    exerciseFolders: number;
+    /** `README.md` / `README.xx.md` files found in those folders. */
+    readmeCount: number;
+    /** Missing inputs, named as in the docs, for quick reporting. */
+    missing: string[];
+    /** True when the manifest backfill cannot produce a manifest at all. */
+    blocking: boolean;
+};
+export type PackageSourcesAuditSummary = {
+    missingConfigJson: number;
+    missingLearnJson: number;
+    missingSyllabus: number;
+    missingSidebar: number;
+    missingManifest: number;
+    noExerciseFolders: number;
+    blocking: number;
+};
+export type PackageSourcesAuditReport = {
+    generatedAt: string;
+    bucket: string;
+    scannedKeys: number;
+    totalPackages: number;
+    summary: PackageSourcesAuditSummary;
+    packages: PackageSourcesAuditEntry[];
+};
+export type AuditOptions = {
+    /** Restrict the scan to a single package (uses a prefixed listing). */
+    slug?: string;
+    /** Keep prefixes with no `index.html` (drafts, leftovers). Default false. */
+    includeUnpublished?: boolean;
+};
+export declare function summarize(packages: PackageSourcesAuditEntry[]): PackageSourcesAuditSummary;
+/**
+ * One paginated pass over the bucket, folding each page into the accumulator.
+ *
+ * Deliberately not built on `listObjectKeys`: that one materializes every key in
+ * the catalogue before the caller sees the first one.
+ */
+export declare function auditPackageSources(s3: AwsClient, bucket: string, options?: AuditOptions): Promise<PackageSourcesAuditReport>;

package/lib/utils/s3/packageSourcesAudit.js

@@ -0,0 +1,184 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarize = summarize;
+exports.auditPackageSources = auditPackageSources;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const packageManifestBackfill_1 = require("./packageManifestBackfill");
+const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i;
+const ROOT_FILES = new Set([
+    "index.html",
+    "learn.json",
+    "config.json",
+    ".learn/config.json",
+    ".learn/initialSyllabus.json",
+    ".learn/sidebar.json",
+    "sidebar.json",
+    ".learn/package-manifest.json",
+]);
+function emptyAccumulator(slug) {
+    return {
+        slug,
+        hasIndexHtml: false,
+        configJsonKey: null,
+        hasLearnJson: false,
+        hasSyllabus: false,
+        sidebarKey: null,
+        hasManifest: false,
+        exerciseFolders: new Set(),
+        readmeCount: 0,
+    };
+}
+/** Fold one object key into the accumulator map. */
+function classifyKey(entries, key) {
+    const separatorIndex = key.indexOf("/");
+    if (separatorIndex <= 0) {
+        // Object at the bucket root: not part of any package.
+        return;
+    }
+    const slug = key.slice(0, separatorIndex);
+    const rest = key.slice(separatorIndex + 1);
+    if (!rest) {
+        return;
+    }
+    const exerciseParts = rest.startsWith("exercises/") ? rest.split("/") : [];
+    // `exercises/<slug>/<file>`: shorter paths are the folder marker itself.
+    const exerciseSlug = exerciseParts.length >= 3 && exerciseParts[1] ? exerciseParts[1] : null;
+    if (!ROOT_FILES.has(rest) && !exerciseSlug) {
+        return;
+    }
+    let entry = entries.get(slug);
+    if (!entry) {
+        entry = emptyAccumulator(slug);
+        entries.set(slug, entry);
+    }
+    if (exerciseSlug) {
+        entry.exerciseFolders.add(exerciseSlug);
+        const filename = exerciseParts[exerciseParts.length - 1];
+        if (exerciseParts.length === 3 && README_FILENAME_PATTERN.test(filename)) {
+            entry.readmeCount += 1;
+        }
+        return;
+    }
+    switch (rest) {
+        case "index.html":
+            entry.hasIndexHtml = true;
+            break;
+        case "learn.json":
+            entry.hasLearnJson = true;
+            break;
+        // The root copy wins over `.learn/config.json`, matching the order
+        // `fetchPackageSources` resolves them in. Same for the sidebar, inverted.
+        case "config.json":
+            entry.configJsonKey = key;
+            break;
+        case ".learn/config.json":
+            entry.configJsonKey = entry.configJsonKey || key;
+            break;
+        case ".learn/sidebar.json":
+            entry.sidebarKey = key;
+            break;
+        case "sidebar.json":
+            entry.sidebarKey = entry.sidebarKey || key;
+            break;
+        case ".learn/initialSyllabus.json":
+            entry.hasSyllabus = true;
+            break;
+        case ".learn/package-manifest.json":
+            entry.hasManifest = true;
+            break;
+        default:
+            break;
+    }
+}
+function finalizeEntry(entry) {
+    const missing = [];
+    if (!entry.configJsonKey) {
+        missing.push("config.json");
+    }
+    if (!entry.hasLearnJson) {
+        missing.push("learn.json");
+    }
+    if (!entry.hasSyllabus) {
+        missing.push("initialSyllabus.json");
+    }
+    if (!entry.sidebarKey) {
+        missing.push("sidebar.json");
+    }
+    if (entry.exerciseFolders.size === 0) {
+        missing.push("exercises/");
+    }
+    if (!entry.hasManifest) {
+        missing.push("package-manifest.json");
+    }
+    return {
+        slug: entry.slug,
+        hasIndexHtml: entry.hasIndexHtml,
+        hasConfigJson: Boolean(entry.configJsonKey),
+        configJsonKey: entry.configJsonKey,
+        hasLearnJson: entry.hasLearnJson,
+        hasSyllabus: entry.hasSyllabus,
+        hasSidebar: Boolean(entry.sidebarKey),
+        sidebarKey: entry.sidebarKey,
+        hasManifest: entry.hasManifest,
+        exerciseFolders: entry.exerciseFolders.size,
+        readmeCount: entry.readmeCount,
+        missing,
+        // A missing `learn.json` is not blocking on its own: `config.config` can
+        // stand in for it, and only a download would tell.
+        blocking: !entry.configJsonKey || entry.exerciseFolders.size === 0,
+    };
+}
+function count(packages, predicate) {
+    return packages.filter(entry => predicate(entry)).length;
+}
+function summarize(packages) {
+    return {
+        missingConfigJson: count(packages, entry => !entry.hasConfigJson),
+        missingLearnJson: count(packages, entry => !entry.hasLearnJson),
+        missingSyllabus: count(packages, entry => !entry.hasSyllabus),
+        missingSidebar: count(packages, entry => !entry.hasSidebar),
+        missingManifest: count(packages, entry => !entry.hasManifest),
+        noExerciseFolders: count(packages, entry => entry.exerciseFolders === 0),
+        blocking: count(packages, entry => entry.blocking),
+    };
+}
+/**
+ * One paginated pass over the bucket, folding each page into the accumulator.
+ *
+ * Deliberately not built on `listObjectKeys`: that one materializes every key in
+ * the catalogue before the caller sees the first one.
+ */
+async function auditPackageSources(s3, bucket, options = {}) {
+    const entries = new Map();
+    const prefix = options.slug ? `${options.slug}/` : undefined;
+    let continuationToken;
+    let scannedKeys = 0;
+    do {
+        // eslint-disable-next-line no-await-in-loop -- S3 pagination is sequential
+        const response = (await (0, packageManifestBackfill_1.withRetry)(() => s3.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: bucket,
+            Prefix: prefix,
+            ContinuationToken: continuationToken,
+        }))));
+        for (const item of response.Contents || []) {
+            if (!item.Key) {
+                continue;
+            }
+            scannedKeys += 1;
+            classifyKey(entries, item.Key);
+        }
+        continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+    const packages = [...entries.values()]
+        .filter(entry => options.includeUnpublished || entry.hasIndexHtml)
+        .map(entry => finalizeEntry(entry))
+        .sort((a, b) => a.slug.localeCompare(b.slug));
+    return {
+        generatedAt: new Date().toISOString(),
+        bucket,
+        scannedKeys,
+        totalPackages: packages.length,
+        summary: summarize(packages),
+        packages,
+    };
+}

package/lib/utils/syllabusSync.d.ts

@@ -0,0 +1,71 @@
+import { Bucket } from "@google-cloud/storage";
+import { Syllabus } from "../models/creator";
+/**
+ * Provisional syllabus ↔ bucket reconciliation.
+ *
+ * The `initialSyllabus.json` occasionally drifts from what actually lives in the
+ * bucket (lessons/translations present as files but missing from the syllabus,
+ * or vice versa). Because step descriptions are stored per (step, lang) in the
+ * syllabus, that drift leaves missing slots and causes skipped/spurious
+ * generations. This reconciles the two.
+ *
+ * Extracted from the inline `/actions/synchronize-syllabus` handler so it can be
+ * reused by the sweep/backfill, and generalized over a minimal storage adapter
+ * so it works for both GCS and S3.
+ *
+ * NOTE: this is a temporary patch until the root cause of the drift is fixed.
+ * Track its removal via the `DESCRIPTIONS_RECONCILE_SYLLABUS` flag on callers.
+ */
+export interface SyllabusSyncStorage {
+    /**
+     * Map of `folderSlug -> fileCount` for every folder directly under
+     * `exercises/`. A single authoritative listing: if it throws, callers must
+     * abort any destructive action (never delete on an errored/unknown listing).
+     */
+    listExerciseFolderFileCounts(courseSlug: string): Promise<Map<string, number>>;
+    /** README language codes (lowercased) present per exercise folder slug. */
+    translationLangsBySlug(courseSlug: string): Promise<Map<string, string[]>>;
+}
+export type SyllabusSyncOptions = {
+    /**
+     * When false (default, "additive" mode) only non-destructive passes run:
+     * add-missing, repair-translations, unstick-status. Removal/dedupe are
+     * skipped. When true (manual dev button / root-cause runs) removals apply.
+     */
+    prune?: boolean;
+};
+export type SyllabusSyncCounters = {
+    totalLessons: number;
+    keptLessons: number;
+    removedLessons: number;
+    duplicatesResolved: number;
+    addedLessons: number;
+    fixedLessons: number;
+    repairedTranslationsInLessons: number;
+    repairedTranslationEntries: number;
+};
+type LessonRef = {
+    id: string;
+    title: string;
+    slug: string;
+};
+type LessonRefWithCount = LessonRef & {
+    fileCount: number;
+};
+export type SyllabusSyncResult = {
+    syllabus: Syllabus;
+    changed: boolean;
+    counters: SyllabusSyncCounters;
+    details: {
+        removed: LessonRef[];
+        duplicates: LessonRefWithCount[];
+        kept: Array<LessonRef & {
+            fileCount?: number;
+        }>;
+        added: LessonRef[];
+    };
+};
+export declare function synchronizeSyllabusWithBucket(storage: SyllabusSyncStorage, courseSlug: string, syllabus: Syllabus, options?: SyllabusSyncOptions): Promise<SyllabusSyncResult>;
+/** GCS-backed storage adapter (uses the bucket listing + buildConfig). */
+export declare function createGcsSyllabusSyncStorage(bucket: Bucket): SyllabusSyncStorage;
+export {};