@learnpack/learnpack 5.0.351 → 5.0.353

package/lib/utils/descriptions/generateCourseDescriptions.d.ts

@@ -0,0 +1,59 @@
+import { Syllabus } from "../../models/creator";
+import { PackageManifest } from "../packageManifest";
+import { TGenerateStepDescriptionsParams, TGenerateStepDescriptionsResult } from "../rigoActions";
+import { SyllabusSyncStorage } from "../syllabusSync";
+import { CourseExercise, CourseReadmes } from "./workList";
+/**
+ * Generate and persist the step descriptions of one published course.
+ *
+ * This is the single implementation behind every flow that produces
+ * descriptions: the background stage that runs after publishing, the sweep that
+ * recovers interrupted publications, and the one-off backfill over the
+ * historical catalogue. They differ only in what triggers them.
+ *
+ * Storage is a port so the service can be driven against the published bucket
+ * (the real case) or an in-memory double (tests), and so the manifest
+ * re-projection stays where the storage-specific knowledge lives.
+ */
+export declare const DEFAULT_DESCRIPTIONS_CONCURRENCY = 5;
+export interface CourseDescriptionsStorage {
+    readSyllabus(courseSlug: string): Promise<Syllabus | null>;
+    writeSyllabus(courseSlug: string, syllabus: Syllabus): Promise<void>;
+    listExercises(courseSlug: string): Promise<CourseExercise[]>;
+    readReadme(courseSlug: string, exerciseSlug: string, fileName: string): Promise<string | null>;
+    /** Re-generate `package-manifest.json` from the syllabus and invalidate the CDN. */
+    reprojectManifest?(courseSlug: string): Promise<void>;
+    /** Read the stored manifest, to report the updated one in the event. */
+    readManifest?(courseSlug: string): Promise<PackageManifest | null>;
+    /** Enables the provisional additive syllabus reconciliation. */
+    syllabusSyncStorage?: SyllabusSyncStorage;
+}
+export type GenerateCourseDescriptionsOptions = {
+    token: string;
+    dryRun?: boolean;
+    force?: boolean;
+    concurrency?: number;
+    targetWordCount?: number;
+    promptVersion?: number;
+    /** Additive syllabus reconciliation before computing the work list. */
+    reconcile?: boolean;
+    /** Injection point for tests. */
+    generate?: (token: string, params: TGenerateStepDescriptionsParams) => Promise<TGenerateStepDescriptionsResult | null>;
+};
+export type GenerateCourseDescriptionsResult = {
+    /** `skipped` means there was nothing to do, not that something went wrong. */
+    status: "skipped" | "processed" | "failed";
+    steps: number;
+    /** (step, language) pairs written. */
+    generated: number;
+    /** Steps whose completion failed outright. */
+    failed: number;
+    /** (step, language) pairs the model never answered; retried later. */
+    missing: number;
+    errors: string[];
+    /** Completion durations, to watch how close we run to the request limits. */
+    durationsSeconds: number[];
+};
+/** Read every README of the course, keyed by exercise slug and language. */
+export declare function readCourseReadmes(storage: CourseDescriptionsStorage, courseSlug: string, exercises: CourseExercise[], concurrency: number): Promise<CourseReadmes>;
+export declare function generateCourseDescriptions(storage: CourseDescriptionsStorage, courseSlug: string, options: GenerateCourseDescriptionsOptions): Promise<GenerateCourseDescriptionsResult>;

package/lib/utils/descriptions/generateCourseDescriptions.js

@@ -0,0 +1,173 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_DESCRIPTIONS_CONCURRENCY = void 0;
+exports.readCourseReadmes = readCourseReadmes;
+exports.generateCourseDescriptions = generateCourseDescriptions;
+const rigoActions_1 = require("../rigoActions");
+const packageManifestBackfill_1 = require("../s3/packageManifestBackfill");
+const syllabusSync_1 = require("../syllabusSync");
+const workList_1 = require("./workList");
+/**
+ * Generate and persist the step descriptions of one published course.
+ *
+ * This is the single implementation behind every flow that produces
+ * descriptions: the background stage that runs after publishing, the sweep that
+ * recovers interrupted publications, and the one-off backfill over the
+ * historical catalogue. They differ only in what triggers them.
+ *
+ * Storage is a port so the service can be driven against the published bucket
+ * (the real case) or an in-memory double (tests), and so the manifest
+ * re-projection stays where the storage-specific knowledge lives.
+ */
+exports.DEFAULT_DESCRIPTIONS_CONCURRENCY = 5;
+function emptyResult(status) {
+    return {
+        status,
+        steps: 0,
+        generated: 0,
+        failed: 0,
+        missing: 0,
+        errors: [],
+        durationsSeconds: [],
+    };
+}
+/** Read every README of the course, keyed by exercise slug and language. */
+async function readCourseReadmes(storage, courseSlug, exercises, concurrency) {
+    const readmes = {};
+    const targets = [];
+    for (const exercise of exercises) {
+        for (const [lang, file] of Object.entries(exercise.translations || {})) {
+            targets.push({ exerciseSlug: exercise.slug, lang, file });
+        }
+    }
+    const contents = await (0, packageManifestBackfill_1.mapWithConcurrency)(targets, concurrency, target => storage.readReadme(courseSlug, target.exerciseSlug, target.file));
+    for (const [index, target] of targets.entries()) {
+        const content = contents[index];
+        if (typeof content !== "string") {
+            continue;
+        }
+        if (!readmes[target.exerciseSlug]) {
+            readmes[target.exerciseSlug] = {};
+        }
+        readmes[target.exerciseSlug][target.lang] = content;
+    }
+    return readmes;
+}
+/** Ensure the translation slot exists before writing a description into it. */
+function ensureSlot(lesson, lang) {
+    if (!lesson.translations) {
+        lesson.translations = {};
+    }
+    if (!lesson.translations[lang]) {
+        lesson.translations[lang] = { completionId: 0, startedAt: Date.now() };
+    }
+    return lesson.translations[lang];
+}
+function applyStepResult(step, result) {
+    let generated = 0;
+    for (const language of step.languages) {
+        if (!(language.lang in result.descriptionsByLanguage)) {
+            // Never answered: leave the slot untouched so a later run retries it.
+            // Writing a null plus a fingerprint here would look settled forever.
+            continue;
+        }
+        const slot = ensureSlot(step.lesson, language.lang);
+        if (slot.descriptionSource === "human") {
+            continue;
+        }
+        const description = result.descriptionsByLanguage[language.lang];
+        slot.description = description;
+        slot.descriptionStatus = description ? "generated" : "none";
+        slot.descriptionSource = "auto";
+        slot.descriptionPromptVersion = result.promptVersion;
+        slot.sourceContentHash = language.fingerprint.sha256;
+        slot.sourceSimHash = language.fingerprint.simhash;
+        generated += 1;
+    }
+    return { generated, missing: result.missingLanguages.length };
+}
+async function generateCourseDescriptions(storage, courseSlug, options) {
+    var _a, _b;
+    const concurrency = (_a = options.concurrency) !== null && _a !== void 0 ? _a : exports.DEFAULT_DESCRIPTIONS_CONCURRENCY;
+    const generate = (_b = options.generate) !== null && _b !== void 0 ? _b : rigoActions_1.generateStepDescriptions;
+    const syllabus = await storage.readSyllabus(courseSlug);
+    if (!syllabus) {
+        return emptyResult("skipped");
+    }
+    const exercises = await storage.listExercises(courseSlug);
+    if (exercises.length === 0) {
+        return emptyResult("skipped");
+    }
+    if (options.reconcile && storage.syllabusSyncStorage) {
+        try {
+            await (0, syllabusSync_1.synchronizeSyllabusWithBucket)(storage.syllabusSyncStorage, courseSlug, syllabus, { prune: false });
+        }
+        catch (error) {
+            // Provisional patch: never let reconciliation block generation.
+            console.error(`[descriptions] Reconciliation failed for "${courseSlug}":`, error.message);
+        }
+    }
+    const readmes = await readCourseReadmes(storage, courseSlug, exercises, concurrency);
+    const work = (0, workList_1.buildDescriptionWorkList)({
+        syllabus,
+        exercises,
+        readmes,
+        promptVersion: options.promptVersion,
+        force: options.force,
+    });
+    if (work.length === 0) {
+        return emptyResult("skipped");
+    }
+    const result = emptyResult("processed");
+    result.steps = work.length;
+    if (options.dryRun) {
+        result.generated = (0, workList_1.countWorkItems)(work);
+        return result;
+    }
+    // One completion per step, several in flight: a slow step never stalls the
+    // course and Rigobot is never hit with an unbounded burst.
+    const outcomes = await (0, packageManifestBackfill_1.mapWithConcurrency)(work, concurrency, async (step) => {
+        var _a;
+        return ({
+            step,
+            generated: await generate(options.token, {
+                readmeContent: step.baseContent,
+                sourceLanguage: step.baseLanguage,
+                outputLanguages: step.languages.map(language => language.lang),
+                lessonTitle: step.lesson.title,
+                courseTitle: (_a = syllabus.courseInfo) === null || _a === void 0 ? void 0 : _a.title,
+                targetWordCount: options.targetWordCount,
+                promptVersion: options.promptVersion,
+            }),
+        });
+    });
+    for (const outcome of outcomes) {
+        if (!outcome.generated) {
+            result.failed += 1;
+            result.errors.push(`generation failed for "${outcome.step.exerciseSlug}"`);
+            continue;
+        }
+        if (outcome.generated.durationSeconds !== null) {
+            result.durationsSeconds.push(outcome.generated.durationSeconds);
+        }
+        const applied = applyStepResult(outcome.step, outcome.generated);
+        result.generated += applied.generated;
+        result.missing += applied.missing;
+    }
+    if (result.generated === 0 && result.failed > 0) {
+        result.status = "failed";
+        return result;
+    }
+    await storage.writeSyllabus(courseSlug, syllabus);
+    if (storage.reprojectManifest) {
+        try {
+            await storage.reprojectManifest(courseSlug);
+        }
+        catch (error) {
+            // The syllabus (the source of truth) is already saved; a failed
+            // projection is recoverable and must not lose the generated text.
+            result.errors.push(`manifest projection failed: ${error.message}`);
+        }
+    }
+    return result;
+}

package/lib/utils/descriptions/mirrorDescriptions.d.ts

@@ -0,0 +1,49 @@
+import { Syllabus } from "../../models/creator";
+import { CourseDescriptionsStorage } from "./generateCourseDescriptions";
+import { CourseExercise, CourseReadmes } from "./workList";
+/**
+ * Copy descriptions from the published syllabus back into the draft one.
+ *
+ * Descriptions are generated against the published package because it is an
+ * immutable snapshot. The draft in GCS, meanwhile, keeps moving: between the
+ * publication and the moment the descriptions land, steps may have been added,
+ * removed, renamed, reordered or edited.
+ *
+ * The guard against that drift is the content hash, not a timestamp or a
+ * position: a description is copied only when the draft README hashes to the
+ * very content the description was generated from. Anything that moved simply
+ * misses and waits for the next publication — a stale description is never
+ * written onto content it does not describe.
+ *
+ * Without this mirror the draft would never hold a description, and every
+ * publication would regenerate the whole course from scratch: the freshness
+ * check reads the fingerprints stored in the syllabus that travels inside the
+ * zip.
+ */
+export type MirrorResult = {
+    /** (step, language) pairs copied into the draft. */
+    copied: number;
+    /** Pairs with no reusable description: content moved on, or none published. */
+    missed: number;
+    /** Pairs the draft already had up to date. */
+    fresh: number;
+};
+export type MirrorOptions = {
+    promptVersion?: number;
+    dryRun?: boolean;
+    concurrency?: number;
+};
+/**
+ * Pure core: writes into `targetSyllabus` in place. Callers own the I/O, which
+ * keeps the drift rules testable without a bucket.
+ */
+export declare function mirrorDescriptionsIntoSyllabus(input: {
+    sourceSyllabus: Syllabus | null;
+    targetSyllabus: Syllabus;
+    exercises: CourseExercise[];
+    /** READMEs of the TARGET (draft), which is what must match the hash. */
+    readmes: CourseReadmes;
+    promptVersion?: number;
+}): MirrorResult;
+/** Read the draft, mirror into it, and save it when something changed. */
+export declare function mirrorDescriptionsToDraft(storage: CourseDescriptionsStorage, courseSlug: string, sourceSyllabus: Syllabus | null, options?: MirrorOptions): Promise<MirrorResult>;

package/lib/utils/descriptions/mirrorDescriptions.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mirrorDescriptionsIntoSyllabus = mirrorDescriptionsIntoSyllabus;
+exports.mirrorDescriptionsToDraft = mirrorDescriptionsToDraft;
+const descriptionHash_1 = require("../descriptionHash");
+const packageManifest_1 = require("../packageManifest");
+const generateCourseDescriptions_1 = require("./generateCourseDescriptions");
+const workList_1 = require("./workList");
+function emptyMirrorResult() {
+    return { copied: 0, missed: 0, fresh: 0 };
+}
+/**
+ * Pure core: writes into `targetSyllabus` in place. Callers own the I/O, which
+ * keeps the drift rules testable without a bucket.
+ */
+function mirrorDescriptionsIntoSyllabus(input) {
+    var _a, _b, _c, _d, _e, _f, _g;
+    const result = emptyMirrorResult();
+    if (!input.sourceSyllabus) {
+        return result;
+    }
+    const promptVersion = (_a = input.promptVersion) !== null && _a !== void 0 ? _a : packageManifest_1.DESCRIPTION_PROMPT_VERSION;
+    for (const exercise of input.exercises) {
+        const targetLesson = (0, workList_1.findSyllabusLesson)(input.targetSyllabus, exercise.slug);
+        if (!targetLesson) {
+            // The step is not in the draft (removed, or renamed after publishing).
+            continue;
+        }
+        const sourceLesson = (0, workList_1.findSyllabusLesson)(input.sourceSyllabus, exercise.slug);
+        const exerciseReadmes = input.readmes[exercise.slug] || {};
+        for (const lang of Object.keys(exercise.translations || {})) {
+            const content = exerciseReadmes[lang];
+            if (typeof content !== "string") {
+                continue;
+            }
+            const fingerprint = (0, descriptionHash_1.fingerprintReadme)(content);
+            const targetSlot = (_b = targetLesson.translations) === null || _b === void 0 ? void 0 : _b[lang];
+            if ((targetSlot === null || targetSlot === void 0 ? void 0 : targetSlot.descriptionSource) === "human") {
+                result.fresh += 1;
+                continue;
+            }
+            if ((targetSlot === null || targetSlot === void 0 ? void 0 : targetSlot.sourceContentHash) === fingerprint.sha256 &&
+                ((_c = targetSlot === null || targetSlot === void 0 ? void 0 : targetSlot.descriptionPromptVersion) !== null && _c !== void 0 ? _c : 0) >= promptVersion) {
+                result.fresh += 1;
+                continue;
+            }
+            const sourceSlot = (_d = sourceLesson === null || sourceLesson === void 0 ? void 0 : sourceLesson.translations) === null || _d === void 0 ? void 0 : _d[lang];
+            // The hash guard: only reuse a description generated from exactly this
+            // content. Note a published `description: null` is reusable too — it is a
+            // real answer ("not enough content to describe"), not a missing one.
+            const reusable = sourceSlot !== undefined &&
+                sourceSlot.sourceContentHash === fingerprint.sha256 &&
+                ((_e = sourceSlot.descriptionPromptVersion) !== null && _e !== void 0 ? _e : 0) >= promptVersion &&
+                sourceSlot.descriptionStatus !== "error";
+            if (!reusable) {
+                result.missed += 1;
+                continue;
+            }
+            if (!targetLesson.translations) {
+                targetLesson.translations = {};
+            }
+            if (!targetLesson.translations[lang]) {
+                targetLesson.translations[lang] = {
+                    completionId: 0,
+                    startedAt: Date.now(),
+                };
+            }
+            const destination = targetLesson.translations[lang];
+            destination.description = (_f = sourceSlot.description) !== null && _f !== void 0 ? _f : null;
+            destination.descriptionStatus = sourceSlot.description ?
+                "generated" :
+                "none";
+            destination.descriptionSource = "auto";
+            destination.descriptionPromptVersion =
+                (_g = sourceSlot.descriptionPromptVersion) !== null && _g !== void 0 ? _g : promptVersion;
+            destination.sourceContentHash = fingerprint.sha256;
+            destination.sourceSimHash = fingerprint.simhash;
+            result.copied += 1;
+        }
+    }
+    return result;
+}
+/** Read the draft, mirror into it, and save it when something changed. */
+async function mirrorDescriptionsToDraft(storage, courseSlug, sourceSyllabus, options = {}) {
+    var _a;
+    if (!sourceSyllabus) {
+        return emptyMirrorResult();
+    }
+    const targetSyllabus = await storage.readSyllabus(courseSlug);
+    if (!targetSyllabus) {
+        return emptyMirrorResult();
+    }
+    const exercises = await storage.listExercises(courseSlug);
+    if (exercises.length === 0) {
+        return emptyMirrorResult();
+    }
+    const readmes = await (0, generateCourseDescriptions_1.readCourseReadmes)(storage, courseSlug, exercises, (_a = options.concurrency) !== null && _a !== void 0 ? _a : generateCourseDescriptions_1.DEFAULT_DESCRIPTIONS_CONCURRENCY);
+    const result = mirrorDescriptionsIntoSyllabus({
+        sourceSyllabus,
+        targetSyllabus,
+        exercises,
+        readmes,
+        promptVersion: options.promptVersion,
+    });
+    if (result.copied > 0 && !options.dryRun) {
+        await storage.writeSyllabus(courseSlug, targetSyllabus);
+    }
+    return result;
+}

package/lib/utils/descriptions/publishStage.d.ts

@@ -0,0 +1,69 @@
+import { Bucket } from "@google-cloud/storage";
+import { JournalStorage, PublishJournal } from "../publishJournal";
+import { CourseDescriptionsStorage, GenerateCourseDescriptionsOptions, GenerateCourseDescriptionsResult } from "./generateCourseDescriptions";
+import { MirrorResult } from "./mirrorDescriptions";
+import { PublishEventContext, sendPublishEvent } from "../publishEvents";
+import { CourseExercise } from "./workList";
+/**
+ * Stage B of a publication: generate the descriptions the new content needs.
+ *
+ * It runs in the background, after the HTTP response, because a single
+ * completion takes ~15s and a course may need several — far past any request
+ * limit. The publication itself is already finished and durable by then; this
+ * stage only enriches the published package.
+ */
+/**
+ * Whether the freshly built package needs any description, decided from the
+ * build directory that is still on disk after zipping.
+ *
+ * Reading from disk (instead of re-reading GCS or S3) is free and, at this
+ * exact moment, all three hold the same content: the answer is what the publish
+ * response promises to breathecode, so it must be cheap enough to compute
+ * before responding.
+ */
+/** Reads a path relative to the build root; null when it does not exist. */
+export type BuildFileReader = (relativePath: string) => string | null;
+export declare function createBuildFileReader(buildRoot: string): BuildFileReader;
+export declare function countPendingDescriptions(buildRoot: string, exercises: CourseExercise[], readFile?: BuildFileReader): number;
+export type PublishDescriptionsStageResult = {
+    generation: GenerateCourseDescriptionsResult | null;
+    mirror: MirrorResult | null;
+    /** False when generation failed outright; drives the event status. */
+    succeeded: boolean;
+};
+/**
+ * Generate against the published package, then mirror what still matches back
+ * into the draft. Stages are recorded as they go so an interrupted run can be
+ * resumed by the sweep.
+ */
+export declare function runPublishDescriptionsStage(params: {
+    courseSlug: string;
+    rigobotToken: string;
+    gcsBucket?: Bucket;
+    journal: PublishJournal;
+    journalStorage: JournalStorage;
+    /** Injection points for tests; production resolves them from env/bucket. */
+    publishedStorage?: CourseDescriptionsStorage;
+    draftStorage?: CourseDescriptionsStorage;
+    generate?: GenerateCourseDescriptionsOptions["generate"];
+}): Promise<PublishDescriptionsStageResult>;
+/**
+ * Entry point for the detached run after publishing. Never throws: it is not
+ * awaited by anyone, so an escaping rejection would only surface as an
+ * unhandled promise.
+ */
+export declare function runDescriptionsStageAfterPublish(params: {
+    courseSlug: string;
+    gcsBucket?: Bucket;
+    journal: PublishJournal;
+    journalStorage: JournalStorage;
+    hasWork: boolean;
+    /** Present when the caller announced `descriptions: "queued"`. */
+    eventContext?: PublishEventContext;
+    breathecodeToken?: string;
+    publishedStorage?: CourseDescriptionsStorage;
+    draftStorage?: CourseDescriptionsStorage;
+    generate?: GenerateCourseDescriptionsOptions["generate"];
+    /** Injection point for tests. */
+    emit?: typeof sendPublishEvent;
+}): Promise<PublishDescriptionsStageResult>;