@learnpack/learnpack 5.0.352 → 5.0.353

package/lib/utils/publishJournal.js

@@ -0,0 +1,275 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.STAGE_NAMES = exports.JOURNAL_PREFIX = void 0;
+exports.journalKey = journalKey;
+exports.coursePrefix = coursePrefix;
+exports.parseJournalKey = parseJournalKey;
+exports.createJournal = createJournal;
+exports.markStage = markStage;
+exports.readJournal = readJournal;
+exports.readJournalByKey = readJournalByKey;
+exports.listJournalRefs = listJournalRefs;
+exports.listJournalRefsForCourse = listJournalRefsForCourse;
+exports.isStageTerminal = isStageTerminal;
+exports.hasFailedStage = hasFailedStage;
+exports.isJournalSettled = isJournalSettled;
+exports.pendingStages = pendingStages;
+exports.isJournalStale = isJournalStale;
+exports.incrementAttempts = incrementAttempts;
+exports.abandonJournal = abandonJournal;
+exports.finalizeJournal = finalizeJournal;
+exports.createGcsJournalStorage = createGcsJournalStorage;
+const uuid_1 = require("uuid");
+/**
+ * Publish journal — per-stage state of a publication.
+ *
+ * Publishing is a distributed transaction across three systems (S3 via Rigobot,
+ * breathecode assets, and our own buckets) plus an asynchronous description
+ * stage. Some of its outcomes are NOT derivable from content afterwards: in
+ * particular "was the manifest-updated event already emitted?" leaves no trace
+ * in S3 or GCS. The journal records what happened per stage so the sweep can
+ * resume an interrupted publication and honour the event contract (a publish
+ * that announced `descriptions: "queued"` must always be followed by exactly
+ * one `package_manifest_updated`).
+ *
+ * Design notes:
+ *
+ * - **Best effort, never a lock.** Every write swallows its errors: if the
+ *   journal cannot be written the publication still proceeds. Losing the
+ *   journal degrades us to the previous behaviour (no recovery, no diagnosis),
+ *   which is acceptable; blocking a publication because a bookkeeping write
+ *   failed is not.
+ * - **Flat key space.** Journals live in `publish-journal/{slug}__{id}.json`,
+ *   a sibling prefix of `courses/`, so the sweep can list every unfinished
+ *   publication with one prefix query and the concurrency guard can list a
+ *   single course with another, without maintaining an index.
+ * - **Explicit cleanup.** A journal is deleted once every stage succeeded, and
+ *   kept when any stage failed (visibility + retry). GCS lifecycle rules would
+ *   be the natural alternative, but the service account only holds
+ *   object-level permissions (`storage.buckets.*` is denied), so the cleanup
+ *   has to happen in code.
+ */
+exports.JOURNAL_PREFIX = "publish-journal/";
+/**
+ * Stages of a publication, in execution order.
+ *
+ * Only stages the orchestrator can observe on its own: writing the asset id
+ * back into Rigobot happens inside the asset sync call, so it is covered by
+ * `assetsSync` rather than being a stage that could never be marked.
+ */
+exports.STAGE_NAMES = [
+    "deploy",
+    "assetsSync",
+    "publishedEvent",
+    "descriptions",
+    "gcsMirror",
+    "manifestEvent",
+];
+function journalKey(courseSlug, publishId) {
+    return `${exports.JOURNAL_PREFIX}${courseSlug}__${publishId}.json`;
+}
+function coursePrefix(courseSlug) {
+    return `${exports.JOURNAL_PREFIX}${courseSlug}__`;
+}
+/** Inverse of `journalKey`; returns null for keys that don't match the shape. */
+function parseJournalKey(key) {
+    if (!key.startsWith(exports.JOURNAL_PREFIX) || !key.endsWith(".json")) {
+        return null;
+    }
+    const name = key.slice(exports.JOURNAL_PREFIX.length, -".json".length);
+    const separator = name.indexOf("__");
+    if (separator <= 0) {
+        return null;
+    }
+    const courseSlug = name.slice(0, separator);
+    const publishId = name.slice(separator + 2);
+    if (!courseSlug || !publishId) {
+        return null;
+    }
+    return { courseSlug, publishId, key };
+}
+function emptyStages() {
+    const stages = {};
+    for (const name of exports.STAGE_NAMES) {
+        stages[name] = { status: "pending" };
+    }
+    return stages;
+}
+/** Persist the journal. Never throws: bookkeeping must not break a publish. */
+async function persist(storage, journal) {
+    try {
+        await storage.save(journalKey(journal.courseSlug, journal.publishId), JSON.stringify(journal, null, 2));
+    }
+    catch (error) {
+        console.warn(`[publish-journal] Could not persist journal for "${journal.courseSlug}":`, error.message);
+    }
+}
+/**
+ * Start a journal for a new publication. The generated `publishId` correlates
+ * the two publish events and is returned even if persisting failed.
+ */
+async function createJournal(storage, courseSlug, publishId = (0, uuid_1.v4)()) {
+    const now = new Date().toISOString();
+    const journal = {
+        publishId,
+        courseSlug,
+        startedAt: now,
+        updatedAt: now,
+        attempts: 0,
+        stages: emptyStages(),
+    };
+    await persist(storage, journal);
+    return journal;
+}
+/** Update one stage and persist. Mutates and returns the same journal object. */
+async function markStage(storage, journal, stage, status, extra = {}) {
+    const next = { status, at: new Date().toISOString() };
+    if (extra.error !== undefined) {
+        next.error = extra.error;
+    }
+    if (extra.meta !== undefined) {
+        next.meta = extra.meta;
+    }
+    journal.stages[stage] = next;
+    journal.updatedAt = next.at;
+    await persist(storage, journal);
+    return journal;
+}
+/** Read a journal. Returns null when missing or unreadable. */
+async function readJournal(storage, courseSlug, publishId) {
+    return readJournalByKey(storage, journalKey(courseSlug, publishId));
+}
+async function readJournalByKey(storage, key) {
+    try {
+        const raw = await storage.read(key);
+        if (!raw) {
+            return null;
+        }
+        const parsed = JSON.parse(raw);
+        if (!parsed || !parsed.publishId || !parsed.stages) {
+            return null;
+        }
+        // Tolerate journals written before a stage was added to STAGE_NAMES.
+        for (const name of exports.STAGE_NAMES) {
+            if (!parsed.stages[name]) {
+                parsed.stages[name] = { status: "pending" };
+            }
+        }
+        return parsed;
+    }
+    catch (error) {
+        console.warn(`[publish-journal] Could not read journal "${key}":`, error.message);
+        return null;
+    }
+}
+/**
+ * Every journal still stored, i.e. every publication that did not finish
+ * cleanly. Returns an empty list on error (the sweep falls back to its deep
+ * scan rather than crashing).
+ */
+async function listJournalRefs(storage) {
+    return listRefsByPrefix(storage, exports.JOURNAL_PREFIX);
+}
+/** Journals of a single course; backs the concurrent-publish guard. */
+async function listJournalRefsForCourse(storage, courseSlug) {
+    return listRefsByPrefix(storage, coursePrefix(courseSlug));
+}
+async function listRefsByPrefix(storage, prefix) {
+    try {
+        const keys = await storage.list(prefix);
+        const refs = [];
+        for (const key of keys) {
+            const ref = parseJournalKey(key);
+            if (ref) {
+                refs.push(ref);
+            }
+        }
+        return refs;
+    }
+    catch (error) {
+        console.warn(`[publish-journal] Could not list journals "${prefix}":`, error.message);
+        return [];
+    }
+}
+function isStageTerminal(stage) {
+    return stage.status === "done" || stage.status === "failed";
+}
+function hasFailedStage(journal) {
+    return exports.STAGE_NAMES.some(name => journal.stages[name].status === "failed");
+}
+/** True when no stage is left to run (all done, or done/failed mix). */
+function isJournalSettled(journal) {
+    return exports.STAGE_NAMES.every(name => isStageTerminal(journal.stages[name]));
+}
+/** Stages the sweep still has work to do on. */
+function pendingStages(journal) {
+    return exports.STAGE_NAMES.filter(name => !isStageTerminal(journal.stages[name]));
+}
+/** A journal untouched for longer than `maxAgeMs` is presumed interrupted. */
+function isJournalStale(journal, maxAgeMs, now = Date.now()) {
+    const updatedAt = Date.parse(journal.updatedAt);
+    if (Number.isNaN(updatedAt)) {
+        return true;
+    }
+    return now - updatedAt > maxAgeMs;
+}
+async function incrementAttempts(storage, journal) {
+    journal.attempts += 1;
+    journal.updatedAt = new Date().toISOString();
+    await persist(storage, journal);
+    return journal;
+}
+/**
+ * Give up on a journal after too many attempts. It is kept, not deleted: a
+ * publication that never completed is exactly what someone needs to see.
+ */
+async function abandonJournal(storage, journal) {
+    journal.abandoned = true;
+    journal.updatedAt = new Date().toISOString();
+    await persist(storage, journal);
+    return journal;
+}
+/**
+ * Close a journal: delete it when every stage succeeded, keep it otherwise so
+ * the sweep can retry and a human can inspect it. Never throws.
+ */
+async function finalizeJournal(storage, journal) {
+    if (!isJournalSettled(journal) || hasFailedStage(journal)) {
+        await persist(storage, journal);
+        return "kept";
+    }
+    try {
+        await storage.remove(journalKey(journal.courseSlug, journal.publishId));
+        return "deleted";
+    }
+    catch (error) {
+        console.warn(`[publish-journal] Could not delete journal for "${journal.courseSlug}":`, error.message);
+        return "kept";
+    }
+}
+/** GCS-backed adapter. Object-level operations only, by design. */
+function createGcsJournalStorage(bucket) {
+    return {
+        async save(key, contents) {
+            await bucket.file(key).save(Buffer.from(contents, "utf8"), {
+                contentType: "application/json",
+            });
+        },
+        async read(key) {
+            try {
+                const [buf] = await bucket.file(key).download();
+                return buf.toString();
+            }
+            catch (_a) {
+                return null;
+            }
+        },
+        async remove(key) {
+            await bucket.file(key).delete();
+        },
+        async list(prefix) {
+            const [files] = await bucket.getFiles({ prefix });
+            return files.map(file => file.name);
+        },
+    };
+}

package/lib/utils/rigoActions.d.ts

@@ -99,6 +99,50 @@ type TInitialContentGeneratorInputs = {
     topic_description?: string;
 };
 export declare const initialContentGenerator: (token: string, inputs: TInitialContentGeneratorInputs, webhookUrl?: string, endpointSlug?: string) => Promise<any>;
+export type TGenerateStepDescriptionsParams = {
+    /** README of the step, in `sourceLanguage`. Images are stripped before sending. */
+    readmeContent: string;
+    sourceLanguage: string;
+    /** Every language to produce; may or may not include `sourceLanguage`. */
+    outputLanguages: string[];
+    lessonTitle: string;
+    courseTitle?: string;
+    targetWordCount?: number;
+    promptVersion?: number;
+};
+export type TGenerateStepDescriptionsResult = {
+    /**
+     * Descriptions keyed by the `language_code` Rigobot answered with — building
+     * this record IS the reassociation, the response order is not contractual.
+     * A `null` value is the deliberate fallback for content too thin to describe.
+     */
+    descriptionsByLanguage: Record<string, string | null>;
+    /** Requested languages absent from the response (a protocol problem, not a
+     * "no description" answer: callers should retry these rather than storing a
+     * null that would look settled). */
+    missingLanguages: string[];
+    promptVersion: number;
+    /** Seconds the completion took; worth logging to watch the 30s Heroku edge. */
+    durationSeconds: number | null;
+    completionId: number | null;
+};
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+export declare const generateStepDescriptions: (token: string, params: TGenerateStepDescriptionsParams, endpointSlug?: string) => Promise<TGenerateStepDescriptionsResult | null>;
 type TAddInteractivityInputs = {
     components: string;
     prev_lesson: string;

package/lib/utils/rigoActions.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateStepSlug = exports.generateCodeChallenge = exports.addInteractivity = exports.initialContentGenerator = exports.getLanguageCodes = exports.isPackageAuthor = exports.fillSidebarJSON = exports.generateCourseShortName = exports.isValidRigoToken = exports.translateCourseMetadata = exports.createStructuredPreviewReadme = exports.readmeCreator = exports.createCodingReadme = exports.createCodeFile = exports.interactiveCreation = exports.generateCourseIntroduction = exports.translateExercise = exports.generateImage = exports.hasCreatorPermission = exports.createReadme = void 0;
+exports.generateStepSlug = exports.generateCodeChallenge = exports.addInteractivity = exports.generateStepDescriptions = exports.initialContentGenerator = exports.getLanguageCodes = exports.isPackageAuthor = exports.fillSidebarJSON = exports.generateCourseShortName = exports.isValidRigoToken = exports.translateCourseMetadata = exports.createStructuredPreviewReadme = exports.readmeCreator = exports.createCodingReadme = exports.createCodeFile = exports.interactiveCreation = exports.generateCourseIntroduction = exports.translateExercise = exports.generateImage = exports.hasCreatorPermission = exports.createReadme = void 0;
 exports.downloadImage = downloadImage;
 exports.createPreviewReadme = createPreviewReadme;
 exports.makeReadmeReadable = makeReadmeReadable;
@@ -10,6 +10,8 @@ const console_1 = require("../utils/console");
 const fs = require("fs");
 const path = require("path");
 const api_1 = require("./api");
+const descriptionHash_1 = require("./descriptionHash");
+const packageManifest_1 = require("./packageManifest");
 const createReadme = async (token, inputs, purpose, webhookUrl) => {
     try {
         const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/create-sequencial-readme/`, {
@@ -304,6 +306,78 @@ const initialContentGenerator = async (token, inputs, webhookUrl, endpointSlug =
     }
 };
 exports.initialContentGenerator = initialContentGenerator;
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+const generateStepDescriptions = async (token, params, endpointSlug = "generate-step-descriptions") => {
+    var _a, _b, _c, _d, _e;
+    const promptVersion = (_a = params.promptVersion) !== null && _a !== void 0 ? _a : packageManifest_1.DESCRIPTION_PROMPT_VERSION;
+    const targetWordCount = (_b = params.targetWordCount) !== null && _b !== void 0 ? _b : packageManifest_1.DESCRIPTION_TARGET_WORD_COUNT;
+    const inputs = {
+        // Images never help a description and their alt text may be a whole
+        // image-generation prompt; markdown structure is kept, it helps the model.
+        readme_content: (0, descriptionHash_1.stripMarkdownImages)(params.readmeContent),
+        source_language: params.sourceLanguage,
+        output_languages: params.outputLanguages.join(","),
+        lesson_title: params.lessonTitle,
+        course_title: (_c = params.courseTitle) !== null && _c !== void 0 ? _c : "",
+        target_word_count: String(targetWordCount),
+        prompt_version: String(promptVersion),
+    };
+    try {
+        const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/${endpointSlug}/`, { inputs, execute_async: false }, {
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: "Token " + token.trim(),
+            },
+        });
+        const data = response.data;
+        if (!data || data.status === "ERROR") {
+            console.error("Error in generateStepDescriptions:", (data === null || data === void 0 ? void 0 : data.status_text) || "completion returned ERROR");
+            return null;
+        }
+        const entries = Array.isArray((_d = data.parsed) === null || _d === void 0 ? void 0 : _d.descriptions) ?
+            data.parsed.descriptions :
+            [];
+        const descriptionsByLanguage = {};
+        for (const entry of entries) {
+            const code = entry === null || entry === void 0 ? void 0 : entry.language_code;
+            if (typeof code !== "string" || !code) {
+                continue;
+            }
+            descriptionsByLanguage[code] =
+                typeof entry.description === "string" ? entry.description : null;
+        }
+        const echoedVersion = Number.parseInt((_e = data.inputs) === null || _e === void 0 ? void 0 : _e.prompt_version, 10);
+        return {
+            descriptionsByLanguage,
+            missingLanguages: params.outputLanguages.filter(lang => !(lang in descriptionsByLanguage)),
+            promptVersion: Number.isFinite(echoedVersion) ?
+                echoedVersion :
+                promptVersion,
+            durationSeconds: typeof data.duration === "number" ? data.duration : null,
+            completionId: typeof data.id === "number" ? data.id : null,
+        };
+    }
+    catch (error) {
+        console.error("Error in generateStepDescriptions:", error);
+        return null;
+    }
+};
+exports.generateStepDescriptions = generateStepDescriptions;
 const addInteractivity = async (token, inputs, webhookUrl) => {
     try {
         const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/add-lesson-interactivy/`, {

package/lib/utils/s3/packageManifestBackfill.d.ts

@@ -64,6 +64,8 @@ export declare function withRetry<T>(fn: () => Promise<T>, attempts?: number, at
 export declare function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
 export declare function listPublishedSlugs(s3: AwsClient, bucket: string): Promise<string[]>;
 export declare function fetchJsonObject<T>(s3: AwsClient, bucket: string, key: string): Promise<T | null>;
+export declare function fetchTextObject(s3: AwsClient, bucket: string, key: string): Promise<string | null>;
+export declare function listObjectKeys(s3: AwsClient, bucket: string, prefix: string): Promise<string[]>;
 export declare function fetchPackageSources(s3: AwsClient, bucket: string, slug: string): Promise<PackageSources>;
 export declare function buildManifestFromS3Sources(slug: string, sources: PackageSources, options: {
     publishedAt: string | null;

package/lib/utils/s3/packageManifestBackfill.js

@@ -5,6 +5,8 @@ exports.withRetry = withRetry;
 exports.mapWithConcurrency = mapWithConcurrency;
 exports.listPublishedSlugs = listPublishedSlugs;
 exports.fetchJsonObject = fetchJsonObject;
+exports.fetchTextObject = fetchTextObject;
+exports.listObjectKeys = listObjectKeys;
 exports.fetchPackageSources = fetchPackageSources;
 exports.buildManifestFromS3Sources = buildManifestFromS3Sources;
 exports.processPackage = processPackage;

package/lib/utils/syllabusSync.d.ts

@@ -0,0 +1,71 @@
+import { Bucket } from "@google-cloud/storage";
+import { Syllabus } from "../models/creator";
+/**
+ * Provisional syllabus ↔ bucket reconciliation.
+ *
+ * The `initialSyllabus.json` occasionally drifts from what actually lives in the
+ * bucket (lessons/translations present as files but missing from the syllabus,
+ * or vice versa). Because step descriptions are stored per (step, lang) in the
+ * syllabus, that drift leaves missing slots and causes skipped/spurious
+ * generations. This reconciles the two.
+ *
+ * Extracted from the inline `/actions/synchronize-syllabus` handler so it can be
+ * reused by the sweep/backfill, and generalized over a minimal storage adapter
+ * so it works for both GCS and S3.
+ *
+ * NOTE: this is a temporary patch until the root cause of the drift is fixed.
+ * Track its removal via the `DESCRIPTIONS_RECONCILE_SYLLABUS` flag on callers.
+ */
+export interface SyllabusSyncStorage {
+    /**
+     * Map of `folderSlug -> fileCount` for every folder directly under
+     * `exercises/`. A single authoritative listing: if it throws, callers must
+     * abort any destructive action (never delete on an errored/unknown listing).
+     */
+    listExerciseFolderFileCounts(courseSlug: string): Promise<Map<string, number>>;
+    /** README language codes (lowercased) present per exercise folder slug. */
+    translationLangsBySlug(courseSlug: string): Promise<Map<string, string[]>>;
+}
+export type SyllabusSyncOptions = {
+    /**
+     * When false (default, "additive" mode) only non-destructive passes run:
+     * add-missing, repair-translations, unstick-status. Removal/dedupe are
+     * skipped. When true (manual dev button / root-cause runs) removals apply.
+     */
+    prune?: boolean;
+};
+export type SyllabusSyncCounters = {
+    totalLessons: number;
+    keptLessons: number;
+    removedLessons: number;
+    duplicatesResolved: number;
+    addedLessons: number;
+    fixedLessons: number;
+    repairedTranslationsInLessons: number;
+    repairedTranslationEntries: number;
+};
+type LessonRef = {
+    id: string;
+    title: string;
+    slug: string;
+};
+type LessonRefWithCount = LessonRef & {
+    fileCount: number;
+};
+export type SyllabusSyncResult = {
+    syllabus: Syllabus;
+    changed: boolean;
+    counters: SyllabusSyncCounters;
+    details: {
+        removed: LessonRef[];
+        duplicates: LessonRefWithCount[];
+        kept: Array<LessonRef & {
+            fileCount?: number;
+        }>;
+        added: LessonRef[];
+    };
+};
+export declare function synchronizeSyllabusWithBucket(storage: SyllabusSyncStorage, courseSlug: string, syllabus: Syllabus, options?: SyllabusSyncOptions): Promise<SyllabusSyncResult>;
+/** GCS-backed storage adapter (uses the bucket listing + buildConfig). */
+export declare function createGcsSyllabusSyncStorage(bucket: Bucket): SyllabusSyncStorage;
+export {};