@learnpack/learnpack 5.0.351 → 5.0.353

package/lib/utils/publishJournal.d.ts

@@ -0,0 +1,119 @@
+import { Bucket } from "@google-cloud/storage";
+/**
+ * Publish journal — per-stage state of a publication.
+ *
+ * Publishing is a distributed transaction across three systems (S3 via Rigobot,
+ * breathecode assets, and our own buckets) plus an asynchronous description
+ * stage. Some of its outcomes are NOT derivable from content afterwards: in
+ * particular "was the manifest-updated event already emitted?" leaves no trace
+ * in S3 or GCS. The journal records what happened per stage so the sweep can
+ * resume an interrupted publication and honour the event contract (a publish
+ * that announced `descriptions: "queued"` must always be followed by exactly
+ * one `package_manifest_updated`).
+ *
+ * Design notes:
+ *
+ * - **Best effort, never a lock.** Every write swallows its errors: if the
+ *   journal cannot be written the publication still proceeds. Losing the
+ *   journal degrades us to the previous behaviour (no recovery, no diagnosis),
+ *   which is acceptable; blocking a publication because a bookkeeping write
+ *   failed is not.
+ * - **Flat key space.** Journals live in `publish-journal/{slug}__{id}.json`,
+ *   a sibling prefix of `courses/`, so the sweep can list every unfinished
+ *   publication with one prefix query and the concurrency guard can list a
+ *   single course with another, without maintaining an index.
+ * - **Explicit cleanup.** A journal is deleted once every stage succeeded, and
+ *   kept when any stage failed (visibility + retry). GCS lifecycle rules would
+ *   be the natural alternative, but the service account only holds
+ *   object-level permissions (`storage.buckets.*` is denied), so the cleanup
+ *   has to happen in code.
+ */
+export declare const JOURNAL_PREFIX = "publish-journal/";
+/**
+ * Stages of a publication, in execution order.
+ *
+ * Only stages the orchestrator can observe on its own: writing the asset id
+ * back into Rigobot happens inside the asset sync call, so it is covered by
+ * `assetsSync` rather than being a stage that could never be marked.
+ */
+export declare const STAGE_NAMES: readonly ["deploy", "assetsSync", "publishedEvent", "descriptions", "gcsMirror", "manifestEvent"];
+export type StageName = typeof STAGE_NAMES[number];
+export type StageStatus = "pending" | "running" | "done" | "failed";
+export type Stage = {
+    status: StageStatus;
+    at?: string;
+    error?: string;
+    meta?: unknown;
+};
+export type PublishJournal = {
+    publishId: string;
+    courseSlug: string;
+    startedAt: string;
+    updatedAt: string;
+    /** Times the sweep has picked this journal up, to bound retries. */
+    attempts: number;
+    /** Set once retries are exhausted: kept for a human, ignored by the sweep. */
+    abandoned?: boolean;
+    stages: Record<StageName, Stage>;
+};
+export type JournalRef = {
+    courseSlug: string;
+    publishId: string;
+    key: string;
+};
+/**
+ * Minimal storage surface the journal needs, so callers can pass the real
+ * bucket adapter or an in-memory double. Object-level operations only.
+ */
+export interface JournalStorage {
+    save(key: string, contents: string): Promise<void>;
+    read(key: string): Promise<string | null>;
+    remove(key: string): Promise<void>;
+    list(prefix: string): Promise<string[]>;
+}
+export declare function journalKey(courseSlug: string, publishId: string): string;
+export declare function coursePrefix(courseSlug: string): string;
+/** Inverse of `journalKey`; returns null for keys that don't match the shape. */
+export declare function parseJournalKey(key: string): JournalRef | null;
+/**
+ * Start a journal for a new publication. The generated `publishId` correlates
+ * the two publish events and is returned even if persisting failed.
+ */
+export declare function createJournal(storage: JournalStorage, courseSlug: string, publishId?: string): Promise<PublishJournal>;
+/** Update one stage and persist. Mutates and returns the same journal object. */
+export declare function markStage(storage: JournalStorage, journal: PublishJournal, stage: StageName, status: StageStatus, extra?: {
+    error?: string;
+    meta?: unknown;
+}): Promise<PublishJournal>;
+/** Read a journal. Returns null when missing or unreadable. */
+export declare function readJournal(storage: JournalStorage, courseSlug: string, publishId: string): Promise<PublishJournal | null>;
+export declare function readJournalByKey(storage: JournalStorage, key: string): Promise<PublishJournal | null>;
+/**
+ * Every journal still stored, i.e. every publication that did not finish
+ * cleanly. Returns an empty list on error (the sweep falls back to its deep
+ * scan rather than crashing).
+ */
+export declare function listJournalRefs(storage: JournalStorage): Promise<JournalRef[]>;
+/** Journals of a single course; backs the concurrent-publish guard. */
+export declare function listJournalRefsForCourse(storage: JournalStorage, courseSlug: string): Promise<JournalRef[]>;
+export declare function isStageTerminal(stage: Stage): boolean;
+export declare function hasFailedStage(journal: PublishJournal): boolean;
+/** True when no stage is left to run (all done, or done/failed mix). */
+export declare function isJournalSettled(journal: PublishJournal): boolean;
+/** Stages the sweep still has work to do on. */
+export declare function pendingStages(journal: PublishJournal): StageName[];
+/** A journal untouched for longer than `maxAgeMs` is presumed interrupted. */
+export declare function isJournalStale(journal: PublishJournal, maxAgeMs: number, now?: number): boolean;
+export declare function incrementAttempts(storage: JournalStorage, journal: PublishJournal): Promise<PublishJournal>;
+/**
+ * Give up on a journal after too many attempts. It is kept, not deleted: a
+ * publication that never completed is exactly what someone needs to see.
+ */
+export declare function abandonJournal(storage: JournalStorage, journal: PublishJournal): Promise<PublishJournal>;
+/**
+ * Close a journal: delete it when every stage succeeded, keep it otherwise so
+ * the sweep can retry and a human can inspect it. Never throws.
+ */
+export declare function finalizeJournal(storage: JournalStorage, journal: PublishJournal): Promise<"deleted" | "kept">;
+/** GCS-backed adapter. Object-level operations only, by design. */
+export declare function createGcsJournalStorage(bucket: Bucket): JournalStorage;

package/lib/utils/publishJournal.js

@@ -0,0 +1,275 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.STAGE_NAMES = exports.JOURNAL_PREFIX = void 0;
+exports.journalKey = journalKey;
+exports.coursePrefix = coursePrefix;
+exports.parseJournalKey = parseJournalKey;
+exports.createJournal = createJournal;
+exports.markStage = markStage;
+exports.readJournal = readJournal;
+exports.readJournalByKey = readJournalByKey;
+exports.listJournalRefs = listJournalRefs;
+exports.listJournalRefsForCourse = listJournalRefsForCourse;
+exports.isStageTerminal = isStageTerminal;
+exports.hasFailedStage = hasFailedStage;
+exports.isJournalSettled = isJournalSettled;
+exports.pendingStages = pendingStages;
+exports.isJournalStale = isJournalStale;
+exports.incrementAttempts = incrementAttempts;
+exports.abandonJournal = abandonJournal;
+exports.finalizeJournal = finalizeJournal;
+exports.createGcsJournalStorage = createGcsJournalStorage;
+const uuid_1 = require("uuid");
+/**
+ * Publish journal — per-stage state of a publication.
+ *
+ * Publishing is a distributed transaction across three systems (S3 via Rigobot,
+ * breathecode assets, and our own buckets) plus an asynchronous description
+ * stage. Some of its outcomes are NOT derivable from content afterwards: in
+ * particular "was the manifest-updated event already emitted?" leaves no trace
+ * in S3 or GCS. The journal records what happened per stage so the sweep can
+ * resume an interrupted publication and honour the event contract (a publish
+ * that announced `descriptions: "queued"` must always be followed by exactly
+ * one `package_manifest_updated`).
+ *
+ * Design notes:
+ *
+ * - **Best effort, never a lock.** Every write swallows its errors: if the
+ *   journal cannot be written the publication still proceeds. Losing the
+ *   journal degrades us to the previous behaviour (no recovery, no diagnosis),
+ *   which is acceptable; blocking a publication because a bookkeeping write
+ *   failed is not.
+ * - **Flat key space.** Journals live in `publish-journal/{slug}__{id}.json`,
+ *   a sibling prefix of `courses/`, so the sweep can list every unfinished
+ *   publication with one prefix query and the concurrency guard can list a
+ *   single course with another, without maintaining an index.
+ * - **Explicit cleanup.** A journal is deleted once every stage succeeded, and
+ *   kept when any stage failed (visibility + retry). GCS lifecycle rules would
+ *   be the natural alternative, but the service account only holds
+ *   object-level permissions (`storage.buckets.*` is denied), so the cleanup
+ *   has to happen in code.
+ */
+exports.JOURNAL_PREFIX = "publish-journal/";
+/**
+ * Stages of a publication, in execution order.
+ *
+ * Only stages the orchestrator can observe on its own: writing the asset id
+ * back into Rigobot happens inside the asset sync call, so it is covered by
+ * `assetsSync` rather than being a stage that could never be marked.
+ */
+exports.STAGE_NAMES = [
+    "deploy",
+    "assetsSync",
+    "publishedEvent",
+    "descriptions",
+    "gcsMirror",
+    "manifestEvent",
+];
+function journalKey(courseSlug, publishId) {
+    return `${exports.JOURNAL_PREFIX}${courseSlug}__${publishId}.json`;
+}
+function coursePrefix(courseSlug) {
+    return `${exports.JOURNAL_PREFIX}${courseSlug}__`;
+}
+/** Inverse of `journalKey`; returns null for keys that don't match the shape. */
+function parseJournalKey(key) {
+    if (!key.startsWith(exports.JOURNAL_PREFIX) || !key.endsWith(".json")) {
+        return null;
+    }
+    const name = key.slice(exports.JOURNAL_PREFIX.length, -".json".length);
+    const separator = name.indexOf("__");
+    if (separator <= 0) {
+        return null;
+    }
+    const courseSlug = name.slice(0, separator);
+    const publishId = name.slice(separator + 2);
+    if (!courseSlug || !publishId) {
+        return null;
+    }
+    return { courseSlug, publishId, key };
+}
+function emptyStages() {
+    const stages = {};
+    for (const name of exports.STAGE_NAMES) {
+        stages[name] = { status: "pending" };
+    }
+    return stages;
+}
+/** Persist the journal. Never throws: bookkeeping must not break a publish. */
+async function persist(storage, journal) {
+    try {
+        await storage.save(journalKey(journal.courseSlug, journal.publishId), JSON.stringify(journal, null, 2));
+    }
+    catch (error) {
+        console.warn(`[publish-journal] Could not persist journal for "${journal.courseSlug}":`, error.message);
+    }
+}
+/**
+ * Start a journal for a new publication. The generated `publishId` correlates
+ * the two publish events and is returned even if persisting failed.
+ */
+async function createJournal(storage, courseSlug, publishId = (0, uuid_1.v4)()) {
+    const now = new Date().toISOString();
+    const journal = {
+        publishId,
+        courseSlug,
+        startedAt: now,
+        updatedAt: now,
+        attempts: 0,
+        stages: emptyStages(),
+    };
+    await persist(storage, journal);
+    return journal;
+}
+/** Update one stage and persist. Mutates and returns the same journal object. */
+async function markStage(storage, journal, stage, status, extra = {}) {
+    const next = { status, at: new Date().toISOString() };
+    if (extra.error !== undefined) {
+        next.error = extra.error;
+    }
+    if (extra.meta !== undefined) {
+        next.meta = extra.meta;
+    }
+    journal.stages[stage] = next;
+    journal.updatedAt = next.at;
+    await persist(storage, journal);
+    return journal;
+}
+/** Read a journal. Returns null when missing or unreadable. */
+async function readJournal(storage, courseSlug, publishId) {
+    return readJournalByKey(storage, journalKey(courseSlug, publishId));
+}
+async function readJournalByKey(storage, key) {
+    try {
+        const raw = await storage.read(key);
+        if (!raw) {
+            return null;
+        }
+        const parsed = JSON.parse(raw);
+        if (!parsed || !parsed.publishId || !parsed.stages) {
+            return null;
+        }
+        // Tolerate journals written before a stage was added to STAGE_NAMES.
+        for (const name of exports.STAGE_NAMES) {
+            if (!parsed.stages[name]) {
+                parsed.stages[name] = { status: "pending" };
+            }
+        }
+        return parsed;
+    }
+    catch (error) {
+        console.warn(`[publish-journal] Could not read journal "${key}":`, error.message);
+        return null;
+    }
+}
+/**
+ * Every journal still stored, i.e. every publication that did not finish
+ * cleanly. Returns an empty list on error (the sweep falls back to its deep
+ * scan rather than crashing).
+ */
+async function listJournalRefs(storage) {
+    return listRefsByPrefix(storage, exports.JOURNAL_PREFIX);
+}
+/** Journals of a single course; backs the concurrent-publish guard. */
+async function listJournalRefsForCourse(storage, courseSlug) {
+    return listRefsByPrefix(storage, coursePrefix(courseSlug));
+}
+async function listRefsByPrefix(storage, prefix) {
+    try {
+        const keys = await storage.list(prefix);
+        const refs = [];
+        for (const key of keys) {
+            const ref = parseJournalKey(key);
+            if (ref) {
+                refs.push(ref);
+            }
+        }
+        return refs;
+    }
+    catch (error) {
+        console.warn(`[publish-journal] Could not list journals "${prefix}":`, error.message);
+        return [];
+    }
+}
+function isStageTerminal(stage) {
+    return stage.status === "done" || stage.status === "failed";
+}
+function hasFailedStage(journal) {
+    return exports.STAGE_NAMES.some(name => journal.stages[name].status === "failed");
+}
+/** True when no stage is left to run (all done, or done/failed mix). */
+function isJournalSettled(journal) {
+    return exports.STAGE_NAMES.every(name => isStageTerminal(journal.stages[name]));
+}
+/** Stages the sweep still has work to do on. */
+function pendingStages(journal) {
+    return exports.STAGE_NAMES.filter(name => !isStageTerminal(journal.stages[name]));
+}
+/** A journal untouched for longer than `maxAgeMs` is presumed interrupted. */
+function isJournalStale(journal, maxAgeMs, now = Date.now()) {
+    const updatedAt = Date.parse(journal.updatedAt);
+    if (Number.isNaN(updatedAt)) {
+        return true;
+    }
+    return now - updatedAt > maxAgeMs;
+}
+async function incrementAttempts(storage, journal) {
+    journal.attempts += 1;
+    journal.updatedAt = new Date().toISOString();
+    await persist(storage, journal);
+    return journal;
+}
+/**
+ * Give up on a journal after too many attempts. It is kept, not deleted: a
+ * publication that never completed is exactly what someone needs to see.
+ */
+async function abandonJournal(storage, journal) {
+    journal.abandoned = true;
+    journal.updatedAt = new Date().toISOString();
+    await persist(storage, journal);
+    return journal;
+}
+/**
+ * Close a journal: delete it when every stage succeeded, keep it otherwise so
+ * the sweep can retry and a human can inspect it. Never throws.
+ */
+async function finalizeJournal(storage, journal) {
+    if (!isJournalSettled(journal) || hasFailedStage(journal)) {
+        await persist(storage, journal);
+        return "kept";
+    }
+    try {
+        await storage.remove(journalKey(journal.courseSlug, journal.publishId));
+        return "deleted";
+    }
+    catch (error) {
+        console.warn(`[publish-journal] Could not delete journal for "${journal.courseSlug}":`, error.message);
+        return "kept";
+    }
+}
+/** GCS-backed adapter. Object-level operations only, by design. */
+function createGcsJournalStorage(bucket) {
+    return {
+        async save(key, contents) {
+            await bucket.file(key).save(Buffer.from(contents, "utf8"), {
+                contentType: "application/json",
+            });
+        },
+        async read(key) {
+            try {
+                const [buf] = await bucket.file(key).download();
+                return buf.toString();
+            }
+            catch (_a) {
+                return null;
+            }
+        },
+        async remove(key) {
+            await bucket.file(key).delete();
+        },
+        async list(prefix) {
+            const [files] = await bucket.getFiles({ prefix });
+            return files.map(file => file.name);
+        },
+    };
+}

package/lib/utils/rigoActions.d.ts

@@ -99,6 +99,50 @@ type TInitialContentGeneratorInputs = {
     topic_description?: string;
 };
 export declare const initialContentGenerator: (token: string, inputs: TInitialContentGeneratorInputs, webhookUrl?: string, endpointSlug?: string) => Promise<any>;
+export type TGenerateStepDescriptionsParams = {
+    /** README of the step, in `sourceLanguage`. Images are stripped before sending. */
+    readmeContent: string;
+    sourceLanguage: string;
+    /** Every language to produce; may or may not include `sourceLanguage`. */
+    outputLanguages: string[];
+    lessonTitle: string;
+    courseTitle?: string;
+    targetWordCount?: number;
+    promptVersion?: number;
+};
+export type TGenerateStepDescriptionsResult = {
+    /**
+     * Descriptions keyed by the `language_code` Rigobot answered with — building
+     * this record IS the reassociation, the response order is not contractual.
+     * A `null` value is the deliberate fallback for content too thin to describe.
+     */
+    descriptionsByLanguage: Record<string, string | null>;
+    /** Requested languages absent from the response (a protocol problem, not a
+     * "no description" answer: callers should retry these rather than storing a
+     * null that would look settled). */
+    missingLanguages: string[];
+    promptVersion: number;
+    /** Seconds the completion took; worth logging to watch the 30s Heroku edge. */
+    durationSeconds: number | null;
+    completionId: number | null;
+};
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+export declare const generateStepDescriptions: (token: string, params: TGenerateStepDescriptionsParams, endpointSlug?: string) => Promise<TGenerateStepDescriptionsResult | null>;
 type TAddInteractivityInputs = {
     components: string;
     prev_lesson: string;

package/lib/utils/rigoActions.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateStepSlug = exports.generateCodeChallenge = exports.addInteractivity = exports.initialContentGenerator = exports.getLanguageCodes = exports.isPackageAuthor = exports.fillSidebarJSON = exports.generateCourseShortName = exports.isValidRigoToken = exports.translateCourseMetadata = exports.createStructuredPreviewReadme = exports.readmeCreator = exports.createCodingReadme = exports.createCodeFile = exports.interactiveCreation = exports.generateCourseIntroduction = exports.translateExercise = exports.generateImage = exports.hasCreatorPermission = exports.createReadme = void 0;
+exports.generateStepSlug = exports.generateCodeChallenge = exports.addInteractivity = exports.generateStepDescriptions = exports.initialContentGenerator = exports.getLanguageCodes = exports.isPackageAuthor = exports.fillSidebarJSON = exports.generateCourseShortName = exports.isValidRigoToken = exports.translateCourseMetadata = exports.createStructuredPreviewReadme = exports.readmeCreator = exports.createCodingReadme = exports.createCodeFile = exports.interactiveCreation = exports.generateCourseIntroduction = exports.translateExercise = exports.generateImage = exports.hasCreatorPermission = exports.createReadme = void 0;
 exports.downloadImage = downloadImage;
 exports.createPreviewReadme = createPreviewReadme;
 exports.makeReadmeReadable = makeReadmeReadable;
@@ -10,6 +10,8 @@ const console_1 = require("../utils/console");
 const fs = require("fs");
 const path = require("path");
 const api_1 = require("./api");
+const descriptionHash_1 = require("./descriptionHash");
+const packageManifest_1 = require("./packageManifest");
 const createReadme = async (token, inputs, purpose, webhookUrl) => {
     try {
         const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/create-sequencial-readme/`, {
@@ -304,6 +306,78 @@ const initialContentGenerator = async (token, inputs, webhookUrl, endpointSlug =
     }
 };
 exports.initialContentGenerator = initialContentGenerator;
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+const generateStepDescriptions = async (token, params, endpointSlug = "generate-step-descriptions") => {
+    var _a, _b, _c, _d, _e;
+    const promptVersion = (_a = params.promptVersion) !== null && _a !== void 0 ? _a : packageManifest_1.DESCRIPTION_PROMPT_VERSION;
+    const targetWordCount = (_b = params.targetWordCount) !== null && _b !== void 0 ? _b : packageManifest_1.DESCRIPTION_TARGET_WORD_COUNT;
+    const inputs = {
+        // Images never help a description and their alt text may be a whole
+        // image-generation prompt; markdown structure is kept, it helps the model.
+        readme_content: (0, descriptionHash_1.stripMarkdownImages)(params.readmeContent),
+        source_language: params.sourceLanguage,
+        output_languages: params.outputLanguages.join(","),
+        lesson_title: params.lessonTitle,
+        course_title: (_c = params.courseTitle) !== null && _c !== void 0 ? _c : "",
+        target_word_count: String(targetWordCount),
+        prompt_version: String(promptVersion),
+    };
+    try {
+        const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/${endpointSlug}/`, { inputs, execute_async: false }, {
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: "Token " + token.trim(),
+            },
+        });
+        const data = response.data;
+        if (!data || data.status === "ERROR") {
+            console.error("Error in generateStepDescriptions:", (data === null || data === void 0 ? void 0 : data.status_text) || "completion returned ERROR");
+            return null;
+        }
+        const entries = Array.isArray((_d = data.parsed) === null || _d === void 0 ? void 0 : _d.descriptions) ?
+            data.parsed.descriptions :
+            [];
+        const descriptionsByLanguage = {};
+        for (const entry of entries) {
+            const code = entry === null || entry === void 0 ? void 0 : entry.language_code;
+            if (typeof code !== "string" || !code) {
+                continue;
+            }
+            descriptionsByLanguage[code] =
+                typeof entry.description === "string" ? entry.description : null;
+        }
+        const echoedVersion = Number.parseInt((_e = data.inputs) === null || _e === void 0 ? void 0 : _e.prompt_version, 10);
+        return {
+            descriptionsByLanguage,
+            missingLanguages: params.outputLanguages.filter(lang => !(lang in descriptionsByLanguage)),
+            promptVersion: Number.isFinite(echoedVersion) ?
+                echoedVersion :
+                promptVersion,
+            durationSeconds: typeof data.duration === "number" ? data.duration : null,
+            completionId: typeof data.id === "number" ? data.id : null,
+        };
+    }
+    catch (error) {
+        console.error("Error in generateStepDescriptions:", error);
+        return null;
+    }
+};
+exports.generateStepDescriptions = generateStepDescriptions;
 const addInteractivity = async (token, inputs, webhookUrl) => {
     try {
         const response = await axios_1.default.post(`${api_1.RIGOBOT_HOST}/v1/prompting/completion/add-lesson-interactivy/`, {

package/lib/utils/s3/packageManifestBackfill.d.ts

@@ -0,0 +1,84 @@
+import { Exercise } from "../configBuilder";
+import { Syllabus } from "../../models/creator";
+import { PackageManifest, ReadmeFrontmatter } from "../packageManifest";
+export type PackageResultStatus = "created" | "updated" | "skipped" | "failed";
+export type PackageResult = {
+    slug: string;
+    status: PackageResultStatus;
+    durationMs: number;
+    error?: string;
+    step?: string;
+    sources?: {
+        learnJson: boolean;
+        configJson: boolean;
+        sidebar: boolean;
+        syllabus: boolean;
+        readmes: number;
+    };
+};
+export type BackfillOptions = {
+    dryRun: boolean;
+    force: boolean;
+    skipExisting: boolean;
+    noInvalidate: boolean;
+    concurrency: number;
+    publishedAt: string | null;
+    resumeFrom?: string;
+    reportOut: string;
+    slug?: string;
+    bucket: string;
+    region: string;
+    cloudFrontDistributionId?: string;
+};
+export type BatchReport = {
+    total: number;
+    created: number;
+    updated: number;
+    skipped: number;
+    failed: number;
+    startedAt: string;
+    finishedAt: string;
+    options: BackfillOptions;
+    results: PackageResult[];
+    invalidation?: {
+        attempted: boolean;
+        skippedReason?: string;
+        id?: string;
+        paths?: string[];
+        error?: string;
+    };
+};
+export type AwsClient = {
+    send(command: unknown): Promise<unknown>;
+};
+type PackageSources = {
+    learnJson: Record<string, unknown>;
+    exercises: Exercise[];
+    sidebar: Record<string, Record<string, string>>;
+    syllabus: Syllabus | null;
+    readmeFrontmatters: Record<string, Record<string, ReadmeFrontmatter>>;
+    meta: PackageResult["sources"];
+};
+export declare function streamToString(body: unknown): Promise<string>;
+export declare function withRetry<T>(fn: () => Promise<T>, attempts?: number, attempt?: number): Promise<T>;
+export declare function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
+export declare function listPublishedSlugs(s3: AwsClient, bucket: string): Promise<string[]>;
+export declare function fetchJsonObject<T>(s3: AwsClient, bucket: string, key: string): Promise<T | null>;
+export declare function fetchTextObject(s3: AwsClient, bucket: string, key: string): Promise<string | null>;
+export declare function listObjectKeys(s3: AwsClient, bucket: string, prefix: string): Promise<string[]>;
+export declare function fetchPackageSources(s3: AwsClient, bucket: string, slug: string): Promise<PackageSources>;
+export declare function buildManifestFromS3Sources(slug: string, sources: PackageSources, options: {
+    publishedAt: string | null;
+    generatedAt?: string;
+}): PackageManifest;
+export declare function processPackage(s3: AwsClient, bucket: string, slug: string, options: Pick<BackfillOptions, "dryRun" | "force" | "skipExisting" | "publishedAt">): Promise<PackageResult>;
+export declare function invalidatePackageManifestPaths(cf: AwsClient, distributionId: string, paths: string[]): Promise<{
+    id: string;
+}>;
+export declare function runBatch(options: BackfillOptions, clients?: {
+    s3?: AwsClient;
+    cf?: AwsClient;
+    onProgress?: (index: number, total: number, result: PackageResult) => void;
+}): Promise<BatchReport>;
+export declare function getBatchExitCode(report: BatchReport): number;
+export {};