@learnpack/learnpack 5.0.353 → 5.0.354

package/lib/scripts/descriptionsS3Backfill.js

@@ -15,34 +15,59 @@ Object.defineProperty(exports, "__esModule", { value: true });
  *     advance through the catalogue.
  *   - Or target a single course with --slug (remediation / on-demand).
  *
- * Writes descriptions into the published initialSyllabus.json. Manifest
- * projection is a separate pass here: run the package-manifest backfill
- * (runBatch) afterwards, which is cheaper than re-projecting course by course.
+ * Writes descriptions into the published initialSyllabus.json. By default the
+ * manifest projection is a separate pass: run the package-manifest backfill
+ * (runBatch) afterwards, which over the whole catalogue is cheaper than
+ * re-projecting course by course.
  *
- * Secrets/infra via env: RIGOBOT_SYSTEM_TOKEN (required), plus S3_PACKAGES_BUCKET
- * and AWS_REGION as fallbacks for --s3-bucket / --region.
+ * That default inverts for small, targeted runs, where three flags reproduce per
+ * course what a publication does, in one process instead of three:
+ *
+ *   --reproject-manifest  projects each course as it is described (the same
+ *                         `processPackage` the separate pass runs, plus a
+ *                         CloudFront invalidation);
+ *   --mirror-draft        copies the new descriptions back into the GCS draft,
+ *                         so the next publication does not regenerate them;
+ *   --emit-events         announces the result to breathecode.
+ *
+ * Secrets/infra via env: RIGOBOT_SYSTEM_TOKEN (required), BREATHECODE_SYSTEM_TOKEN
+ * (required by --emit-events), GCP_CREDENTIALS_JSON and GCP_BUCKET_NAME (required
+ * by --mirror-draft), plus S3_PACKAGES_BUCKET, AWS_REGION and
+ * CLOUDFRONT_DISTRIBUTION_ID as fallbacks/optionals.
  *
  * Per-run flags:
- *   --s3-bucket <name>   S3 bucket (or env S3_PACKAGES_BUCKET, default learnpack-paquetes)
- *   --region <region>    AWS region (default us-east-1, or env AWS_REGION)
- *   --slug <slug>        process a single course (remediation / on-demand)
- *   --limit <n>          courses per run (default 50)
- *   --target-words <n>   words per description (default 25)
- *   --concurrency <n>    completions in flight per course (default 5)
- *   --dry-run            preview without writing/generating
- *   --no-reconcile       skip the additive syllabus reconciliation
- *   --force              regenerate even courses already settled at this prompt version
+ *   --s3-bucket <name>    S3 bucket (or env S3_PACKAGES_BUCKET, default learnpack-paquetes)
+ *   --gcs-bucket <name>   GCS draft bucket for --mirror-draft (or env GCP_BUCKET_NAME)
+ *   --region <region>     AWS region (default us-east-1, or env AWS_REGION)
+ *   --slug <slug>         process a single course (remediation / on-demand)
+ *   --limit <n>           courses per run (default 50)
+ *   --target-words <n>    words per description (default 25)
+ *   --concurrency <n>     completions in flight per course (default 5)
+ *   --dry-run             preview without writing/generating
+ *   --no-reconcile        skip the additive syllabus reconciliation
+ *   --force               regenerate even courses already settled at this prompt version
+ *   --reproject-manifest  re-project package-manifest.json per course, and invalidate it
+ *   --mirror-draft        mirror the descriptions back into the GCS draft
+ *   --emit-events         send package_manifest_updated per course (implies --reproject-manifest)
  *
  * Examples:
  *   node lib/scripts/descriptionsS3Backfill.js --limit 50
  *   node lib/scripts/descriptionsS3Backfill.js --slug my-course
  *   node lib/scripts/descriptionsS3Backfill.js --dry-run --limit 10
+ *   node lib/scripts/descriptionsS3Backfill.js --slug my-course \
+ *     --reproject-manifest --mirror-draft --emit-events
  */
 const node_util_1 = require("node:util");
 const client_s3_1 = require("@aws-sdk/client-s3");
+const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
+const storage_1 = require("@google-cloud/storage");
 const packageManifest_1 = require("../utils/packageManifest");
 const generateCourseDescriptions_1 = require("../utils/descriptions/generateCourseDescriptions");
 const s3Storage_1 = require("../utils/descriptions/s3Storage");
+const backfillEvents_1 = require("../utils/descriptions/backfillEvents");
+const gcsStorage_1 = require("../utils/descriptions/gcsStorage");
+const mirrorDescriptions_1 = require("../utils/descriptions/mirrorDescriptions");
+const gcsBucketName_1 = require("../utils/gcsBucketName");
 const workList_1 = require("../utils/descriptions/workList");
 const SYLLABUS_KEY_PATTERN = /^([^/]+)\/\.learn\/initialSyllabus\.json$/;
 /** Courses (by slug) ordered by initialSyllabus.json LastModified, newest first. */
@@ -67,11 +92,28 @@ async function listCoursesByRecency(s3, bucket) {
         .sort((a, b) => b[1] - a[1])
         .map(([slug]) => slug);
 }
+/**
+ * CDN invalidation for the re-projected manifests. Optional: a stale edge cache
+ * expires on its own, so a missing distribution id degrades the run instead of
+ * stopping it.
+ */
+function cloudFrontFor(region) {
+    const distributionId = (process.env.CLOUDFRONT_DISTRIBUTION_ID || "").trim();
+    if (!distributionId) {
+        console.warn("[s3-backfill] CLOUDFRONT_DISTRIBUTION_ID is not set: manifests will be re-projected but not invalidated");
+        return undefined;
+    }
+    return {
+        client: new client_cloudfront_1.CloudFrontClient({ region }),
+        distributionId,
+    };
+}
 async function main() {
-    var _a, _b, _c;
+    var _a, _b, _c, _d, _e, _f;
     const { values } = (0, node_util_1.parseArgs)({
         options: {
             "s3-bucket": { type: "string" },
+            "gcs-bucket": { type: "string" },
             region: { type: "string" },
             slug: { type: "string" },
             limit: { type: "string" },
@@ -80,6 +122,9 @@ async function main() {
             "dry-run": { type: "boolean", default: false },
             "no-reconcile": { type: "boolean", default: false },
             force: { type: "boolean", default: false },
+            "reproject-manifest": { type: "boolean", default: false },
+            "mirror-draft": { type: "boolean", default: false },
+            "emit-events": { type: "boolean", default: false },
         },
     });
     const token = process.env.RIGOBOT_SYSTEM_TOKEN;
@@ -94,19 +139,61 @@ async function main() {
     const limit = Number.parseInt(values.limit || "50", 10) || 50;
     const dryRun = (_a = values["dry-run"]) !== null && _a !== void 0 ? _a : false;
     const force = (_b = values.force) !== null && _b !== void 0 ? _b : false;
+    const emitEvents = (_c = values["emit-events"]) !== null && _c !== void 0 ? _c : false;
+    const reprojectManifest = (_d = values["reproject-manifest"]) !== null && _d !== void 0 ? _d : false;
+    const mirrorDraft = (_e = values["mirror-draft"]) !== null && _e !== void 0 ? _e : false;
+    // The event announces that the manifest changed, so emitting it without
+    // projecting the manifest would state something untrue. Required rather than
+    // implied: turning on a write to S3 and the CDN as a side effect of another
+    // flag is exactly the kind of surprise a backfill should not have.
+    if (emitEvents && !reprojectManifest) {
+        console.error("[s3-backfill] --emit-events requires --reproject-manifest: the event " +
+            "reports a manifest update that would not have happened");
+        process.exit(1);
+    }
+    const breathecodeToken = process.env.BREATHECODE_SYSTEM_TOKEN;
+    if (emitEvents && !breathecodeToken) {
+        console.error("[s3-backfill] BREATHECODE_SYSTEM_TOKEN (env) is required by --emit-events");
+        process.exit(1);
+    }
+    if (emitEvents && dryRun) {
+        console.error("[s3-backfill] --emit-events cannot be combined with --dry-run: there is " +
+            "no dry run of an event that breathecode already received");
+        process.exit(1);
+    }
+    // Built up front, not on first use: a misconfigured draft bucket must fail at
+    // boot, not after the first course has already been billed to Rigobot.
+    let draftStorage;
+    if (mirrorDraft) {
+        const credentialsEnv = process.env.GCP_CREDENTIALS_JSON;
+        if (!credentialsEnv) {
+            console.error("[s3-backfill] GCP_CREDENTIALS_JSON (env) is required by --mirror-draft");
+            process.exit(1);
+        }
+        draftStorage = (0, gcsStorage_1.createGcsDescriptionsStorage)(new storage_1.Storage({ credentials: JSON.parse(credentialsEnv) }).bucket(values["gcs-bucket"] || (0, gcsBucketName_1.requireGcsBucketName)()));
+    }
     const s3 = new client_s3_1.S3Client({ region });
-    // Manifests are projected in a separate pass after this backfill.
-    const storage = (0, s3Storage_1.createS3DescriptionsStorage)(s3, bucket, {
-        reprojectManifest: false,
-    });
+    const storage = (0, s3Storage_1.createS3DescriptionsStorage)(s3, bucket, 
+    // Off by default: over the whole catalogue manifests are cheaper to project
+    // in a single later pass than course by course.
+    reprojectManifest ?
+        { cloudFront: cloudFrontFor(region) } :
+        { reprojectManifest: false });
+    const runId = (0, backfillEvents_1.newBackfillRunId)();
     const singleSlug = values.slug;
     const slugs = singleSlug ?
         [singleSlug] :
         await listCoursesByRecency(s3, bucket);
-    console.log(`[s3-backfill] Starting${singleSlug ? ` (slug=${singleSlug})` : ` (limit=${limit})`}${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)`);
+    console.log(`[s3-backfill] Starting${singleSlug ? ` (slug=${singleSlug})` : ` (limit=${limit})`}${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)${reprojectManifest ? ", re-projecting manifests" : ""}${mirrorDraft ? ", mirroring into the draft" : ""}${emitEvents ? `, run ${runId}` : ""}`);
     let processed = 0;
     let generated = 0;
     let failed = 0;
+    let mirrored = 0;
+    const events = {
+        skipped: 0,
+        delivered: 0,
+        failed: 0,
+    };
     for (const slug of slugs) {
         if (!singleSlug && processed >= limit) {
             break;
@@ -127,7 +214,7 @@ async function main() {
                 token,
                 dryRun,
                 force,
-                reconcile: !((_c = values["no-reconcile"]) !== null && _c !== void 0 ? _c : false),
+                reconcile: !((_f = values["no-reconcile"]) !== null && _f !== void 0 ? _f : false),
                 concurrency: Number.parseInt(values.concurrency || "", 10) ||
                     generateCourseDescriptions_1.DEFAULT_DESCRIPTIONS_CONCURRENCY,
                 targetWordCount: Number.parseInt(values["target-words"] || "", 10) ||
@@ -140,14 +227,47 @@ async function main() {
             generated += result.generated;
             failed += result.failed;
             console.log(`[s3-backfill] "${slug}": ${result.steps} step(s), ${result.generated} description(s)${dryRun ? " (dry-run)" : ""}${result.failed > 0 ? `, ${result.failed} failed` : ""}${result.missing > 0 ? `, ${result.missing} unanswered` : ""}`);
+            if (draftStorage) {
+                // Before the event, so a course is fully settled by the time it is
+                // announced. A failure here does NOT change the event status: the
+                // published package and its manifest are already correct, and only the
+                // draft lags behind — the next publication regenerates what is missing.
+                try {
+                    const published = await storage.readSyllabus(slug);
+                    const mirror = await (0, mirrorDescriptions_1.mirrorDescriptionsToDraft)(draftStorage, slug, published, { dryRun });
+                    mirrored += mirror.copied;
+                    console.log(`[s3-backfill] "${slug}": ${mirror.copied} mirrored, ${mirror.missed} missed, ${mirror.fresh} already fresh`);
+                }
+                catch (error) {
+                    console.error(`[s3-backfill] Could not mirror "${slug}" into the draft:`, error.message);
+                }
+            }
+            if (emitEvents) {
+                const outcome = await (0, backfillEvents_1.emitBackfillManifestEvent)({
+                    courseSlug: slug,
+                    runId,
+                    result,
+                    storage,
+                    rigobotToken: token,
+                    breathecodeToken: breathecodeToken,
+                });
+                events[outcome] += 1;
+                console.log(`[s3-backfill] "${slug}": manifest event ${outcome}`);
+            }
         }
         catch (error) {
             processed += 1;
             failed += 1;
+            // No event here on purpose: unlike a publication, a backfill never
+            // announced one, so nothing is owed. The course is simply re-run.
             console.error(`[s3-backfill] Failed processing "${slug}":`, error.message);
         }
     }
-    console.log(`[s3-backfill] Done. ${processed} course(s), ${generated} description(s) written, ${failed} failure(s).`);
+    console.log(`[s3-backfill] Done. ${processed} course(s), ${generated} description(s) written, ${failed} failure(s).` +
+        (mirrorDraft ? ` ${mirrored} mirrored into the draft.` : "") +
+        (emitEvents ?
+            ` Events: ${events.delivered} delivered, ${events.failed} failed (run ${runId}).` :
+            ""));
 }
 main()
     .then(() => process.exit(0))

package/lib/utils/awsCredentials.d.ts

@@ -0,0 +1,20 @@
+/**
+ * AWS credentials for the published bucket, required rather than implicit.
+ *
+ * Building an `S3Client` with no credentials succeeds: the SDK defers to its
+ * provider chain and only fails at the first request, with
+ * "Could not load credentials from any providers". Inside a background job that
+ * surfaces minutes later, detached from the cause. Validating up front turns it
+ * into an obvious misconfiguration.
+ *
+ * Note this deliberately rejects the empty-string fallback pattern
+ * (`process.env.X || ""`): passing empty credentials defeats the provider chain
+ * instead of failing, which is how the opaque error appeared in the first place.
+ */
+export type AwsCredentials = {
+    accessKeyId: string;
+    secretAccessKey: string;
+};
+export declare function requireAwsCredentials(): AwsCredentials;
+export declare function requireS3PackagesBucket(): string;
+export declare function awsRegion(): string;

package/lib/utils/awsCredentials.js

@@ -0,0 +1,43 @@
+"use strict";
+/**
+ * AWS credentials for the published bucket, required rather than implicit.
+ *
+ * Building an `S3Client` with no credentials succeeds: the SDK defers to its
+ * provider chain and only fails at the first request, with
+ * "Could not load credentials from any providers". Inside a background job that
+ * surfaces minutes later, detached from the cause. Validating up front turns it
+ * into an obvious misconfiguration.
+ *
+ * Note this deliberately rejects the empty-string fallback pattern
+ * (`process.env.X || ""`): passing empty credentials defeats the provider chain
+ * instead of failing, which is how the opaque error appeared in the first place.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.requireAwsCredentials = requireAwsCredentials;
+exports.requireS3PackagesBucket = requireS3PackagesBucket;
+exports.awsRegion = awsRegion;
+function requireAwsCredentials() {
+    const accessKeyId = (process.env.AWS_ACCESS_KEY_ID || "").trim();
+    const secretAccessKey = (process.env.AWS_SECRET_ACCESS_KEY || "").trim();
+    const missing = [];
+    if (!accessKeyId) {
+        missing.push("AWS_ACCESS_KEY_ID");
+    }
+    if (!secretAccessKey) {
+        missing.push("AWS_SECRET_ACCESS_KEY");
+    }
+    if (missing.length > 0) {
+        throw new Error(`${missing.join(" and ")} (env) ${missing.length > 1 ? "are" : "is"} required to reach the published package bucket`);
+    }
+    return { accessKeyId, secretAccessKey };
+}
+function requireS3PackagesBucket() {
+    const bucket = (process.env.S3_PACKAGES_BUCKET || "").trim();
+    if (!bucket) {
+        throw new Error("S3_PACKAGES_BUCKET (env) is required: it names the bucket holding published packages");
+    }
+    return bucket;
+}
+function awsRegion() {
+    return (process.env.AWS_REGION || "").trim() || "us-east-1";
+}

package/lib/utils/descriptions/backfillEvents.d.ts

@@ -0,0 +1,60 @@
+import { fetchPackageInfo, sendPublishEvent } from "../publishEvents";
+import { CourseDescriptionsStorage, GenerateCourseDescriptionsResult } from "./generateCourseDescriptions";
+/**
+ * `package_manifest_updated` for courses reconciled by the backfill.
+ *
+ * The backfill regenerates descriptions of packages that are already published,
+ * so breathecode needs the same notification a publication sends — but there is
+ * no publication here: no `package_published` precedes this event, and no
+ * journal tracks it (the sweep resumes publications, and a backfill is simply
+ * re-run instead).
+ *
+ * That is what makes `publish_id` awkward. The field exists to correlate the two
+ * events of one publication, a pair this event has no half of. Since it is
+ * required, the value has to be something that cannot be mistaken for a real
+ * publication: a bare uuid — the obvious choice — would be exactly that mistake,
+ * and if breathecode ever starts pairing events it would search forever for a
+ * `package_published` that was never sent.
+ *
+ * Hence `backfill-{runId}-{slug}`:
+ *
+ * - the `backfill-` prefix makes the origin readable in the webhook log and
+ *   discriminable in code (`isBackfillPublishId`); real ids are bare uuids from
+ *   `createJournal`, so the two spaces cannot collide;
+ * - `{slug}` keeps it unique per event, which a per-run id alone would not be;
+ * - a `{runId}` shared by every course of one invocation makes "every event from
+ *   the run I fired at 15:40" a single substring query, which is the question
+ *   you actually ask when the catalogue is backfilled in small batches.
+ *
+ * Encoding this in `publish_id`, rather than adding an `origin` field, keeps the
+ * event contract untouched: breathecode ignores `publish_id` today, so this
+ * costs them nothing and needs no coordination. If they ever start consuming the
+ * provenance, it can be promoted to a field of its own.
+ */
+export declare const BACKFILL_PUBLISH_ID_PREFIX = "backfill-";
+/** One per script invocation, shared by every course it touches. */
+export declare function newBackfillRunId(): string;
+export declare function backfillPublishId(runId: string, courseSlug: string): string;
+export declare function isBackfillPublishId(publishId: string): boolean;
+/**
+ * `skipped` means the event was never attempted, which is not a failure: a
+ * course the backfill did not change has nothing to announce.
+ */
+export type BackfillEventOutcome = "skipped" | "delivered" | "failed";
+export type EmitBackfillManifestEventParams = {
+    courseSlug: string;
+    runId: string;
+    result: GenerateCourseDescriptionsResult;
+    storage: CourseDescriptionsStorage;
+    rigobotToken: string;
+    breathecodeToken: string;
+    /** Injection points for tests. */
+    emit?: typeof sendPublishEvent;
+    readPackageInfo?: typeof fetchPackageInfo;
+};
+/**
+ * Announce one reconciled course. Never throws: the descriptions and the
+ * manifest are already written by the time this runs, and losing the
+ * notification must not turn a successful course into a failed one.
+ */
+export declare function emitBackfillManifestEvent(params: EmitBackfillManifestEventParams): Promise<BackfillEventOutcome>;

package/lib/utils/descriptions/backfillEvents.js

@@ -0,0 +1,107 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BACKFILL_PUBLISH_ID_PREFIX = void 0;
+exports.newBackfillRunId = newBackfillRunId;
+exports.backfillPublishId = backfillPublishId;
+exports.isBackfillPublishId = isBackfillPublishId;
+exports.emitBackfillManifestEvent = emitBackfillManifestEvent;
+const uuid_1 = require("uuid");
+const publishEvents_1 = require("../publishEvents");
+/**
+ * `package_manifest_updated` for courses reconciled by the backfill.
+ *
+ * The backfill regenerates descriptions of packages that are already published,
+ * so breathecode needs the same notification a publication sends — but there is
+ * no publication here: no `package_published` precedes this event, and no
+ * journal tracks it (the sweep resumes publications, and a backfill is simply
+ * re-run instead).
+ *
+ * That is what makes `publish_id` awkward. The field exists to correlate the two
+ * events of one publication, a pair this event has no half of. Since it is
+ * required, the value has to be something that cannot be mistaken for a real
+ * publication: a bare uuid — the obvious choice — would be exactly that mistake,
+ * and if breathecode ever starts pairing events it would search forever for a
+ * `package_published` that was never sent.
+ *
+ * Hence `backfill-{runId}-{slug}`:
+ *
+ * - the `backfill-` prefix makes the origin readable in the webhook log and
+ *   discriminable in code (`isBackfillPublishId`); real ids are bare uuids from
+ *   `createJournal`, so the two spaces cannot collide;
+ * - `{slug}` keeps it unique per event, which a per-run id alone would not be;
+ * - a `{runId}` shared by every course of one invocation makes "every event from
+ *   the run I fired at 15:40" a single substring query, which is the question
+ *   you actually ask when the catalogue is backfilled in small batches.
+ *
+ * Encoding this in `publish_id`, rather than adding an `origin` field, keeps the
+ * event contract untouched: breathecode ignores `publish_id` today, so this
+ * costs them nothing and needs no coordination. If they ever start consuming the
+ * provenance, it can be promoted to a field of its own.
+ */
+exports.BACKFILL_PUBLISH_ID_PREFIX = "backfill-";
+/** One per script invocation, shared by every course it touches. */
+function newBackfillRunId() {
+    return (0, uuid_1.v4)();
+}
+function backfillPublishId(runId, courseSlug) {
+    return `${exports.BACKFILL_PUBLISH_ID_PREFIX}${runId}-${courseSlug}`;
+}
+function isBackfillPublishId(publishId) {
+    return publishId.startsWith(exports.BACKFILL_PUBLISH_ID_PREFIX);
+}
+/**
+ * Announce one reconciled course. Never throws: the descriptions and the
+ * manifest are already written by the time this runs, and losing the
+ * notification must not turn a successful course into a failed one.
+ */
+async function emitBackfillManifestEvent(params) {
+    var _a, _b;
+    const { courseSlug, result } = params;
+    if (result.status === "skipped") {
+        return "skipped";
+    }
+    const emit = (_a = params.emit) !== null && _a !== void 0 ? _a : publishEvents_1.sendPublishEvent;
+    const readPackageInfo = (_b = params.readPackageInfo) !== null && _b !== void 0 ? _b : publishEvents_1.fetchPackageInfo;
+    // Enriching the payload must never prevent the delivery: both the manifest and
+    // the package info are optional in the contract, and an event with less
+    // context beats an event that never arrives.
+    let manifest = null;
+    try {
+        if (params.storage.readManifest) {
+            manifest = await params.storage.readManifest(courseSlug);
+        }
+    }
+    catch (error) {
+        console.error(`[backfill-events] Could not read the manifest of "${courseSlug}", reporting without it:`, error.message);
+    }
+    let packageInfo = null;
+    try {
+        packageInfo = await readPackageInfo(courseSlug, params.rigobotToken);
+    }
+    catch (error) {
+        console.error(`[backfill-events] Could not read the package "${courseSlug}", reporting without it:`, error.message);
+    }
+    try {
+        const delivered = await emit((0, publishEvents_1.buildPackageManifestUpdatedEvent)({
+            publishId: backfillPublishId(params.runId, courseSlug),
+            courseSlug,
+            packageInfo,
+            manifest,
+        }, 
+        // A failed projection does not fail the run — the syllabus is saved
+        // either way — but this event reports the manifest, so it cannot claim
+        // success over one that was never rewritten.
+        result.status === "failed" || result.manifestProjected === false ?
+            "failed" :
+            "success", {
+            generated: result.generated,
+            failed: result.failed,
+            missing: result.missing,
+        }), params.breathecodeToken);
+        return delivered ? "delivered" : "failed";
+    }
+    catch (error) {
+        console.error(`[backfill-events] Could not deliver the manifest event for "${courseSlug}":`, error.message);
+        return "failed";
+    }
+}

package/lib/utils/descriptions/generateCourseDescriptions.d.ts

@@ -53,6 +53,13 @@ export type GenerateCourseDescriptionsResult = {
     errors: string[];
     /** Completion durations, to watch how close we run to the request limits. */
     durationsSeconds: number[];
+    /**
+     * Whether `package-manifest.json` was re-projected; null when the storage does
+     * not project at all. A failed projection is not a failed run — the syllabus,
+     * which is the source of truth, is already saved — but callers that announce
+     * the manifest downstream must not report success on it.
+     */
+    manifestProjected: boolean | null;
 };
 /** Read every README of the course, keyed by exercise slug and language. */
 export declare function readCourseReadmes(storage: CourseDescriptionsStorage, courseSlug: string, exercises: CourseExercise[], concurrency: number): Promise<CourseReadmes>;

package/lib/utils/descriptions/generateCourseDescriptions.js

@@ -29,6 +29,7 @@ function emptyResult(status) {
         missing: 0,
         errors: [],
         durationsSeconds: [],
+        manifestProjected: null,
     };
 }
 /** Read every README of the course, keyed by exercise slug and language. */
@@ -162,10 +163,12 @@ async function generateCourseDescriptions(storage, courseSlug, options) {
     if (storage.reprojectManifest) {
         try {
             await storage.reprojectManifest(courseSlug);
+            result.manifestProjected = true;
         }
         catch (error) {
             // The syllabus (the source of truth) is already saved; a failed
             // projection is recoverable and must not lose the generated text.
+            result.manifestProjected = false;
             result.errors.push(`manifest projection failed: ${error.message}`);
         }
     }

package/lib/utils/descriptions/publishStage.js

@@ -212,10 +212,21 @@ async function emitManifestUpdated(params) {
         });
         return;
     }
+    // Enriching the payload must never prevent the delivery. The manifest is
+    // optional in the contract (`manifest: null` is valid); the event is not:
+    // breathecode was promised one and is waiting for it. Reading the manifest
+    // from the published bucket can fail for reasons that have nothing to do with
+    // the event — a bad AWS credential, say — and an event with less context still
+    // beats an event that never arrives.
+    let updatedManifest = params.eventContext.manifest;
+    try {
+        updatedManifest =
+            (_c = (await ((_b = (_a = params.publishedStorage).readManifest) === null || _b === void 0 ? void 0 : _b.call(_a, params.courseSlug)))) !== null && _c !== void 0 ? _c : params.eventContext.manifest;
+    }
+    catch (error) {
+        console.error(`[descriptions] Could not read the updated manifest for "${params.courseSlug}", reporting without it:`, error.message);
+    }
     try {
-        // Report the manifest as it is NOW: the descriptions are the whole point of
-        // this event, so the one captured at publish time would be stale.
-        const updatedManifest = (_c = (await ((_b = (_a = params.publishedStorage).readManifest) === null || _b === void 0 ? void 0 : _b.call(_a, params.courseSlug)))) !== null && _c !== void 0 ? _c : params.eventContext.manifest;
         const generation = params.outcome.generation;
         const delivered = await params.emit((0, publishEvents_1.buildPackageManifestUpdatedEvent)(Object.assign(Object.assign({}, params.eventContext), { manifest: updatedManifest }), params.outcome.succeeded ? "success" : "failed", generation ?
             {

package/lib/utils/descriptions/resumePublication.js

@@ -92,15 +92,30 @@ async function emitPendingManifestEvent(journal, deps, succeeded, stats) {
         });
         return;
     }
+    // Enriching the payload must never prevent the delivery: this event is the
+    // one breathecode has been waiting for since the publication stalled, and
+    // both the manifest and the package info are optional in the contract.
+    let manifest = null;
+    let packageInfo = null;
     try {
-        let manifest = null;
         if (deps.publishedStorage.readManifest) {
             manifest = await deps.publishedStorage.readManifest(journal.courseSlug);
         }
+    }
+    catch (error) {
+        console.error(`[sweep] Could not read the manifest of "${journal.courseSlug}", reporting without it:`, error.message);
+    }
+    try {
+        packageInfo = await readPackageInfo(journal.courseSlug, deps.rigobotToken);
+    }
+    catch (error) {
+        console.error(`[sweep] Could not read the package of "${journal.courseSlug}", reporting without it:`, error.message);
+    }
+    try {
         const delivered = await emit((0, publishEvents_1.buildPackageManifestUpdatedEvent)({
             publishId: journal.publishId,
             courseSlug: journal.courseSlug,
-            packageInfo: await readPackageInfo(journal.courseSlug, deps.rigobotToken),
+            packageInfo,
             manifest,
         }, succeeded ? "success" : "failed", stats), deps.breathecodeToken);
         await (0, publishJournal_1.markStage)(deps.journalStorage, journal, "manifestEvent", delivered ? "done" : "failed", { error: delivered ? undefined : "delivery failed" });

package/lib/utils/descriptions/s3Storage.js

@@ -9,6 +9,7 @@ const client_s3_1 = require("@aws-sdk/client-s3");
 const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
 const packageManifest_1 = require("../packageManifest");
 const packageManifestBackfill_1 = require("../s3/packageManifestBackfill");
+const awsCredentials_1 = require("../awsCredentials");
 /**
  * Published-bucket (S3) adapter for the description flows.
  *
@@ -68,14 +69,18 @@ function createS3SyllabusSyncStorage(s3, bucket) {
  * post-publish stage and the sweep, which have no CLI flags to read.
  */
 function createS3DescriptionsStorageFromEnv() {
-    const s3 = new client_s3_1.S3Client({
-        region: process.env.AWS_REGION || "us-east-1",
-    });
-    const distributionId = process.env.CLOUDFRONT_DISTRIBUTION_ID;
+    // Validated here so a missing variable fails with its own name, instead of
+    // surfacing minutes later as "Could not load credentials from any providers"
+    // from inside a background job.
+    const credentials = (0, awsCredentials_1.requireAwsCredentials)();
+    const region = (0, awsCredentials_1.awsRegion)();
+    const s3 = new client_s3_1.S3Client({ region, credentials });
+    const distributionId = (process.env.CLOUDFRONT_DISTRIBUTION_ID || "").trim();
     const cloudFront = distributionId ?
         {
             client: new client_cloudfront_1.CloudFrontClient({
-                region: process.env.AWS_REGION || "us-east-1",
+                region,
+                credentials,
             }),
             distributionId,
         } :
@@ -83,7 +88,9 @@ function createS3DescriptionsStorageFromEnv() {
     if (!cloudFront) {
         console.warn("[descriptions] CLOUDFRONT_DISTRIBUTION_ID is not set: the manifest will be re-projected but not invalidated");
     }
-    return createS3DescriptionsStorage(s3, process.env.S3_PACKAGES_BUCKET || "learnpack-paquetes", { cloudFront });
+    return createS3DescriptionsStorage(s3, (0, awsCredentials_1.requireS3PackagesBucket)(), {
+        cloudFront,
+    });
 }
 function createS3DescriptionsStorage(s3, bucket, options = {}) {
     const storage = {

package/lib/utils/packageManifest.d.ts

@@ -48,6 +48,14 @@ export type PackageManifestBuildInput = {
 };
 export declare function packageManifestGcsPath(courseSlug: string): string;
 export declare function resolvePreviewUrl(slug: string, preview?: string): string;
+/**
+ * Lesson id when no syllabus entry matches: the numeric prefix of the folder.
+ *
+ * Exported because `legacyPackageRepair` has to synthesize syllabus ids with
+ * exactly this rule — a repaired package whose ids differed from the ones its
+ * manifest already carries would look like a content change to every consumer.
+ */
+export declare function deriveLessonId(exerciseSlug: string): string;
 export declare function extractReadmeH1(body: string): string | null;
 export declare function normalizeReadmeTitle(raw: string): string | null;
 export declare function buildPackageManifestFromSources(input: PackageManifestBuildInput): PackageManifest;

package/lib/utils/packageManifest.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DESCRIPTION_TARGET_WORD_COUNT = exports.DESCRIPTION_PROMPT_VERSION = exports.SCHEMA_VERSION = exports.PACKAGE_MANIFEST_REL_PATH = exports.PACKAGE_MANIFEST_FILENAME = void 0;
 exports.packageManifestGcsPath = packageManifestGcsPath;
 exports.resolvePreviewUrl = resolvePreviewUrl;
+exports.deriveLessonId = deriveLessonId;
 exports.extractReadmeH1 = extractReadmeH1;
 exports.normalizeReadmeTitle = normalizeReadmeTitle;
 exports.buildPackageManifestFromSources = buildPackageManifestFromSources;
@@ -49,6 +50,13 @@ function findSyllabusLesson(exerciseSlug, lessons) {
         return candidates.includes(exerciseSlug);
     });
 }
+/**
+ * Lesson id when no syllabus entry matches: the numeric prefix of the folder.
+ *
+ * Exported because `legacyPackageRepair` has to synthesize syllabus ids with
+ * exactly this rule — a repaired package whose ids differed from the ones its
+ * manifest already carries would look like a content change to every consumer.
+ */
 function deriveLessonId(exerciseSlug) {
     const match = exerciseSlug.match(/^(\d+(?:\.\d+)?)/);
     return match ? match[1] : exerciseSlug;