@learnpack/learnpack 5.0.353 → 5.0.354

package/src/scripts/descriptionsS3Backfill.ts

@@ -13,31 +13,52 @@
  *     advance through the catalogue.
  *   - Or target a single course with --slug (remediation / on-demand).
  *
- * Writes descriptions into the published initialSyllabus.json. Manifest
- * projection is a separate pass here: run the package-manifest backfill
- * (runBatch) afterwards, which is cheaper than re-projecting course by course.
+ * Writes descriptions into the published initialSyllabus.json. By default the
+ * manifest projection is a separate pass: run the package-manifest backfill
+ * (runBatch) afterwards, which over the whole catalogue is cheaper than
+ * re-projecting course by course.
  *
- * Secrets/infra via env: RIGOBOT_SYSTEM_TOKEN (required), plus S3_PACKAGES_BUCKET
- * and AWS_REGION as fallbacks for --s3-bucket / --region.
+ * That default inverts for small, targeted runs, where three flags reproduce per
+ * course what a publication does, in one process instead of three:
+ *
+ *   --reproject-manifest  projects each course as it is described (the same
+ *                         `processPackage` the separate pass runs, plus a
+ *                         CloudFront invalidation);
+ *   --mirror-draft        copies the new descriptions back into the GCS draft,
+ *                         so the next publication does not regenerate them;
+ *   --emit-events         announces the result to breathecode.
+ *
+ * Secrets/infra via env: RIGOBOT_SYSTEM_TOKEN (required), BREATHECODE_SYSTEM_TOKEN
+ * (required by --emit-events), GCP_CREDENTIALS_JSON and GCP_BUCKET_NAME (required
+ * by --mirror-draft), plus S3_PACKAGES_BUCKET, AWS_REGION and
+ * CLOUDFRONT_DISTRIBUTION_ID as fallbacks/optionals.
  *
  * Per-run flags:
- *   --s3-bucket <name>   S3 bucket (or env S3_PACKAGES_BUCKET, default learnpack-paquetes)
- *   --region <region>    AWS region (default us-east-1, or env AWS_REGION)
- *   --slug <slug>        process a single course (remediation / on-demand)
- *   --limit <n>          courses per run (default 50)
- *   --target-words <n>   words per description (default 25)
- *   --concurrency <n>    completions in flight per course (default 5)
- *   --dry-run            preview without writing/generating
- *   --no-reconcile       skip the additive syllabus reconciliation
- *   --force              regenerate even courses already settled at this prompt version
+ *   --s3-bucket <name>    S3 bucket (or env S3_PACKAGES_BUCKET, default learnpack-paquetes)
+ *   --gcs-bucket <name>   GCS draft bucket for --mirror-draft (or env GCP_BUCKET_NAME)
+ *   --region <region>     AWS region (default us-east-1, or env AWS_REGION)
+ *   --slug <slug>         process a single course (remediation / on-demand)
+ *   --limit <n>           courses per run (default 50)
+ *   --target-words <n>    words per description (default 25)
+ *   --concurrency <n>     completions in flight per course (default 5)
+ *   --dry-run             preview without writing/generating
+ *   --no-reconcile        skip the additive syllabus reconciliation
+ *   --force               regenerate even courses already settled at this prompt version
+ *   --reproject-manifest  re-project package-manifest.json per course, and invalidate it
+ *   --mirror-draft        mirror the descriptions back into the GCS draft
+ *   --emit-events         send package_manifest_updated per course (implies --reproject-manifest)
  *
  * Examples:
  *   node lib/scripts/descriptionsS3Backfill.js --limit 50
  *   node lib/scripts/descriptionsS3Backfill.js --slug my-course
  *   node lib/scripts/descriptionsS3Backfill.js --dry-run --limit 10
+ *   node lib/scripts/descriptionsS3Backfill.js --slug my-course \
+ *     --reproject-manifest --mirror-draft --emit-events
  */
 import { parseArgs } from "node:util"
 import { ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3"
+import { CloudFrontClient } from "@aws-sdk/client-cloudfront"
+import { Storage } from "@google-cloud/storage"
 import { DESCRIPTION_TARGET_WORD_COUNT } from "../utils/packageManifest"
 import { AwsClient } from "../utils/s3/packageManifestBackfill"
 import {
@@ -47,7 +68,16 @@ import {
 import {
   createS3DescriptionsStorage,
   fetchExercises,
+  S3DescriptionsStorageOptions,
 } from "../utils/descriptions/s3Storage"
+import {
+  BackfillEventOutcome,
+  emitBackfillManifestEvent,
+  newBackfillRunId,
+} from "../utils/descriptions/backfillEvents"
+import { createGcsDescriptionsStorage } from "../utils/descriptions/gcsStorage"
+import { mirrorDescriptionsToDraft } from "../utils/descriptions/mirrorDescriptions"
+import { requireGcsBucketName } from "../utils/gcsBucketName"
 import { isCourseSettled } from "../utils/descriptions/workList"
 
 const SYLLABUS_KEY_PATTERN = /^([^/]+)\/\.learn\/initialSyllabus\.json$/
@@ -86,10 +116,33 @@ async function listCoursesByRecency(
     .map(([slug]) => slug)
 }
 
+/**
+ * CDN invalidation for the re-projected manifests. Optional: a stale edge cache
+ * expires on its own, so a missing distribution id degrades the run instead of
+ * stopping it.
+ */
+function cloudFrontFor(
+  region: string
+): S3DescriptionsStorageOptions["cloudFront"] {
+  const distributionId = (process.env.CLOUDFRONT_DISTRIBUTION_ID || "").trim()
+  if (!distributionId) {
+    console.warn(
+      "[s3-backfill] CLOUDFRONT_DISTRIBUTION_ID is not set: manifests will be re-projected but not invalidated"
+    )
+    return undefined
+  }
+
+  return {
+    client: new CloudFrontClient({ region }) as unknown as AwsClient,
+    distributionId,
+  }
+}
+
 async function main(): Promise<void> {
   const { values } = parseArgs({
     options: {
       "s3-bucket": { type: "string" },
+      "gcs-bucket": { type: "string" },
       region: { type: "string" },
       slug: { type: "string" },
       limit: { type: "string" },
@@ -98,6 +151,9 @@ async function main(): Promise<void> {
       "dry-run": { type: "boolean", default: false },
       "no-reconcile": { type: "boolean", default: false },
       force: { type: "boolean", default: false },
+      "reproject-manifest": { type: "boolean", default: false },
+      "mirror-draft": { type: "boolean", default: false },
+      "emit-events": { type: "boolean", default: false },
     },
   })
 
@@ -116,12 +172,69 @@ async function main(): Promise<void> {
   const limit = Number.parseInt(values.limit || "50", 10) || 50
   const dryRun = values["dry-run"] ?? false
   const force = values.force ?? false
+  const emitEvents = values["emit-events"] ?? false
+  const reprojectManifest = values["reproject-manifest"] ?? false
+  const mirrorDraft = values["mirror-draft"] ?? false
+
+  // The event announces that the manifest changed, so emitting it without
+  // projecting the manifest would state something untrue. Required rather than
+  // implied: turning on a write to S3 and the CDN as a side effect of another
+  // flag is exactly the kind of surprise a backfill should not have.
+  if (emitEvents && !reprojectManifest) {
+    console.error(
+      "[s3-backfill] --emit-events requires --reproject-manifest: the event " +
+        "reports a manifest update that would not have happened"
+    )
+    process.exit(1)
+  }
+
+  const breathecodeToken = process.env.BREATHECODE_SYSTEM_TOKEN
+  if (emitEvents && !breathecodeToken) {
+    console.error(
+      "[s3-backfill] BREATHECODE_SYSTEM_TOKEN (env) is required by --emit-events"
+    )
+    process.exit(1)
+  }
+
+  if (emitEvents && dryRun) {
+    console.error(
+      "[s3-backfill] --emit-events cannot be combined with --dry-run: there is " +
+        "no dry run of an event that breathecode already received"
+    )
+    process.exit(1)
+  }
+
+  // Built up front, not on first use: a misconfigured draft bucket must fail at
+  // boot, not after the first course has already been billed to Rigobot.
+  let draftStorage
+  if (mirrorDraft) {
+    const credentialsEnv = process.env.GCP_CREDENTIALS_JSON
+    if (!credentialsEnv) {
+      console.error(
+        "[s3-backfill] GCP_CREDENTIALS_JSON (env) is required by --mirror-draft"
+      )
+      process.exit(1)
+    }
+
+    draftStorage = createGcsDescriptionsStorage(
+      new Storage({ credentials: JSON.parse(credentialsEnv) }).bucket(
+        values["gcs-bucket"] || requireGcsBucketName()
+      )
+    )
+  }
 
   const s3 = new S3Client({ region }) as unknown as AwsClient
-  // Manifests are projected in a separate pass after this backfill.
-  const storage = createS3DescriptionsStorage(s3, bucket, {
-    reprojectManifest: false,
-  })
+  const storage = createS3DescriptionsStorage(
+    s3,
+    bucket,
+    // Off by default: over the whole catalogue manifests are cheaper to project
+    // in a single later pass than course by course.
+    reprojectManifest ?
+      { cloudFront: cloudFrontFor(region) } :
+      { reprojectManifest: false }
+  )
+
+  const runId = newBackfillRunId()
 
   const singleSlug = values.slug
   const slugs = singleSlug ?
@@ -131,12 +244,22 @@ async function main(): Promise<void> {
   console.log(
     `[s3-backfill] Starting${
       singleSlug ? ` (slug=${singleSlug})` : ` (limit=${limit})`
-    }${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)`
+    }${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)${
+      reprojectManifest ? ", re-projecting manifests" : ""
+    }${mirrorDraft ? ", mirroring into the draft" : ""}${
+      emitEvents ? `, run ${runId}` : ""
+    }`
   )
 
   let processed = 0
   let generated = 0
   let failed = 0
+  let mirrored = 0
+  const events: Record<BackfillEventOutcome, number> = {
+    skipped: 0,
+    delivered: 0,
+    failed: 0,
+  }
 
   for (const slug of slugs) {
     if (!singleSlug && processed >= limit) {
@@ -185,9 +308,50 @@ async function main(): Promise<void> {
           result.failed > 0 ? `, ${result.failed} failed` : ""
         }${result.missing > 0 ? `, ${result.missing} unanswered` : ""}`
       )
+
+      if (draftStorage) {
+        // Before the event, so a course is fully settled by the time it is
+        // announced. A failure here does NOT change the event status: the
+        // published package and its manifest are already correct, and only the
+        // draft lags behind — the next publication regenerates what is missing.
+        try {
+          const published = await storage.readSyllabus(slug)
+          const mirror = await mirrorDescriptionsToDraft(
+            draftStorage,
+            slug,
+            published,
+            { dryRun }
+          )
+          mirrored += mirror.copied
+          console.log(
+            `[s3-backfill] "${slug}": ${mirror.copied} mirrored, ${mirror.missed} missed, ${mirror.fresh} already fresh`
+          )
+        } catch (error) {
+          console.error(
+            `[s3-backfill] Could not mirror "${slug}" into the draft:`,
+            (error as Error).message
+          )
+        }
+      }
+
+      if (emitEvents) {
+        const outcome = await emitBackfillManifestEvent({
+          courseSlug: slug,
+          runId,
+          result,
+          storage,
+          rigobotToken: token,
+          breathecodeToken: breathecodeToken as string,
+        })
+
+        events[outcome] += 1
+        console.log(`[s3-backfill] "${slug}": manifest event ${outcome}`)
+      }
     } catch (error) {
       processed += 1
       failed += 1
+      // No event here on purpose: unlike a publication, a backfill never
+      // announced one, so nothing is owed. The course is simply re-run.
       console.error(
         `[s3-backfill] Failed processing "${slug}":`,
         (error as Error).message
@@ -196,7 +360,11 @@ async function main(): Promise<void> {
   }
 
   console.log(
-    `[s3-backfill] Done. ${processed} course(s), ${generated} description(s) written, ${failed} failure(s).`
+    `[s3-backfill] Done. ${processed} course(s), ${generated} description(s) written, ${failed} failure(s).` +
+      (mirrorDraft ? ` ${mirrored} mirrored into the draft.` : "") +
+      (emitEvents ?
+        ` Events: ${events.delivered} delivered, ${events.failed} failed (run ${runId}).` :
+        "")
   )
 }
 

package/src/utils/awsCredentials.ts

@@ -0,0 +1,57 @@
+/**
+ * AWS credentials for the published bucket, required rather than implicit.
+ *
+ * Building an `S3Client` with no credentials succeeds: the SDK defers to its
+ * provider chain and only fails at the first request, with
+ * "Could not load credentials from any providers". Inside a background job that
+ * surfaces minutes later, detached from the cause. Validating up front turns it
+ * into an obvious misconfiguration.
+ *
+ * Note this deliberately rejects the empty-string fallback pattern
+ * (`process.env.X || ""`): passing empty credentials defeats the provider chain
+ * instead of failing, which is how the opaque error appeared in the first place.
+ */
+
+export type AwsCredentials = {
+  accessKeyId: string;
+  secretAccessKey: string;
+};
+
+export function requireAwsCredentials(): AwsCredentials {
+  const accessKeyId = (process.env.AWS_ACCESS_KEY_ID || "").trim()
+  const secretAccessKey = (process.env.AWS_SECRET_ACCESS_KEY || "").trim()
+
+  const missing: string[] = []
+  if (!accessKeyId) {
+    missing.push("AWS_ACCESS_KEY_ID")
+  }
+
+  if (!secretAccessKey) {
+    missing.push("AWS_SECRET_ACCESS_KEY")
+  }
+
+  if (missing.length > 0) {
+    throw new Error(
+      `${missing.join(" and ")} (env) ${
+        missing.length > 1 ? "are" : "is"
+      } required to reach the published package bucket`
+    )
+  }
+
+  return { accessKeyId, secretAccessKey }
+}
+
+export function requireS3PackagesBucket(): string {
+  const bucket = (process.env.S3_PACKAGES_BUCKET || "").trim()
+  if (!bucket) {
+    throw new Error(
+      "S3_PACKAGES_BUCKET (env) is required: it names the bucket holding published packages"
+    )
+  }
+
+  return bucket
+}
+
+export function awsRegion(): string {
+  return (process.env.AWS_REGION || "").trim() || "us-east-1"
+}

package/src/utils/descriptions/backfillEvents.ts

@@ -0,0 +1,152 @@
+import { v4 as uuidv4 } from "uuid"
+import { PackageManifest } from "../packageManifest"
+import {
+  buildPackageManifestUpdatedEvent,
+  fetchPackageInfo,
+  sendPublishEvent,
+} from "../publishEvents"
+import {
+  CourseDescriptionsStorage,
+  GenerateCourseDescriptionsResult,
+} from "./generateCourseDescriptions"
+
+/**
+ * `package_manifest_updated` for courses reconciled by the backfill.
+ *
+ * The backfill regenerates descriptions of packages that are already published,
+ * so breathecode needs the same notification a publication sends — but there is
+ * no publication here: no `package_published` precedes this event, and no
+ * journal tracks it (the sweep resumes publications, and a backfill is simply
+ * re-run instead).
+ *
+ * That is what makes `publish_id` awkward. The field exists to correlate the two
+ * events of one publication, a pair this event has no half of. Since it is
+ * required, the value has to be something that cannot be mistaken for a real
+ * publication: a bare uuid — the obvious choice — would be exactly that mistake,
+ * and if breathecode ever starts pairing events it would search forever for a
+ * `package_published` that was never sent.
+ *
+ * Hence `backfill-{runId}-{slug}`:
+ *
+ * - the `backfill-` prefix makes the origin readable in the webhook log and
+ *   discriminable in code (`isBackfillPublishId`); real ids are bare uuids from
+ *   `createJournal`, so the two spaces cannot collide;
+ * - `{slug}` keeps it unique per event, which a per-run id alone would not be;
+ * - a `{runId}` shared by every course of one invocation makes "every event from
+ *   the run I fired at 15:40" a single substring query, which is the question
+ *   you actually ask when the catalogue is backfilled in small batches.
+ *
+ * Encoding this in `publish_id`, rather than adding an `origin` field, keeps the
+ * event contract untouched: breathecode ignores `publish_id` today, so this
+ * costs them nothing and needs no coordination. If they ever start consuming the
+ * provenance, it can be promoted to a field of its own.
+ */
+
+export const BACKFILL_PUBLISH_ID_PREFIX = "backfill-"
+
+/** One per script invocation, shared by every course it touches. */
+export function newBackfillRunId(): string {
+  return uuidv4()
+}
+
+export function backfillPublishId(runId: string, courseSlug: string): string {
+  return `${BACKFILL_PUBLISH_ID_PREFIX}${runId}-${courseSlug}`
+}
+
+export function isBackfillPublishId(publishId: string): boolean {
+  return publishId.startsWith(BACKFILL_PUBLISH_ID_PREFIX)
+}
+
+/**
+ * `skipped` means the event was never attempted, which is not a failure: a
+ * course the backfill did not change has nothing to announce.
+ */
+export type BackfillEventOutcome = "skipped" | "delivered" | "failed";
+
+export type EmitBackfillManifestEventParams = {
+  courseSlug: string;
+  runId: string;
+  result: GenerateCourseDescriptionsResult;
+  storage: CourseDescriptionsStorage;
+  rigobotToken: string;
+  breathecodeToken: string;
+  /** Injection points for tests. */
+  emit?: typeof sendPublishEvent;
+  readPackageInfo?: typeof fetchPackageInfo;
+};
+
+/**
+ * Announce one reconciled course. Never throws: the descriptions and the
+ * manifest are already written by the time this runs, and losing the
+ * notification must not turn a successful course into a failed one.
+ */
+export async function emitBackfillManifestEvent(
+  params: EmitBackfillManifestEventParams
+): Promise<BackfillEventOutcome> {
+  const { courseSlug, result } = params
+
+  if (result.status === "skipped") {
+    return "skipped"
+  }
+
+  const emit = params.emit ?? sendPublishEvent
+  const readPackageInfo = params.readPackageInfo ?? fetchPackageInfo
+
+  // Enriching the payload must never prevent the delivery: both the manifest and
+  // the package info are optional in the contract, and an event with less
+  // context beats an event that never arrives.
+  let manifest: PackageManifest | null = null
+  try {
+    if (params.storage.readManifest) {
+      manifest = await params.storage.readManifest(courseSlug)
+    }
+  } catch (error) {
+    console.error(
+      `[backfill-events] Could not read the manifest of "${courseSlug}", reporting without it:`,
+      (error as Error).message
+    )
+  }
+
+  let packageInfo: Record<string, unknown> | null = null
+  try {
+    packageInfo = await readPackageInfo(courseSlug, params.rigobotToken)
+  } catch (error) {
+    console.error(
+      `[backfill-events] Could not read the package "${courseSlug}", reporting without it:`,
+      (error as Error).message
+    )
+  }
+
+  try {
+    const delivered = await emit(
+      buildPackageManifestUpdatedEvent(
+        {
+          publishId: backfillPublishId(params.runId, courseSlug),
+          courseSlug,
+          packageInfo,
+          manifest,
+        },
+        // A failed projection does not fail the run — the syllabus is saved
+        // either way — but this event reports the manifest, so it cannot claim
+        // success over one that was never rewritten.
+        result.status === "failed" || result.manifestProjected === false ?
+          "failed" :
+          "success",
+        {
+          generated: result.generated,
+          failed: result.failed,
+          missing: result.missing,
+        }
+      ),
+      params.breathecodeToken
+    )
+
+    return delivered ? "delivered" : "failed"
+  } catch (error) {
+    console.error(
+      `[backfill-events] Could not deliver the manifest event for "${courseSlug}":`,
+      (error as Error).message
+    )
+    return "failed"
+  }
+}

package/src/utils/descriptions/generateCourseDescriptions.ts

@@ -79,6 +79,13 @@ export type GenerateCourseDescriptionsResult = {
   errors: string[];
   /** Completion durations, to watch how close we run to the request limits. */
   durationsSeconds: number[];
+  /**
+   * Whether `package-manifest.json` was re-projected; null when the storage does
+   * not project at all. A failed projection is not a failed run — the syllabus,
+   * which is the source of truth, is already saved — but callers that announce
+   * the manifest downstream must not report success on it.
+   */
+  manifestProjected: boolean | null;
 };
 
 function emptyResult(
@@ -92,6 +99,7 @@ function emptyResult(
     missing: 0,
     errors: [],
     durationsSeconds: [],
+    manifestProjected: null,
   }
 }
 
@@ -288,9 +296,11 @@ export async function generateCourseDescriptions(
   if (storage.reprojectManifest) {
     try {
       await storage.reprojectManifest(courseSlug)
+      result.manifestProjected = true
     } catch (error) {
       // The syllabus (the source of truth) is already saved; a failed
       // projection is recoverable and must not lose the generated text.
+      result.manifestProjected = false
       result.errors.push(
         `manifest projection failed: ${(error as Error).message}`
       )