@learnpack/learnpack 5.0.352 → 5.0.354

package/src/scripts/descriptionsS3Backfill.ts

@@ -0,0 +1,376 @@
+/* eslint-disable no-await-in-loop */
+/**
+ * Fase 0a — S3 descriptions backfill (step 2 of the S3 pipeline).
+ *
+ * One-off fill of the published catalogue (~400+ courses). The per-course work
+ * is the shared description service (`generateCourseDescriptions`), the same one
+ * the post-publish stage and the sweep use; this script only decides WHICH
+ * courses to run it on and in what batches.
+ *
+ *   - Processes at most --limit courses per run (default 50), ordered by the
+ *     initialSyllabus.json modification time (most recently edited first),
+ *     skipping courses already settled at the current prompt version. Re-run to
+ *     advance through the catalogue.
+ *   - Or target a single course with --slug (remediation / on-demand).
+ *
+ * Writes descriptions into the published initialSyllabus.json. By default the
+ * manifest projection is a separate pass: run the package-manifest backfill
+ * (runBatch) afterwards, which over the whole catalogue is cheaper than
+ * re-projecting course by course.
+ *
+ * That default inverts for small, targeted runs, where three flags reproduce per
+ * course what a publication does, in one process instead of three:
+ *
+ *   --reproject-manifest  projects each course as it is described (the same
+ *                         `processPackage` the separate pass runs, plus a
+ *                         CloudFront invalidation);
+ *   --mirror-draft        copies the new descriptions back into the GCS draft,
+ *                         so the next publication does not regenerate them;
+ *   --emit-events         announces the result to breathecode.
+ *
+ * Secrets/infra via env: RIGOBOT_SYSTEM_TOKEN (required), BREATHECODE_SYSTEM_TOKEN
+ * (required by --emit-events), GCP_CREDENTIALS_JSON and GCP_BUCKET_NAME (required
+ * by --mirror-draft), plus S3_PACKAGES_BUCKET, AWS_REGION and
+ * CLOUDFRONT_DISTRIBUTION_ID as fallbacks/optionals.
+ *
+ * Per-run flags:
+ *   --s3-bucket <name>    S3 bucket (or env S3_PACKAGES_BUCKET, default learnpack-paquetes)
+ *   --gcs-bucket <name>   GCS draft bucket for --mirror-draft (or env GCP_BUCKET_NAME)
+ *   --region <region>     AWS region (default us-east-1, or env AWS_REGION)
+ *   --slug <slug>         process a single course (remediation / on-demand)
+ *   --limit <n>           courses per run (default 50)
+ *   --target-words <n>    words per description (default 25)
+ *   --concurrency <n>     completions in flight per course (default 5)
+ *   --dry-run             preview without writing/generating
+ *   --no-reconcile        skip the additive syllabus reconciliation
+ *   --force               regenerate even courses already settled at this prompt version
+ *   --reproject-manifest  re-project package-manifest.json per course, and invalidate it
+ *   --mirror-draft        mirror the descriptions back into the GCS draft
+ *   --emit-events         send package_manifest_updated per course (implies --reproject-manifest)
+ *
+ * Examples:
+ *   node lib/scripts/descriptionsS3Backfill.js --limit 50
+ *   node lib/scripts/descriptionsS3Backfill.js --slug my-course
+ *   node lib/scripts/descriptionsS3Backfill.js --dry-run --limit 10
+ *   node lib/scripts/descriptionsS3Backfill.js --slug my-course \
+ *     --reproject-manifest --mirror-draft --emit-events
+ */
+import { parseArgs } from "node:util"
+import { ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3"
+import { CloudFrontClient } from "@aws-sdk/client-cloudfront"
+import { Storage } from "@google-cloud/storage"
+import { DESCRIPTION_TARGET_WORD_COUNT } from "../utils/packageManifest"
+import { AwsClient } from "../utils/s3/packageManifestBackfill"
+import {
+  DEFAULT_DESCRIPTIONS_CONCURRENCY,
+  generateCourseDescriptions,
+} from "../utils/descriptions/generateCourseDescriptions"
+import {
+  createS3DescriptionsStorage,
+  fetchExercises,
+  S3DescriptionsStorageOptions,
+} from "../utils/descriptions/s3Storage"
+import {
+  BackfillEventOutcome,
+  emitBackfillManifestEvent,
+  newBackfillRunId,
+} from "../utils/descriptions/backfillEvents"
+import { createGcsDescriptionsStorage } from "../utils/descriptions/gcsStorage"
+import { mirrorDescriptionsToDraft } from "../utils/descriptions/mirrorDescriptions"
+import { requireGcsBucketName } from "../utils/gcsBucketName"
+import { isCourseSettled } from "../utils/descriptions/workList"
+
+const SYLLABUS_KEY_PATTERN = /^([^/]+)\/\.learn\/initialSyllabus\.json$/
+
+/** Courses (by slug) ordered by initialSyllabus.json LastModified, newest first. */
+async function listCoursesByRecency(
+  s3: AwsClient,
+  bucket: string
+): Promise<string[]> {
+  const mtimes = new Map<string, number>()
+  let continuationToken: string | undefined
+
+  do {
+    const response = (await s3.send(
+      new ListObjectsV2Command({
+        Bucket: bucket,
+        ContinuationToken: continuationToken,
+      })
+    )) as {
+      Contents?: Array<{ Key?: string; LastModified?: Date }>;
+      NextContinuationToken?: string;
+    }
+
+    for (const object of response.Contents || []) {
+      const match = object.Key?.match(SYLLABUS_KEY_PATTERN)
+      if (match) {
+        mtimes.set(match[1], object.LastModified?.getTime() || 0)
+      }
+    }
+
+    continuationToken = response.NextContinuationToken
+  } while (continuationToken)
+
+  return [...mtimes.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .map(([slug]) => slug)
+}
+
+/**
+ * CDN invalidation for the re-projected manifests. Optional: a stale edge cache
+ * expires on its own, so a missing distribution id degrades the run instead of
+ * stopping it.
+ */
+function cloudFrontFor(
+  region: string
+): S3DescriptionsStorageOptions["cloudFront"] {
+  const distributionId = (process.env.CLOUDFRONT_DISTRIBUTION_ID || "").trim()
+  if (!distributionId) {
+    console.warn(
+      "[s3-backfill] CLOUDFRONT_DISTRIBUTION_ID is not set: manifests will be re-projected but not invalidated"
+    )
+    return undefined
+  }
+
+  return {
+    client: new CloudFrontClient({ region }) as unknown as AwsClient,
+    distributionId,
+  }
+}
+
+async function main(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      "s3-bucket": { type: "string" },
+      "gcs-bucket": { type: "string" },
+      region: { type: "string" },
+      slug: { type: "string" },
+      limit: { type: "string" },
+      "target-words": { type: "string" },
+      concurrency: { type: "string" },
+      "dry-run": { type: "boolean", default: false },
+      "no-reconcile": { type: "boolean", default: false },
+      force: { type: "boolean", default: false },
+      "reproject-manifest": { type: "boolean", default: false },
+      "mirror-draft": { type: "boolean", default: false },
+      "emit-events": { type: "boolean", default: false },
+    },
+  })
+
+  const token = process.env.RIGOBOT_SYSTEM_TOKEN
+  if (!token) {
+    console.error("[s3-backfill] RIGOBOT_SYSTEM_TOKEN (env) is required")
+    process.exit(1)
+  }
+
+  const bucket =
+    values["s3-bucket"] ||
+    process.env.S3_PACKAGES_BUCKET ||
+    "learnpack-paquetes"
+
+  const region = values.region || process.env.AWS_REGION || "us-east-1"
+  const limit = Number.parseInt(values.limit || "50", 10) || 50
+  const dryRun = values["dry-run"] ?? false
+  const force = values.force ?? false
+  const emitEvents = values["emit-events"] ?? false
+  const reprojectManifest = values["reproject-manifest"] ?? false
+  const mirrorDraft = values["mirror-draft"] ?? false
+
+  // The event announces that the manifest changed, so emitting it without
+  // projecting the manifest would state something untrue. Required rather than
+  // implied: turning on a write to S3 and the CDN as a side effect of another
+  // flag is exactly the kind of surprise a backfill should not have.
+  if (emitEvents && !reprojectManifest) {
+    console.error(
+      "[s3-backfill] --emit-events requires --reproject-manifest: the event " +
+        "reports a manifest update that would not have happened"
+    )
+    process.exit(1)
+  }
+
+  const breathecodeToken = process.env.BREATHECODE_SYSTEM_TOKEN
+  if (emitEvents && !breathecodeToken) {
+    console.error(
+      "[s3-backfill] BREATHECODE_SYSTEM_TOKEN (env) is required by --emit-events"
+    )
+    process.exit(1)
+  }
+
+  if (emitEvents && dryRun) {
+    console.error(
+      "[s3-backfill] --emit-events cannot be combined with --dry-run: there is " +
+        "no dry run of an event that breathecode already received"
+    )
+    process.exit(1)
+  }
+
+  // Built up front, not on first use: a misconfigured draft bucket must fail at
+  // boot, not after the first course has already been billed to Rigobot.
+  let draftStorage
+  if (mirrorDraft) {
+    const credentialsEnv = process.env.GCP_CREDENTIALS_JSON
+    if (!credentialsEnv) {
+      console.error(
+        "[s3-backfill] GCP_CREDENTIALS_JSON (env) is required by --mirror-draft"
+      )
+      process.exit(1)
+    }
+
+    draftStorage = createGcsDescriptionsStorage(
+      new Storage({ credentials: JSON.parse(credentialsEnv) }).bucket(
+        values["gcs-bucket"] || requireGcsBucketName()
+      )
+    )
+  }
+
+  const s3 = new S3Client({ region }) as unknown as AwsClient
+  const storage = createS3DescriptionsStorage(
+    s3,
+    bucket,
+    // Off by default: over the whole catalogue manifests are cheaper to project
+    // in a single later pass than course by course.
+    reprojectManifest ?
+      { cloudFront: cloudFrontFor(region) } :
+      { reprojectManifest: false }
+  )
+
+  const runId = newBackfillRunId()
+
+  const singleSlug = values.slug
+  const slugs = singleSlug ?
+    [singleSlug] :
+    await listCoursesByRecency(s3, bucket)
+
+  console.log(
+    `[s3-backfill] Starting${
+      singleSlug ? ` (slug=${singleSlug})` : ` (limit=${limit})`
+    }${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)${
+      reprojectManifest ? ", re-projecting manifests" : ""
+    }${mirrorDraft ? ", mirroring into the draft" : ""}${
+      emitEvents ? `, run ${runId}` : ""
+    }`
+  )
+
+  let processed = 0
+  let generated = 0
+  let failed = 0
+  let mirrored = 0
+  const events: Record<BackfillEventOutcome, number> = {
+    skipped: 0,
+    delivered: 0,
+    failed: 0,
+  }
+
+  for (const slug of slugs) {
+    if (!singleSlug && processed >= limit) {
+      break
+    }
+
+    try {
+      // Cheap pre-filter: skip finished courses without reading their READMEs.
+      if (!force) {
+        const syllabus = await storage.readSyllabus(slug)
+        if (!syllabus) {
+          continue
+        }
+
+        const exercises = await fetchExercises(s3, bucket, slug)
+        if (exercises.length === 0 || isCourseSettled(syllabus, exercises)) {
+          continue
+        }
+      }
+
+      const result = await generateCourseDescriptions(storage, slug, {
+        token,
+        dryRun,
+        force,
+        reconcile: !(values["no-reconcile"] ?? false),
+        concurrency:
+          Number.parseInt(values.concurrency || "", 10) ||
+          DEFAULT_DESCRIPTIONS_CONCURRENCY,
+        targetWordCount:
+          Number.parseInt(values["target-words"] || "", 10) ||
+          DESCRIPTION_TARGET_WORD_COUNT,
+      })
+
+      if (result.status === "skipped") {
+        continue
+      }
+
+      processed += 1
+      generated += result.generated
+      failed += result.failed
+
+      console.log(
+        `[s3-backfill] "${slug}": ${result.steps} step(s), ${
+          result.generated
+        } description(s)${dryRun ? " (dry-run)" : ""}${
+          result.failed > 0 ? `, ${result.failed} failed` : ""
+        }${result.missing > 0 ? `, ${result.missing} unanswered` : ""}`
+      )
+
+      if (draftStorage) {
+        // Before the event, so a course is fully settled by the time it is
+        // announced. A failure here does NOT change the event status: the
+        // published package and its manifest are already correct, and only the
+        // draft lags behind — the next publication regenerates what is missing.
+        try {
+          const published = await storage.readSyllabus(slug)
+          const mirror = await mirrorDescriptionsToDraft(
+            draftStorage,
+            slug,
+            published,
+            { dryRun }
+          )
+          mirrored += mirror.copied
+          console.log(
+            `[s3-backfill] "${slug}": ${mirror.copied} mirrored, ${mirror.missed} missed, ${mirror.fresh} already fresh`
+          )
+        } catch (error) {
+          console.error(
+            `[s3-backfill] Could not mirror "${slug}" into the draft:`,
+            (error as Error).message
+          )
+        }
+      }
+
+      if (emitEvents) {
+        const outcome = await emitBackfillManifestEvent({
+          courseSlug: slug,
+          runId,
+          result,
+          storage,
+          rigobotToken: token,
+          breathecodeToken: breathecodeToken as string,
+        })
+
+        events[outcome] += 1
+        console.log(`[s3-backfill] "${slug}": manifest event ${outcome}`)
+      }
+    } catch (error) {
+      processed += 1
+      failed += 1
+      // No event here on purpose: unlike a publication, a backfill never
+      // announced one, so nothing is owed. The course is simply re-run.
+      console.error(
+        `[s3-backfill] Failed processing "${slug}":`,
+        (error as Error).message
+      )
+    }
+  }
+
+  console.log(
+    `[s3-backfill] Done. ${processed} course(s), ${generated} description(s) written, ${failed} failure(s).` +
+      (mirrorDraft ? ` ${mirrored} mirrored into the draft.` : "") +
+      (emitEvents ?
+        ` Events: ${events.delivered} delivered, ${events.failed} failed (run ${runId}).` :
+        "")
+  )
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch(error => {
+    console.error("[s3-backfill] Fatal error:", error)
+    process.exit(1)
+  })

package/src/scripts/descriptionsSweep.ts

@@ -0,0 +1,185 @@
+/* eslint-disable no-await-in-loop */
+/**
+ * Descriptions sweep — safety net for the post-publish stage.
+ *
+ * The stage that generates descriptions runs inside the web process, so a dyno
+ * restart mid-flight leaves a package published but not described and, worse,
+ * breathecode waiting for a manifest event that was promised and never arrived.
+ * This job finishes those publications.
+ *
+ * Runs as a one-off dyno via Heroku Scheduler (hourly is plenty):
+ *   node lib/scripts/descriptionsSweep.js
+ *
+ * Two sources of work:
+ *   1. Publish journals (`publish-journal/` in GCS) — the primary one. Each
+ *      journal that stalled is resumed: missing descriptions are generated, the
+ *      draft is mirrored, and the owed event is emitted. Journals are dropped
+ *      once complete and abandoned after a few attempts.
+ *   2. A deep scan of published courses (DESCRIPTIONS_SWEEP_FULL=1) — catches
+ *      packages that never got a journal at all, at the cost of reading every
+ *      course. Off by default.
+ *
+ * Required env: GCP_CREDENTIALS_JSON, RIGOBOT_SYSTEM_TOKEN.
+ * Optional env: GCP_BUCKET_NAME, S3_PACKAGES_BUCKET, AWS_REGION,
+ * CLOUDFRONT_DISTRIBUTION_ID, BREATHECODE_SYSTEM_TOKEN (needed to emit the
+ * pending events), DESCRIPTIONS_SWEEP_FULL, DESCRIPTIONS_SWEEP_LIMIT,
+ * DESCRIPTION_STALE_AFTER_MS.
+ */
+import { Storage } from "@google-cloud/storage"
+import { S3Client } from "@aws-sdk/client-s3"
+import {
+  createGcsJournalStorage,
+  listJournalRefs,
+  readJournalByKey,
+} from "../utils/publishJournal"
+import {
+  AwsClient,
+  listPublishedSlugs,
+} from "../utils/s3/packageManifestBackfill"
+import { requireGcsBucketName } from "../utils/gcsBucketName"
+import { createGcsDescriptionsStorage } from "../utils/descriptions/gcsStorage"
+import { createS3DescriptionsStorageFromEnv } from "../utils/descriptions/s3Storage"
+import { generateCourseDescriptions } from "../utils/descriptions/generateCourseDescriptions"
+import {
+  DEFAULT_STALE_AFTER_MS,
+  resumePublication,
+  ResumeOutcome,
+} from "../utils/descriptions/resumePublication"
+
+async function main(): Promise<void> {
+  const credentialsEnv = process.env.GCP_CREDENTIALS_JSON
+  if (!credentialsEnv) {
+    console.error("[sweep] GCP_CREDENTIALS_JSON (env) is required")
+    process.exit(1)
+  }
+
+  const rigobotToken = process.env.RIGOBOT_SYSTEM_TOKEN
+  if (!rigobotToken) {
+    console.error("[sweep] RIGOBOT_SYSTEM_TOKEN (env) is required")
+    process.exit(1)
+  }
+
+  const breathecodeToken = process.env.BREATHECODE_SYSTEM_TOKEN
+  if (!breathecodeToken) {
+    console.warn(
+      "[sweep] BREATHECODE_SYSTEM_TOKEN is not set: pending events cannot be emitted"
+    )
+  }
+
+  const bucket = new Storage({
+    credentials: JSON.parse(credentialsEnv),
+  }).bucket(requireGcsBucketName())
+
+  const journalStorage = createGcsJournalStorage(bucket)
+  const publishedStorage = createS3DescriptionsStorageFromEnv()
+  const draftStorage = createGcsDescriptionsStorage(bucket)
+  const staleAfterMs =
+    Number.parseInt(process.env.DESCRIPTION_STALE_AFTER_MS || "", 10) ||
+    DEFAULT_STALE_AFTER_MS
+
+  const refs = await listJournalRefs(journalStorage)
+  console.log(`[sweep] ${refs.length} unfinished publication(s)`)
+
+  const tally: Record<ResumeOutcome, number> = {
+    "skipped-recent": 0,
+    "skipped-abandoned": 0,
+    abandoned: 0,
+    resumed: 0,
+    "nothing-to-do": 0,
+  }
+
+  for (const ref of refs) {
+    try {
+      const journal = await readJournalByKey(journalStorage, ref.key)
+      if (!journal) {
+        continue
+      }
+
+      const outcome = await resumePublication(journal, {
+        journalStorage,
+        publishedStorage,
+        draftStorage,
+        rigobotToken,
+        breathecodeToken,
+        staleAfterMs,
+      })
+
+      tally[outcome] += 1
+      if (outcome === "resumed") {
+        console.log(`[sweep] Resumed "${ref.courseSlug}" (${ref.publishId})`)
+      }
+    } catch (error) {
+      console.error(
+        `[sweep] Failed resuming "${ref.courseSlug}":`,
+        (error as Error).message
+      )
+    }
+  }
+
+  console.log(
+    `[sweep] Journals: ${tally.resumed} resumed, ${tally["skipped-recent"]} still in flight, ` +
+      `${tally["nothing-to-do"]} nothing to do, ${tally.abandoned} abandoned, ` +
+      `${tally["skipped-abandoned"]} already abandoned`
+  )
+
+  if (process.env.DESCRIPTIONS_SWEEP_FULL === "1") {
+    await deepScan(rigobotToken, publishedStorage)
+  }
+}
+
+/**
+ * Fallback for packages with no journal at all (published before this existed,
+ * or whose journal write failed). Expensive: it reads every published course,
+ * so it is opt-in rather than the default path.
+ */
+async function deepScan(
+  rigobotToken: string,
+  publishedStorage: ReturnType<typeof createS3DescriptionsStorageFromEnv>
+): Promise<void> {
+  const s3 = new S3Client({
+    region: process.env.AWS_REGION || "us-east-1",
+  }) as unknown as AwsClient
+  const bucket = process.env.S3_PACKAGES_BUCKET || "learnpack-paquetes"
+  const limit =
+    Number.parseInt(process.env.DESCRIPTIONS_SWEEP_LIMIT || "", 10) || 25
+
+  const slugs = await listPublishedSlugs(s3, bucket)
+  console.log(
+    `[sweep] Deep scan over ${slugs.length} published course(s), limit ${limit}`
+  )
+
+  let processed = 0
+  for (const slug of slugs) {
+    if (processed >= limit) {
+      break
+    }
+
+    try {
+      const result = await generateCourseDescriptions(publishedStorage, slug, {
+        token: rigobotToken,
+        reconcile: true,
+      })
+
+      if (result.status === "skipped") {
+        continue
+      }
+
+      processed += 1
+      console.log(
+        `[sweep] "${slug}": ${result.generated} description(s), ${result.failed} failed`
+      )
+    } catch (error) {
+      processed += 1
+      console.error(`[sweep] Failed on "${slug}":`, (error as Error).message)
+    }
+  }
+
+  console.log(`[sweep] Deep scan done. ${processed} course(s) touched.`)
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch(error => {
+    console.error("[sweep] Fatal error:", error)
+    process.exit(1)
+  })