@learnpack/learnpack 5.0.351 → 5.0.353

package/src/scripts/descriptionsGcsBackfill.ts

@@ -0,0 +1,193 @@
+/* eslint-disable no-await-in-loop */
+/**
+ * Fase 0b — GCS descriptions backfill (reuse from S3, no LLM).
+ *
+ * Seeds the GCS (draft) syllabus with descriptions already generated for the
+ * published S3 syllabus (Fase 0a) — a pure COPY, guarded by the content hash,
+ * performed by the shared mirror (`mirrorDescriptionsToDraft`). It NEVER calls
+ * the LLM: the ~800 GCS drafts are mostly unused, so anything not reusable from
+ * S3 is left for the next publication of that course (or a single-slug 0a run).
+ * Run AFTER 0a has completed.
+ *
+ * Batched like 0a: --limit courses per run (default 50), ordered by
+ * initialSyllabus modification time (newest first).
+ *
+ * Secrets/infra via env: GCP_CREDENTIALS_JSON (required), plus
+ * S3_PACKAGES_BUCKET / GCP_BUCKET_NAME / AWS_REGION as fallbacks for the
+ * corresponding flags.
+ *
+ * Per-run flags:
+ *   --s3-bucket <name>   published S3 bucket to reuse from (or env S3_PACKAGES_BUCKET)
+ *   --gcs-bucket <name>  GCS bucket (or env GCP_BUCKET_NAME, which is required)
+ *   --region <region>    AWS region (default us-east-1, or env AWS_REGION)
+ *   --slug <slug>        process a single course
+ *   --limit <n>          courses per run (default 50)
+ *   --dry-run            preview without writing
+ *   --no-reconcile       skip the additive syllabus reconciliation
+ *
+ * Examples:
+ *   node lib/scripts/descriptionsGcsBackfill.js --limit 200
+ *   node lib/scripts/descriptionsGcsBackfill.js --slug my-course
+ */
+import { parseArgs } from "node:util"
+import { Bucket, Storage } from "@google-cloud/storage"
+import { S3Client } from "@aws-sdk/client-s3"
+import { Syllabus } from "../models/creator"
+import { synchronizeSyllabusWithBucket } from "../utils/syllabusSync"
+import {
+  AwsClient,
+  fetchJsonObject,
+} from "../utils/s3/packageManifestBackfill"
+import { requireGcsBucketName } from "../utils/gcsBucketName"
+import { createGcsDescriptionsStorage } from "../utils/descriptions/gcsStorage"
+import { mirrorDescriptionsToDraft } from "../utils/descriptions/mirrorDescriptions"
+import { syllabusKey as s3SyllabusKey } from "../utils/descriptions/s3Storage"
+
+const GCS_SYLLABUS_PATTERN =
+  /^courses\/([^/]+)\/\.learn\/initialSyllabus\.json$/
+
+/** Courses (by slug) ordered by initialSyllabus.json update time, newest first. */
+async function listCoursesByRecency(bucket: Bucket): Promise<string[]> {
+  const [files] = await bucket.getFiles({ prefix: "courses/" })
+  const mtimes = new Map<string, number>()
+  for (const file of files) {
+    const match = file.name.match(GCS_SYLLABUS_PATTERN)
+    if (match) {
+      const updated = (file.metadata as { updated?: string } | undefined)
+        ?.updated
+      mtimes.set(match[1], updated ? new Date(updated).getTime() : 0)
+    }
+  }
+
+  return [...mtimes.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .map(entry => entry[0])
+}
+
+async function main(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      "s3-bucket": { type: "string" },
+      "gcs-bucket": { type: "string" },
+      region: { type: "string" },
+      slug: { type: "string" },
+      limit: { type: "string" },
+      "dry-run": { type: "boolean", default: false },
+      "no-reconcile": { type: "boolean", default: false },
+    },
+  })
+
+  const s3Bucket =
+    values["s3-bucket"] ||
+    process.env.S3_PACKAGES_BUCKET ||
+    "learnpack-paquetes"
+
+  const credentialsEnv = process.env.GCP_CREDENTIALS_JSON
+  if (!credentialsEnv) {
+    console.error("[gcs-backfill] GCP_CREDENTIALS_JSON (env) is required")
+    process.exit(1)
+  }
+
+  const bucket = new Storage({
+    credentials: JSON.parse(credentialsEnv),
+  }).bucket(values["gcs-bucket"] || requireGcsBucketName())
+
+  const s3 = new S3Client({
+    region: values.region || process.env.AWS_REGION || "us-east-1",
+  }) as unknown as AwsClient
+
+  const dryRun = values["dry-run"] ?? false
+  const reconcile = !(values["no-reconcile"] ?? false)
+  const limit = Number.parseInt(values.limit || "50", 10) || 50
+  const storage = createGcsDescriptionsStorage(bucket)
+
+  const singleSlug = values.slug
+  const slugs = singleSlug ? [singleSlug] : await listCoursesByRecency(bucket)
+
+  console.log(
+    `[gcs-backfill] Starting${
+      singleSlug ? ` (slug=${singleSlug})` : ` (limit=${limit})`
+    }${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)`
+  )
+
+  let processed = 0
+  let copied = 0
+  let missed = 0
+
+  for (const slug of slugs) {
+    if (!singleSlug && processed >= limit) {
+      break
+    }
+
+    try {
+      const publishedSyllabus = await fetchJsonObject<Syllabus>(
+        s3,
+        s3Bucket,
+        s3SyllabusKey(slug)
+      )
+      if (!publishedSyllabus) {
+        continue
+      }
+
+      if (reconcile && storage.syllabusSyncStorage) {
+        const draft = await storage.readSyllabus(slug)
+        if (draft) {
+          try {
+            const sync = await synchronizeSyllabusWithBucket(
+              storage.syllabusSyncStorage,
+              slug,
+              draft,
+              { prune: false }
+            )
+            if (sync.changed && !dryRun) {
+              await storage.writeSyllabus(slug, sync.syllabus)
+            }
+          } catch (error) {
+            console.error(
+              `[gcs-backfill] Reconciliation failed for "${slug}":`,
+              (error as Error).message
+            )
+          }
+        }
+      }
+
+      const result = await mirrorDescriptionsToDraft(
+        storage,
+        slug,
+        publishedSyllabus,
+        { dryRun }
+      )
+
+      if (result.copied === 0 && result.missed === 0) {
+        continue
+      }
+
+      processed += 1
+      copied += result.copied
+      missed += result.missed
+
+      console.log(
+        `[gcs-backfill] "${slug}": copied ${result.copied}, ${
+          result.missed
+        } miss(es)${dryRun ? " (dry-run)" : ""}`
+      )
+    } catch (error) {
+      processed += 1
+      console.error(
+        `[gcs-backfill] Failed processing "${slug}":`,
+        (error as Error).message
+      )
+    }
+  }
+
+  console.log(
+    `[gcs-backfill] Done. ${processed} course(s), ${copied} description(s) copied, ${missed} miss(es).`
+  )
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch(error => {
+    console.error("[gcs-backfill] Fatal error:", error)
+    process.exit(1)
+  })

package/src/scripts/descriptionsS3Backfill.ts

@@ -0,0 +1,208 @@
+/* eslint-disable no-await-in-loop */
+/**
+ * Fase 0a — S3 descriptions backfill (step 2 of the S3 pipeline).
+ *
+ * One-off fill of the published catalogue (~400+ courses). The per-course work
+ * is the shared description service (`generateCourseDescriptions`), the same one
+ * the post-publish stage and the sweep use; this script only decides WHICH
+ * courses to run it on and in what batches.
+ *
+ *   - Processes at most --limit courses per run (default 50), ordered by the
+ *     initialSyllabus.json modification time (most recently edited first),
+ *     skipping courses already settled at the current prompt version. Re-run to
+ *     advance through the catalogue.
+ *   - Or target a single course with --slug (remediation / on-demand).
+ *
+ * Writes descriptions into the published initialSyllabus.json. Manifest
+ * projection is a separate pass here: run the package-manifest backfill
+ * (runBatch) afterwards, which is cheaper than re-projecting course by course.
+ *
+ * Secrets/infra via env: RIGOBOT_SYSTEM_TOKEN (required), plus S3_PACKAGES_BUCKET
+ * and AWS_REGION as fallbacks for --s3-bucket / --region.
+ *
+ * Per-run flags:
+ *   --s3-bucket <name>   S3 bucket (or env S3_PACKAGES_BUCKET, default learnpack-paquetes)
+ *   --region <region>    AWS region (default us-east-1, or env AWS_REGION)
+ *   --slug <slug>        process a single course (remediation / on-demand)
+ *   --limit <n>          courses per run (default 50)
+ *   --target-words <n>   words per description (default 25)
+ *   --concurrency <n>    completions in flight per course (default 5)
+ *   --dry-run            preview without writing/generating
+ *   --no-reconcile       skip the additive syllabus reconciliation
+ *   --force              regenerate even courses already settled at this prompt version
+ *
+ * Examples:
+ *   node lib/scripts/descriptionsS3Backfill.js --limit 50
+ *   node lib/scripts/descriptionsS3Backfill.js --slug my-course
+ *   node lib/scripts/descriptionsS3Backfill.js --dry-run --limit 10
+ */
+import { parseArgs } from "node:util"
+import { ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3"
+import { DESCRIPTION_TARGET_WORD_COUNT } from "../utils/packageManifest"
+import { AwsClient } from "../utils/s3/packageManifestBackfill"
+import {
+  DEFAULT_DESCRIPTIONS_CONCURRENCY,
+  generateCourseDescriptions,
+} from "../utils/descriptions/generateCourseDescriptions"
+import {
+  createS3DescriptionsStorage,
+  fetchExercises,
+} from "../utils/descriptions/s3Storage"
+import { isCourseSettled } from "../utils/descriptions/workList"
+
+const SYLLABUS_KEY_PATTERN = /^([^/]+)\/\.learn\/initialSyllabus\.json$/
+
+/** Courses (by slug) ordered by initialSyllabus.json LastModified, newest first. */
+async function listCoursesByRecency(
+  s3: AwsClient,
+  bucket: string
+): Promise<string[]> {
+  const mtimes = new Map<string, number>()
+  let continuationToken: string | undefined
+
+  do {
+    const response = (await s3.send(
+      new ListObjectsV2Command({
+        Bucket: bucket,
+        ContinuationToken: continuationToken,
+      })
+    )) as {
+      Contents?: Array<{ Key?: string; LastModified?: Date }>;
+      NextContinuationToken?: string;
+    }
+
+    for (const object of response.Contents || []) {
+      const match = object.Key?.match(SYLLABUS_KEY_PATTERN)
+      if (match) {
+        mtimes.set(match[1], object.LastModified?.getTime() || 0)
+      }
+    }
+
+    continuationToken = response.NextContinuationToken
+  } while (continuationToken)
+
+  return [...mtimes.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .map(([slug]) => slug)
+}
+
+async function main(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      "s3-bucket": { type: "string" },
+      region: { type: "string" },
+      slug: { type: "string" },
+      limit: { type: "string" },
+      "target-words": { type: "string" },
+      concurrency: { type: "string" },
+      "dry-run": { type: "boolean", default: false },
+      "no-reconcile": { type: "boolean", default: false },
+      force: { type: "boolean", default: false },
+    },
+  })
+
+  const token = process.env.RIGOBOT_SYSTEM_TOKEN
+  if (!token) {
+    console.error("[s3-backfill] RIGOBOT_SYSTEM_TOKEN (env) is required")
+    process.exit(1)
+  }
+
+  const bucket =
+    values["s3-bucket"] ||
+    process.env.S3_PACKAGES_BUCKET ||
+    "learnpack-paquetes"
+
+  const region = values.region || process.env.AWS_REGION || "us-east-1"
+  const limit = Number.parseInt(values.limit || "50", 10) || 50
+  const dryRun = values["dry-run"] ?? false
+  const force = values.force ?? false
+
+  const s3 = new S3Client({ region }) as unknown as AwsClient
+  // Manifests are projected in a separate pass after this backfill.
+  const storage = createS3DescriptionsStorage(s3, bucket, {
+    reprojectManifest: false,
+  })
+
+  const singleSlug = values.slug
+  const slugs = singleSlug ?
+    [singleSlug] :
+    await listCoursesByRecency(s3, bucket)
+
+  console.log(
+    `[s3-backfill] Starting${
+      singleSlug ? ` (slug=${singleSlug})` : ` (limit=${limit})`
+    }${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)`
+  )
+
+  let processed = 0
+  let generated = 0
+  let failed = 0
+
+  for (const slug of slugs) {
+    if (!singleSlug && processed >= limit) {
+      break
+    }
+
+    try {
+      // Cheap pre-filter: skip finished courses without reading their READMEs.
+      if (!force) {
+        const syllabus = await storage.readSyllabus(slug)
+        if (!syllabus) {
+          continue
+        }
+
+        const exercises = await fetchExercises(s3, bucket, slug)
+        if (exercises.length === 0 || isCourseSettled(syllabus, exercises)) {
+          continue
+        }
+      }
+
+      const result = await generateCourseDescriptions(storage, slug, {
+        token,
+        dryRun,
+        force,
+        reconcile: !(values["no-reconcile"] ?? false),
+        concurrency:
+          Number.parseInt(values.concurrency || "", 10) ||
+          DEFAULT_DESCRIPTIONS_CONCURRENCY,
+        targetWordCount:
+          Number.parseInt(values["target-words"] || "", 10) ||
+          DESCRIPTION_TARGET_WORD_COUNT,
+      })
+
+      if (result.status === "skipped") {
+        continue
+      }
+
+      processed += 1
+      generated += result.generated
+      failed += result.failed
+
+      console.log(
+        `[s3-backfill] "${slug}": ${result.steps} step(s), ${
+          result.generated
+        } description(s)${dryRun ? " (dry-run)" : ""}${
+          result.failed > 0 ? `, ${result.failed} failed` : ""
+        }${result.missing > 0 ? `, ${result.missing} unanswered` : ""}`
+      )
+    } catch (error) {
+      processed += 1
+      failed += 1
+      console.error(
+        `[s3-backfill] Failed processing "${slug}":`,
+        (error as Error).message
+      )
+    }
+  }
+
+  console.log(
+    `[s3-backfill] Done. ${processed} course(s), ${generated} description(s) written, ${failed} failure(s).`
+  )
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch(error => {
+    console.error("[s3-backfill] Fatal error:", error)
+    process.exit(1)
+  })

package/src/scripts/descriptionsSweep.ts

@@ -0,0 +1,185 @@
+/* eslint-disable no-await-in-loop */
+/**
+ * Descriptions sweep — safety net for the post-publish stage.
+ *
+ * The stage that generates descriptions runs inside the web process, so a dyno
+ * restart mid-flight leaves a package published but not described and, worse,
+ * breathecode waiting for a manifest event that was promised and never arrived.
+ * This job finishes those publications.
+ *
+ * Runs as a one-off dyno via Heroku Scheduler (hourly is plenty):
+ *   node lib/scripts/descriptionsSweep.js
+ *
+ * Two sources of work:
+ *   1. Publish journals (`publish-journal/` in GCS) — the primary one. Each
+ *      journal that stalled is resumed: missing descriptions are generated, the
+ *      draft is mirrored, and the owed event is emitted. Journals are dropped
+ *      once complete and abandoned after a few attempts.
+ *   2. A deep scan of published courses (DESCRIPTIONS_SWEEP_FULL=1) — catches
+ *      packages that never got a journal at all, at the cost of reading every
+ *      course. Off by default.
+ *
+ * Required env: GCP_CREDENTIALS_JSON, RIGOBOT_SYSTEM_TOKEN.
+ * Optional env: GCP_BUCKET_NAME, S3_PACKAGES_BUCKET, AWS_REGION,
+ * CLOUDFRONT_DISTRIBUTION_ID, BREATHECODE_SYSTEM_TOKEN (needed to emit the
+ * pending events), DESCRIPTIONS_SWEEP_FULL, DESCRIPTIONS_SWEEP_LIMIT,
+ * DESCRIPTION_STALE_AFTER_MS.
+ */
+import { Storage } from "@google-cloud/storage"
+import { S3Client } from "@aws-sdk/client-s3"
+import {
+  createGcsJournalStorage,
+  listJournalRefs,
+  readJournalByKey,
+} from "../utils/publishJournal"
+import {
+  AwsClient,
+  listPublishedSlugs,
+} from "../utils/s3/packageManifestBackfill"
+import { requireGcsBucketName } from "../utils/gcsBucketName"
+import { createGcsDescriptionsStorage } from "../utils/descriptions/gcsStorage"
+import { createS3DescriptionsStorageFromEnv } from "../utils/descriptions/s3Storage"
+import { generateCourseDescriptions } from "../utils/descriptions/generateCourseDescriptions"
+import {
+  DEFAULT_STALE_AFTER_MS,
+  resumePublication,
+  ResumeOutcome,
+} from "../utils/descriptions/resumePublication"
+
+async function main(): Promise<void> {
+  const credentialsEnv = process.env.GCP_CREDENTIALS_JSON
+  if (!credentialsEnv) {
+    console.error("[sweep] GCP_CREDENTIALS_JSON (env) is required")
+    process.exit(1)
+  }
+
+  const rigobotToken = process.env.RIGOBOT_SYSTEM_TOKEN
+  if (!rigobotToken) {
+    console.error("[sweep] RIGOBOT_SYSTEM_TOKEN (env) is required")
+    process.exit(1)
+  }
+
+  const breathecodeToken = process.env.BREATHECODE_SYSTEM_TOKEN
+  if (!breathecodeToken) {
+    console.warn(
+      "[sweep] BREATHECODE_SYSTEM_TOKEN is not set: pending events cannot be emitted"
+    )
+  }
+
+  const bucket = new Storage({
+    credentials: JSON.parse(credentialsEnv),
+  }).bucket(requireGcsBucketName())
+
+  const journalStorage = createGcsJournalStorage(bucket)
+  const publishedStorage = createS3DescriptionsStorageFromEnv()
+  const draftStorage = createGcsDescriptionsStorage(bucket)
+  const staleAfterMs =
+    Number.parseInt(process.env.DESCRIPTION_STALE_AFTER_MS || "", 10) ||
+    DEFAULT_STALE_AFTER_MS
+
+  const refs = await listJournalRefs(journalStorage)
+  console.log(`[sweep] ${refs.length} unfinished publication(s)`)
+
+  const tally: Record<ResumeOutcome, number> = {
+    "skipped-recent": 0,
+    "skipped-abandoned": 0,
+    abandoned: 0,
+    resumed: 0,
+    "nothing-to-do": 0,
+  }
+
+  for (const ref of refs) {
+    try {
+      const journal = await readJournalByKey(journalStorage, ref.key)
+      if (!journal) {
+        continue
+      }
+
+      const outcome = await resumePublication(journal, {
+        journalStorage,
+        publishedStorage,
+        draftStorage,
+        rigobotToken,
+        breathecodeToken,
+        staleAfterMs,
+      })
+
+      tally[outcome] += 1
+      if (outcome === "resumed") {
+        console.log(`[sweep] Resumed "${ref.courseSlug}" (${ref.publishId})`)
+      }
+    } catch (error) {
+      console.error(
+        `[sweep] Failed resuming "${ref.courseSlug}":`,
+        (error as Error).message
+      )
+    }
+  }
+
+  console.log(
+    `[sweep] Journals: ${tally.resumed} resumed, ${tally["skipped-recent"]} still in flight, ` +
+      `${tally["nothing-to-do"]} nothing to do, ${tally.abandoned} abandoned, ` +
+      `${tally["skipped-abandoned"]} already abandoned`
+  )
+
+  if (process.env.DESCRIPTIONS_SWEEP_FULL === "1") {
+    await deepScan(rigobotToken, publishedStorage)
+  }
+}
+
+/**
+ * Fallback for packages with no journal at all (published before this existed,
+ * or whose journal write failed). Expensive: it reads every published course,
+ * so it is opt-in rather than the default path.
+ */
+async function deepScan(
+  rigobotToken: string,
+  publishedStorage: ReturnType<typeof createS3DescriptionsStorageFromEnv>
+): Promise<void> {
+  const s3 = new S3Client({
+    region: process.env.AWS_REGION || "us-east-1",
+  }) as unknown as AwsClient
+  const bucket = process.env.S3_PACKAGES_BUCKET || "learnpack-paquetes"
+  const limit =
+    Number.parseInt(process.env.DESCRIPTIONS_SWEEP_LIMIT || "", 10) || 25
+
+  const slugs = await listPublishedSlugs(s3, bucket)
+  console.log(
+    `[sweep] Deep scan over ${slugs.length} published course(s), limit ${limit}`
+  )
+
+  let processed = 0
+  for (const slug of slugs) {
+    if (processed >= limit) {
+      break
+    }
+
+    try {
+      const result = await generateCourseDescriptions(publishedStorage, slug, {
+        token: rigobotToken,
+        reconcile: true,
+      })
+
+      if (result.status === "skipped") {
+        continue
+      }
+
+      processed += 1
+      console.log(
+        `[sweep] "${slug}": ${result.generated} description(s), ${result.failed} failed`
+      )
+    } catch (error) {
+      processed += 1
+      console.error(`[sweep] Failed on "${slug}":`, (error as Error).message)
+    }
+  }
+
+  console.log(`[sweep] Deep scan done. ${processed} course(s) touched.`)
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch(error => {
+    console.error("[sweep] Fatal error:", error)
+    process.exit(1)
+  })