@learnpack/learnpack 5.0.352 → 5.0.353

package/src/utils/descriptions/resumePublication.ts

@@ -0,0 +1,200 @@
+import {
+  abandonJournal,
+  finalizeJournal,
+  incrementAttempts,
+  isJournalStale,
+  JournalStorage,
+  markStage,
+  PublishJournal,
+} from "../publishJournal"
+import { PackageManifest } from "../packageManifest"
+import {
+  buildPackageManifestUpdatedEvent,
+  fetchPackageInfo,
+  sendPublishEvent,
+} from "../publishEvents"
+import {
+  CourseDescriptionsStorage,
+  GenerateCourseDescriptionsOptions,
+} from "./generateCourseDescriptions"
+import { runPublishDescriptionsStage } from "./publishStage"
+
+/**
+ * Recovery of a publication whose background stage never finished.
+ *
+ * The stage runs inside the web process, so a dyno restart mid-flight leaves a
+ * package published but not described, and — worse — breathecode waiting for a
+ * manifest event that was promised and will never arrive. Content-wise the
+ * missing descriptions would eventually be picked up by the next publication,
+ * but the promised event has no such second chance: emitting it is the reason
+ * this exists.
+ */
+
+export const MAX_SWEEP_ATTEMPTS = 3
+
+/** Only resume publications nobody is working on any more. */
+export const DEFAULT_STALE_AFTER_MS = 15 * 60 * 1000
+
+export type ResumeOutcome =
+  | "skipped-recent"
+  | "skipped-abandoned"
+  | "abandoned"
+  | "resumed"
+  | "nothing-to-do";
+
+export type ResumeDeps = {
+  journalStorage: JournalStorage;
+  publishedStorage: CourseDescriptionsStorage;
+  draftStorage: CourseDescriptionsStorage;
+  rigobotToken: string;
+  breathecodeToken?: string;
+  staleAfterMs?: number;
+  now?: number;
+  /** Injection points for tests. */
+  emit?: typeof sendPublishEvent;
+  readPackageInfo?: typeof fetchPackageInfo;
+  generate?: GenerateCourseDescriptionsOptions["generate"];
+};
+
+function isDone(
+  journal: PublishJournal,
+  stage: "descriptions" | "gcsMirror" | "manifestEvent"
+): boolean {
+  return journal.stages[stage].status === "done"
+}
+
+/**
+ * Whether breathecode is still owed a manifest event: one was promised (the
+ * publish event announced `queued`) and it never went out.
+ */
+export function owesManifestEvent(journal: PublishJournal): boolean {
+  const promised =
+    (
+      journal.stages.publishedEvent.meta as
+        | { descriptions?: string }
+        | undefined
+    )?.descriptions === "queued"
+  return promised && !isDone(journal, "manifestEvent")
+}
+
+export async function resumePublication(
+  journal: PublishJournal,
+  deps: ResumeDeps
+): Promise<ResumeOutcome> {
+  const { journalStorage } = deps
+  const now = deps.now ?? Date.now()
+
+  if (journal.abandoned) {
+    return "skipped-abandoned"
+  }
+
+  // A publication still in flight would race with its own stage.
+  if (
+    !isJournalStale(journal, deps.staleAfterMs ?? DEFAULT_STALE_AFTER_MS, now)
+  ) {
+    return "skipped-recent"
+  }
+
+  const needsDescriptions =
+    !isDone(journal, "descriptions") || !isDone(journal, "gcsMirror")
+  const needsEvent = owesManifestEvent(journal)
+
+  if (!needsDescriptions && !needsEvent) {
+    // Nothing left that the sweep can do; stage A failures are not its job.
+    await finalizeJournal(journalStorage, journal)
+    return "nothing-to-do"
+  }
+
+  if (journal.attempts >= MAX_SWEEP_ATTEMPTS) {
+    await abandonJournal(journalStorage, journal)
+    console.error(
+      `[sweep] Giving up on "${journal.courseSlug}" (${journal.publishId}) after ${journal.attempts} attempts`
+    )
+    return "abandoned"
+  }
+
+  await incrementAttempts(journalStorage, journal)
+
+  let succeeded = true
+  let stats: { generated: number; failed: number; missing: number } | undefined
+
+  if (needsDescriptions) {
+    const outcome = await runPublishDescriptionsStage({
+      courseSlug: journal.courseSlug,
+      rigobotToken: deps.rigobotToken,
+      journal,
+      journalStorage,
+      publishedStorage: deps.publishedStorage,
+      draftStorage: deps.draftStorage,
+      generate: deps.generate,
+    })
+    succeeded = outcome.succeeded
+    if (outcome.generation) {
+      stats = {
+        generated: outcome.generation.generated,
+        failed: outcome.generation.failed,
+        missing: outcome.generation.missing,
+      }
+    }
+  }
+
+  if (needsEvent) {
+    await emitPendingManifestEvent(journal, deps, succeeded, stats)
+  }
+
+  await finalizeJournal(journalStorage, journal)
+  return "resumed"
+}
+
+async function emitPendingManifestEvent(
+  journal: PublishJournal,
+  deps: ResumeDeps,
+  succeeded: boolean,
+  stats?: { generated: number; failed: number; missing: number }
+): Promise<void> {
+  const emit = deps.emit ?? sendPublishEvent
+  const readPackageInfo = deps.readPackageInfo ?? fetchPackageInfo
+
+  if (!deps.breathecodeToken) {
+    await markStage(deps.journalStorage, journal, "manifestEvent", "failed", {
+      error: "BREATHECODE_SYSTEM_TOKEN is not configured",
+    })
+    return
+  }
+
+  try {
+    let manifest: PackageManifest | null = null
+    if (deps.publishedStorage.readManifest) {
+      manifest = await deps.publishedStorage.readManifest(journal.courseSlug)
+    }
+
+    const delivered = await emit(
+      buildPackageManifestUpdatedEvent(
+        {
+          publishId: journal.publishId,
+          courseSlug: journal.courseSlug,
+          packageInfo: await readPackageInfo(
+            journal.courseSlug,
+            deps.rigobotToken
+          ),
+          manifest,
+        },
+        succeeded ? "success" : "failed",
+        stats
+      ),
+      deps.breathecodeToken
+    )
+
+    await markStage(
+      deps.journalStorage,
+      journal,
+      "manifestEvent",
+      delivered ? "done" : "failed",
+      { error: delivered ? undefined : "delivery failed" }
+    )
+  } catch (error) {
+    await markStage(deps.journalStorage, journal, "manifestEvent", "failed", {
+      error: (error as Error).message,
+    })
+  }
+}

package/src/utils/descriptions/s3Storage.ts

@@ -0,0 +1,206 @@
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"
+import { CloudFrontClient } from "@aws-sdk/client-cloudfront"
+import { Syllabus } from "../../models/creator"
+import { PACKAGE_MANIFEST_REL_PATH, PackageManifest } from "../packageManifest"
+import {
+  AwsClient,
+  fetchJsonObject,
+  fetchTextObject,
+  invalidatePackageManifestPaths,
+  listObjectKeys,
+  processPackage,
+  withRetry,
+} from "../s3/packageManifestBackfill"
+import { SyllabusSyncStorage } from "../syllabusSync"
+import { CourseDescriptionsStorage } from "./generateCourseDescriptions"
+import { CourseExercise } from "./workList"
+
+/**
+ * Published-bucket (S3) adapter for the description flows.
+ *
+ * The published package is the snapshot descriptions are generated from: it is
+ * immutable between publications, unlike the draft in GCS, which keeps moving
+ * while the teacher edits.
+ */
+
+export const syllabusKey = (slug: string) =>
+  `${slug}/.learn/initialSyllabus.json`
+
+export const manifestKey = (slug: string) =>
+  `${slug}/${PACKAGE_MANIFEST_REL_PATH}`
+
+const configKeys = (slug: string) => [
+  `${slug}/.learn/config.json`,
+  `${slug}/config.json`,
+]
+
+type ConfigShape = {
+  exercises?: CourseExercise[];
+};
+
+export async function fetchExercises(
+  s3: AwsClient,
+  bucket: string,
+  slug: string
+): Promise<CourseExercise[]> {
+  for (const key of configKeys(slug)) {
+    // eslint-disable-next-line no-await-in-loop -- the second key is a fallback
+    const config = await fetchJsonObject<ConfigShape>(s3, bucket, key)
+    if (config?.exercises && Array.isArray(config.exercises)) {
+      return config.exercises
+    }
+  }
+
+  return []
+}
+
+export function createS3SyllabusSyncStorage(
+  s3: AwsClient,
+  bucket: string
+): SyllabusSyncStorage {
+  return {
+    async listExerciseFolderFileCounts(courseSlug) {
+      const prefix = `${courseSlug}/exercises/`
+      const keys = await listObjectKeys(s3, bucket, prefix)
+      const counts = new Map<string, number>()
+      for (const key of keys) {
+        const segment = key.slice(prefix.length).split("/")[0]
+        if (segment) {
+          counts.set(segment, (counts.get(segment) || 0) + 1)
+        }
+      }
+
+      return counts
+    },
+    async translationLangsBySlug(courseSlug) {
+      const exercises = await fetchExercises(s3, bucket, courseSlug)
+      const map = new Map<string, string[]>()
+      for (const exercise of exercises) {
+        const langs = Object.keys(exercise.translations || {})
+          .map(lang => lang.toLowerCase())
+          .filter(Boolean)
+        if (langs.length > 0) {
+          map.set(exercise.slug, [...new Set(langs)])
+        }
+      }
+
+      return map
+    },
+  }
+}
+
+export type S3DescriptionsStorageOptions = {
+  /** CloudFront distribution to invalidate after re-projecting the manifest. */
+  cloudFront?: { client: AwsClient; distributionId: string };
+  /** Set false in the backfill, where manifests are projected in a later pass. */
+  reprojectManifest?: boolean;
+};
+
+/**
+ * Storage for the published bucket, wired from the environment. Used by the
+ * post-publish stage and the sweep, which have no CLI flags to read.
+ */
+export function createS3DescriptionsStorageFromEnv(): CourseDescriptionsStorage {
+  const s3 = new S3Client({
+    region: process.env.AWS_REGION || "us-east-1",
+  }) as unknown as AwsClient
+
+  const distributionId = process.env.CLOUDFRONT_DISTRIBUTION_ID
+  const cloudFront = distributionId ?
+    {
+        client: new CloudFrontClient({
+          region: process.env.AWS_REGION || "us-east-1",
+        }) as unknown as AwsClient,
+        distributionId,
+      } :
+    undefined
+
+  if (!cloudFront) {
+    console.warn(
+      "[descriptions] CLOUDFRONT_DISTRIBUTION_ID is not set: the manifest will be re-projected but not invalidated"
+    )
+  }
+
+  return createS3DescriptionsStorage(
+    s3,
+    process.env.S3_PACKAGES_BUCKET || "learnpack-paquetes",
+    { cloudFront }
+  )
+}
+
+export function createS3DescriptionsStorage(
+  s3: AwsClient,
+  bucket: string,
+  options: S3DescriptionsStorageOptions = {}
+): CourseDescriptionsStorage {
+  const storage: CourseDescriptionsStorage = {
+    async readSyllabus(courseSlug) {
+      return fetchJsonObject<Syllabus>(s3, bucket, syllabusKey(courseSlug))
+    },
+    async writeSyllabus(courseSlug, syllabus) {
+      await withRetry(() =>
+        s3.send(
+          new PutObjectCommand({
+            Bucket: bucket,
+            Key: syllabusKey(courseSlug),
+            Body: JSON.stringify(syllabus, null, 2),
+            ContentType: "application/json",
+          })
+        )
+      )
+    },
+    async listExercises(courseSlug) {
+      return fetchExercises(s3, bucket, courseSlug)
+    },
+    async readReadme(courseSlug, exerciseSlug, fileName) {
+      return fetchTextObject(
+        s3,
+        bucket,
+        `${courseSlug}/exercises/${exerciseSlug}/${fileName}`
+      )
+    },
+    async readManifest(courseSlug) {
+      return fetchJsonObject<PackageManifest>(
+        s3,
+        bucket,
+        manifestKey(courseSlug)
+      )
+    },
+    syllabusSyncStorage: createS3SyllabusSyncStorage(s3, bucket),
+  }
+
+  if (options.reprojectManifest !== false) {
+    storage.reprojectManifest = async courseSlug => {
+      // Preserve the publication timestamp: this is a re-projection of an
+      // already published package, not a new publication.
+      const existing = await fetchJsonObject<PackageManifest>(
+        s3,
+        bucket,
+        manifestKey(courseSlug)
+      )
+
+      const result = await processPackage(s3, bucket, courseSlug, {
+        dryRun: false,
+        force: false,
+        skipExisting: false,
+        publishedAt: existing?.publishedAt ?? null,
+      })
+
+      if (result.status === "failed") {
+        throw new Error(result.error || "manifest projection failed")
+      }
+
+      if (result.status === "skipped" || !options.cloudFront) {
+        return
+      }
+
+      await invalidatePackageManifestPaths(
+        options.cloudFront.client,
+        options.cloudFront.distributionId,
+        [`/${manifestKey(courseSlug)}`]
+      )
+    }
+  }
+
+  return storage
+}

package/src/utils/descriptions/workList.ts

@@ -0,0 +1,283 @@
+import { Lesson, Syllabus } from "../../models/creator"
+import { slugify } from "../creatorUtilities"
+import {
+  ContentFingerprint,
+  fingerprintReadme,
+  getSimhashThreshold,
+  isDescriptionStale,
+} from "../descriptionHash"
+import { DESCRIPTION_PROMPT_VERSION } from "../packageManifest"
+
+/**
+ * Deciding WHAT needs a description, separated from generating it.
+ *
+ * The publish route needs the answer before doing any work (to tell breathecode
+ * whether a second event is coming), and the generation service needs the very
+ * same list right after. Keeping the decision here, pure and content-only,
+ * means both answer identically — the alternative, two implementations of
+ * "is this stale?", is exactly how the backfill and the sweep drifted apart.
+ */
+
+export type CourseExercise = {
+  slug: string;
+  /** language code -> README file name */
+  translations: Record<string, string>;
+};
+
+/** README contents by exercise slug and language. */
+export type CourseReadmes = Record<string, Record<string, string>>;
+
+export type LanguageWork = {
+  lang: string;
+  fingerprint: ContentFingerprint;
+};
+
+export type StepWork = {
+  exerciseSlug: string;
+  /** Reference into the syllabus; the generation service writes through it. */
+  lesson: Lesson;
+  /** Language the description is written from and translated out of. */
+  baseLanguage: string;
+  baseContent: string;
+  languages: LanguageWork[];
+};
+
+export type BuildWorkListInput = {
+  syllabus: Syllabus;
+  exercises: CourseExercise[];
+  readmes: CourseReadmes;
+  promptVersion?: number;
+  simhashThreshold?: number;
+  /** Regenerate everything except human-edited descriptions. */
+  force?: boolean;
+};
+
+export function findSyllabusLesson(
+  syllabus: Syllabus,
+  exerciseSlug: string
+): Lesson | undefined {
+  return syllabus.lessons.find(
+    lesson =>
+      lesson.uid === exerciseSlug ||
+      slugify(lesson.id + "-" + lesson.title) === exerciseSlug
+  )
+}
+
+/**
+ * Language a course is actually written in.
+ *
+ * `courseInfo.language` is trusted only when some exercise really has a README
+ * in it; otherwise the language is inferred from the content itself (the most
+ * frequent translation across exercises). Defaulting to "en" when the field is
+ * missing would label Spanish courses as English and generate their
+ * descriptions from a translation instead of the original.
+ */
+export function inferCourseBaseLanguage(
+  syllabus: Syllabus,
+  exercises: CourseExercise[]
+): string | null {
+  const counts = new Map<string, number>()
+  for (const exercise of exercises) {
+    for (const lang of Object.keys(exercise.translations || {})) {
+      counts.set(lang, (counts.get(lang) || 0) + 1)
+    }
+  }
+
+  if (counts.size === 0) {
+    return null
+  }
+
+  const declared = syllabus.courseInfo?.language?.toLowerCase()
+  if (declared) {
+    for (const lang of counts.keys()) {
+      if (lang.toLowerCase() === declared) {
+        return lang
+      }
+    }
+  }
+
+  let best: string | null = null
+  let bestCount = 0
+  for (const [lang, count] of counts) {
+    if (count > bestCount) {
+      best = lang
+      bestCount = count
+    }
+  }
+
+  return best
+}
+
+/** Language to generate this step from: the course's, when it has it. */
+function resolveStepBaseLanguage(
+  exercise: CourseExercise,
+  courseBaseLanguage: string | null
+): string | null {
+  const langs = Object.keys(exercise.translations || {})
+  if (langs.length === 0) {
+    return null
+  }
+
+  if (courseBaseLanguage && langs.includes(courseBaseLanguage)) {
+    return courseBaseLanguage
+  }
+
+  return langs[0]
+}
+
+/**
+ * Whether a (step, language) needs generating. Human-edited descriptions are
+ * never touched; an explicit "not enough content" answer (a stored null with a
+ * fingerprint) is respected until the content itself changes.
+ */
+export function needsGeneration(
+  slot: NonNullable<Lesson["translations"]>[string] | undefined,
+  fingerprint: ContentFingerprint,
+  options: {
+    promptVersion: number;
+    simhashThreshold: number;
+    force: boolean;
+  }
+): boolean {
+  if (slot?.descriptionSource === "human") {
+    return false
+  }
+
+  if (!slot) {
+    return true
+  }
+
+  if (options.force) {
+    return true
+  }
+
+  if (slot.descriptionStatus === "error") {
+    return true
+  }
+
+  if ((slot.descriptionPromptVersion ?? 0) < options.promptVersion) {
+    return true
+  }
+
+  return isDescriptionStale(
+    { sha256: slot.sourceContentHash, simhash: slot.sourceSimHash },
+    fingerprint,
+    options.simhashThreshold
+  )
+}
+
+/** The (step, language) pairs whose description is missing or out of date. */
+export function buildDescriptionWorkList(
+  input: BuildWorkListInput
+): StepWork[] {
+  const promptVersion = input.promptVersion ?? DESCRIPTION_PROMPT_VERSION
+  const simhashThreshold = input.simhashThreshold ?? getSimhashThreshold()
+  const force = input.force ?? false
+  const courseBaseLanguage = inferCourseBaseLanguage(
+    input.syllabus,
+    input.exercises
+  )
+
+  const work: StepWork[] = []
+
+  for (const exercise of input.exercises) {
+    const lesson = findSyllabusLesson(input.syllabus, exercise.slug)
+    if (!lesson) {
+      continue
+    }
+
+    const baseLanguage = resolveStepBaseLanguage(exercise, courseBaseLanguage)
+    if (!baseLanguage) {
+      continue
+    }
+
+    const exerciseReadmes = input.readmes[exercise.slug] || {}
+    const baseContent = exerciseReadmes[baseLanguage]
+    if (typeof baseContent !== "string") {
+      continue
+    }
+
+    const languages: LanguageWork[] = []
+    for (const lang of Object.keys(exercise.translations || {})) {
+      const content = exerciseReadmes[lang]
+      if (typeof content !== "string") {
+        continue
+      }
+
+      // Each language's staleness is anchored to its OWN content, even though
+      // the description is generated from the base language: that is what makes
+      // "the Spanish README changed" regenerate only Spanish.
+      const fingerprint = fingerprintReadme(content)
+      if (
+        !needsGeneration(lesson.translations?.[lang], fingerprint, {
+          promptVersion,
+          simhashThreshold,
+          force,
+        })
+      ) {
+        continue
+      }
+
+      languages.push({ lang, fingerprint })
+    }
+
+    if (languages.length === 0) {
+      continue
+    }
+
+    work.push({
+      exerciseSlug: exercise.slug,
+      lesson,
+      baseLanguage,
+      baseContent,
+      languages,
+    })
+  }
+
+  return work
+}
+
+/** Total (step, language) pairs in a work list. */
+export function countWorkItems(work: StepWork[]): number {
+  return work.reduce((total, step) => total + step.languages.length, 0)
+}
+
+/**
+ * Cheap "has this course ever been fully processed at this prompt version?"
+ * check, from the syllabus alone.
+ *
+ * Deliberately weaker than the work list: it cannot see content changes because
+ * it never reads a README. That is the point — it lets a batched run skip
+ * finished courses without paying thousands of object reads. Use the work list
+ * whenever correctness matters; use this only as a pre-filter.
+ */
+export function isCourseSettled(
+  syllabus: Syllabus,
+  exercises: CourseExercise[],
+  promptVersion: number = DESCRIPTION_PROMPT_VERSION
+): boolean {
+  for (const exercise of exercises) {
+    const lesson = findSyllabusLesson(syllabus, exercise.slug)
+    if (!lesson) {
+      return false
+    }
+
+    for (const lang of Object.keys(exercise.translations || {})) {
+      const slot = lesson.translations?.[lang]
+      if (slot?.descriptionSource === "human") {
+        continue
+      }
+
+      if (
+        !slot ||
+        slot.descriptionStatus === "error" ||
+        !slot.sourceContentHash ||
+        (slot.descriptionPromptVersion ?? 0) < promptVersion
+      ) {
+        return false
+      }
+    }
+  }
+
+  return true
+}

package/src/utils/gcsBucketName.ts

@@ -0,0 +1,19 @@
+/**
+ * The draft bucket name, required rather than defaulted.
+ *
+ * The code used to fall back to "learnpack-packages", a name the service
+ * account cannot reach: with the variable missing, the process would start
+ * cleanly and then fail with a 403 on every single object operation. Failing at
+ * boot with the reason turns a puzzling runtime outage into an obvious
+ * misconfiguration.
+ */
+export function requireGcsBucketName(): string {
+  const name = (process.env.GCP_BUCKET_NAME || "").trim()
+  if (!name) {
+    throw new Error(
+      "GCP_BUCKET_NAME (env) is required: it names the bucket holding course drafts"
+    )
+  }
+
+  return name
+}