@learnpack/learnpack 5.0.352 → 5.0.354

package/src/utils/descriptions/s3Storage.ts

@@ -0,0 +1,214 @@
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"
+import { CloudFrontClient } from "@aws-sdk/client-cloudfront"
+import { Syllabus } from "../../models/creator"
+import { PACKAGE_MANIFEST_REL_PATH, PackageManifest } from "../packageManifest"
+import {
+  AwsClient,
+  fetchJsonObject,
+  fetchTextObject,
+  invalidatePackageManifestPaths,
+  listObjectKeys,
+  processPackage,
+  withRetry,
+} from "../s3/packageManifestBackfill"
+import { SyllabusSyncStorage } from "../syllabusSync"
+import {
+  awsRegion,
+  requireAwsCredentials,
+  requireS3PackagesBucket,
+} from "../awsCredentials"
+import { CourseDescriptionsStorage } from "./generateCourseDescriptions"
+import { CourseExercise } from "./workList"
+
+/**
+ * Published-bucket (S3) adapter for the description flows.
+ *
+ * The published package is the snapshot descriptions are generated from: it is
+ * immutable between publications, unlike the draft in GCS, which keeps moving
+ * while the teacher edits.
+ */
+
+export const syllabusKey = (slug: string) =>
+  `${slug}/.learn/initialSyllabus.json`
+
+export const manifestKey = (slug: string) =>
+  `${slug}/${PACKAGE_MANIFEST_REL_PATH}`
+
+const configKeys = (slug: string) => [
+  `${slug}/.learn/config.json`,
+  `${slug}/config.json`,
+]
+
+type ConfigShape = {
+  exercises?: CourseExercise[];
+};
+
+export async function fetchExercises(
+  s3: AwsClient,
+  bucket: string,
+  slug: string
+): Promise<CourseExercise[]> {
+  for (const key of configKeys(slug)) {
+    // eslint-disable-next-line no-await-in-loop -- the second key is a fallback
+    const config = await fetchJsonObject<ConfigShape>(s3, bucket, key)
+    if (config?.exercises && Array.isArray(config.exercises)) {
+      return config.exercises
+    }
+  }
+
+  return []
+}
+
+export function createS3SyllabusSyncStorage(
+  s3: AwsClient,
+  bucket: string
+): SyllabusSyncStorage {
+  return {
+    async listExerciseFolderFileCounts(courseSlug) {
+      const prefix = `${courseSlug}/exercises/`
+      const keys = await listObjectKeys(s3, bucket, prefix)
+      const counts = new Map<string, number>()
+      for (const key of keys) {
+        const segment = key.slice(prefix.length).split("/")[0]
+        if (segment) {
+          counts.set(segment, (counts.get(segment) || 0) + 1)
+        }
+      }
+
+      return counts
+    },
+    async translationLangsBySlug(courseSlug) {
+      const exercises = await fetchExercises(s3, bucket, courseSlug)
+      const map = new Map<string, string[]>()
+      for (const exercise of exercises) {
+        const langs = Object.keys(exercise.translations || {})
+          .map(lang => lang.toLowerCase())
+          .filter(Boolean)
+        if (langs.length > 0) {
+          map.set(exercise.slug, [...new Set(langs)])
+        }
+      }
+
+      return map
+    },
+  }
+}
+
+export type S3DescriptionsStorageOptions = {
+  /** CloudFront distribution to invalidate after re-projecting the manifest. */
+  cloudFront?: { client: AwsClient; distributionId: string };
+  /** Set false in the backfill, where manifests are projected in a later pass. */
+  reprojectManifest?: boolean;
+};
+
+/**
+ * Storage for the published bucket, wired from the environment. Used by the
+ * post-publish stage and the sweep, which have no CLI flags to read.
+ */
+export function createS3DescriptionsStorageFromEnv(): CourseDescriptionsStorage {
+  // Validated here so a missing variable fails with its own name, instead of
+  // surfacing minutes later as "Could not load credentials from any providers"
+  // from inside a background job.
+  const credentials = requireAwsCredentials()
+  const region = awsRegion()
+
+  const s3 = new S3Client({ region, credentials }) as unknown as AwsClient
+
+  const distributionId = (process.env.CLOUDFRONT_DISTRIBUTION_ID || "").trim()
+  const cloudFront = distributionId ?
+    {
+        client: new CloudFrontClient({
+          region,
+          credentials,
+        }) as unknown as AwsClient,
+        distributionId,
+      } :
+    undefined
+
+  if (!cloudFront) {
+    console.warn(
+      "[descriptions] CLOUDFRONT_DISTRIBUTION_ID is not set: the manifest will be re-projected but not invalidated"
+    )
+  }
+
+  return createS3DescriptionsStorage(s3, requireS3PackagesBucket(), {
+    cloudFront,
+  })
+}
+
+export function createS3DescriptionsStorage(
+  s3: AwsClient,
+  bucket: string,
+  options: S3DescriptionsStorageOptions = {}
+): CourseDescriptionsStorage {
+  const storage: CourseDescriptionsStorage = {
+    async readSyllabus(courseSlug) {
+      return fetchJsonObject<Syllabus>(s3, bucket, syllabusKey(courseSlug))
+    },
+    async writeSyllabus(courseSlug, syllabus) {
+      await withRetry(() =>
+        s3.send(
+          new PutObjectCommand({
+            Bucket: bucket,
+            Key: syllabusKey(courseSlug),
+            Body: JSON.stringify(syllabus, null, 2),
+            ContentType: "application/json",
+          })
+        )
+      )
+    },
+    async listExercises(courseSlug) {
+      return fetchExercises(s3, bucket, courseSlug)
+    },
+    async readReadme(courseSlug, exerciseSlug, fileName) {
+      return fetchTextObject(
+        s3,
+        bucket,
+        `${courseSlug}/exercises/${exerciseSlug}/${fileName}`
+      )
+    },
+    async readManifest(courseSlug) {
+      return fetchJsonObject<PackageManifest>(
+        s3,
+        bucket,
+        manifestKey(courseSlug)
+      )
+    },
+    syllabusSyncStorage: createS3SyllabusSyncStorage(s3, bucket),
+  }
+
+  if (options.reprojectManifest !== false) {
+    storage.reprojectManifest = async courseSlug => {
+      // Preserve the publication timestamp: this is a re-projection of an
+      // already published package, not a new publication.
+      const existing = await fetchJsonObject<PackageManifest>(
+        s3,
+        bucket,
+        manifestKey(courseSlug)
+      )
+
+      const result = await processPackage(s3, bucket, courseSlug, {
+        dryRun: false,
+        force: false,
+        skipExisting: false,
+        publishedAt: existing?.publishedAt ?? null,
+      })
+
+      if (result.status === "failed") {
+        throw new Error(result.error || "manifest projection failed")
+      }
+
+      if (result.status === "skipped" || !options.cloudFront) {
+        return
+      }
+
+      await invalidatePackageManifestPaths(
+        options.cloudFront.client,
+        options.cloudFront.distributionId,
+        [`/${manifestKey(courseSlug)}`]
+      )
+    }
+  }
+
+  return storage
+}

package/src/utils/descriptions/workList.ts

@@ -0,0 +1,283 @@
+import { Lesson, Syllabus } from "../../models/creator"
+import { slugify } from "../creatorUtilities"
+import {
+  ContentFingerprint,
+  fingerprintReadme,
+  getSimhashThreshold,
+  isDescriptionStale,
+} from "../descriptionHash"
+import { DESCRIPTION_PROMPT_VERSION } from "../packageManifest"
+
+/**
+ * Deciding WHAT needs a description, separated from generating it.
+ *
+ * The publish route needs the answer before doing any work (to tell breathecode
+ * whether a second event is coming), and the generation service needs the very
+ * same list right after. Keeping the decision here, pure and content-only,
+ * means both answer identically — the alternative, two implementations of
+ * "is this stale?", is exactly how the backfill and the sweep drifted apart.
+ */
+
+export type CourseExercise = {
+  slug: string;
+  /** language code -> README file name */
+  translations: Record<string, string>;
+};
+
+/** README contents by exercise slug and language. */
+export type CourseReadmes = Record<string, Record<string, string>>;
+
+export type LanguageWork = {
+  lang: string;
+  fingerprint: ContentFingerprint;
+};
+
+export type StepWork = {
+  exerciseSlug: string;
+  /** Reference into the syllabus; the generation service writes through it. */
+  lesson: Lesson;
+  /** Language the description is written from and translated out of. */
+  baseLanguage: string;
+  baseContent: string;
+  languages: LanguageWork[];
+};
+
+export type BuildWorkListInput = {
+  syllabus: Syllabus;
+  exercises: CourseExercise[];
+  readmes: CourseReadmes;
+  promptVersion?: number;
+  simhashThreshold?: number;
+  /** Regenerate everything except human-edited descriptions. */
+  force?: boolean;
+};
+
+export function findSyllabusLesson(
+  syllabus: Syllabus,
+  exerciseSlug: string
+): Lesson | undefined {
+  return syllabus.lessons.find(
+    lesson =>
+      lesson.uid === exerciseSlug ||
+      slugify(lesson.id + "-" + lesson.title) === exerciseSlug
+  )
+}
+
+/**
+ * Language a course is actually written in.
+ *
+ * `courseInfo.language` is trusted only when some exercise really has a README
+ * in it; otherwise the language is inferred from the content itself (the most
+ * frequent translation across exercises). Defaulting to "en" when the field is
+ * missing would label Spanish courses as English and generate their
+ * descriptions from a translation instead of the original.
+ */
+export function inferCourseBaseLanguage(
+  syllabus: Syllabus,
+  exercises: CourseExercise[]
+): string | null {
+  const counts = new Map<string, number>()
+  for (const exercise of exercises) {
+    for (const lang of Object.keys(exercise.translations || {})) {
+      counts.set(lang, (counts.get(lang) || 0) + 1)
+    }
+  }
+
+  if (counts.size === 0) {
+    return null
+  }
+
+  const declared = syllabus.courseInfo?.language?.toLowerCase()
+  if (declared) {
+    for (const lang of counts.keys()) {
+      if (lang.toLowerCase() === declared) {
+        return lang
+      }
+    }
+  }
+
+  let best: string | null = null
+  let bestCount = 0
+  for (const [lang, count] of counts) {
+    if (count > bestCount) {
+      best = lang
+      bestCount = count
+    }
+  }
+
+  return best
+}
+
+/** Language to generate this step from: the course's, when it has it. */
+function resolveStepBaseLanguage(
+  exercise: CourseExercise,
+  courseBaseLanguage: string | null
+): string | null {
+  const langs = Object.keys(exercise.translations || {})
+  if (langs.length === 0) {
+    return null
+  }
+
+  if (courseBaseLanguage && langs.includes(courseBaseLanguage)) {
+    return courseBaseLanguage
+  }
+
+  return langs[0]
+}
+
+/**
+ * Whether a (step, language) needs generating. Human-edited descriptions are
+ * never touched; an explicit "not enough content" answer (a stored null with a
+ * fingerprint) is respected until the content itself changes.
+ */
+export function needsGeneration(
+  slot: NonNullable<Lesson["translations"]>[string] | undefined,
+  fingerprint: ContentFingerprint,
+  options: {
+    promptVersion: number;
+    simhashThreshold: number;
+    force: boolean;
+  }
+): boolean {
+  if (slot?.descriptionSource === "human") {
+    return false
+  }
+
+  if (!slot) {
+    return true
+  }
+
+  if (options.force) {
+    return true
+  }
+
+  if (slot.descriptionStatus === "error") {
+    return true
+  }
+
+  if ((slot.descriptionPromptVersion ?? 0) < options.promptVersion) {
+    return true
+  }
+
+  return isDescriptionStale(
+    { sha256: slot.sourceContentHash, simhash: slot.sourceSimHash },
+    fingerprint,
+    options.simhashThreshold
+  )
+}
+
+/** The (step, language) pairs whose description is missing or out of date. */
+export function buildDescriptionWorkList(
+  input: BuildWorkListInput
+): StepWork[] {
+  const promptVersion = input.promptVersion ?? DESCRIPTION_PROMPT_VERSION
+  const simhashThreshold = input.simhashThreshold ?? getSimhashThreshold()
+  const force = input.force ?? false
+  const courseBaseLanguage = inferCourseBaseLanguage(
+    input.syllabus,
+    input.exercises
+  )
+
+  const work: StepWork[] = []
+
+  for (const exercise of input.exercises) {
+    const lesson = findSyllabusLesson(input.syllabus, exercise.slug)
+    if (!lesson) {
+      continue
+    }
+
+    const baseLanguage = resolveStepBaseLanguage(exercise, courseBaseLanguage)
+    if (!baseLanguage) {
+      continue
+    }
+
+    const exerciseReadmes = input.readmes[exercise.slug] || {}
+    const baseContent = exerciseReadmes[baseLanguage]
+    if (typeof baseContent !== "string") {
+      continue
+    }
+
+    const languages: LanguageWork[] = []
+    for (const lang of Object.keys(exercise.translations || {})) {
+      const content = exerciseReadmes[lang]
+      if (typeof content !== "string") {
+        continue
+      }
+
+      // Each language's staleness is anchored to its OWN content, even though
+      // the description is generated from the base language: that is what makes
+      // "the Spanish README changed" regenerate only Spanish.
+      const fingerprint = fingerprintReadme(content)
+      if (
+        !needsGeneration(lesson.translations?.[lang], fingerprint, {
+          promptVersion,
+          simhashThreshold,
+          force,
+        })
+      ) {
+        continue
+      }
+
+      languages.push({ lang, fingerprint })
+    }
+
+    if (languages.length === 0) {
+      continue
+    }
+
+    work.push({
+      exerciseSlug: exercise.slug,
+      lesson,
+      baseLanguage,
+      baseContent,
+      languages,
+    })
+  }
+
+  return work
+}
+
+/** Total (step, language) pairs in a work list. */
+export function countWorkItems(work: StepWork[]): number {
+  return work.reduce((total, step) => total + step.languages.length, 0)
+}
+
+/**
+ * Cheap "has this course ever been fully processed at this prompt version?"
+ * check, from the syllabus alone.
+ *
+ * Deliberately weaker than the work list: it cannot see content changes because
+ * it never reads a README. That is the point — it lets a batched run skip
+ * finished courses without paying thousands of object reads. Use the work list
+ * whenever correctness matters; use this only as a pre-filter.
+ */
+export function isCourseSettled(
+  syllabus: Syllabus,
+  exercises: CourseExercise[],
+  promptVersion: number = DESCRIPTION_PROMPT_VERSION
+): boolean {
+  for (const exercise of exercises) {
+    const lesson = findSyllabusLesson(syllabus, exercise.slug)
+    if (!lesson) {
+      return false
+    }
+
+    for (const lang of Object.keys(exercise.translations || {})) {
+      const slot = lesson.translations?.[lang]
+      if (slot?.descriptionSource === "human") {
+        continue
+      }
+
+      if (
+        !slot ||
+        slot.descriptionStatus === "error" ||
+        !slot.sourceContentHash ||
+        (slot.descriptionPromptVersion ?? 0) < promptVersion
+      ) {
+        return false
+      }
+    }
+  }
+
+  return true
+}

package/src/utils/gcsBucketName.ts

@@ -0,0 +1,19 @@
+/**
+ * The draft bucket name, required rather than defaulted.
+ *
+ * The code used to fall back to "learnpack-packages", a name the service
+ * account cannot reach: with the variable missing, the process would start
+ * cleanly and then fail with a 403 on every single object operation. Failing at
+ * boot with the reason turns a puzzling runtime outage into an obvious
+ * misconfiguration.
+ */
+export function requireGcsBucketName(): string {
+  const name = (process.env.GCP_BUCKET_NAME || "").trim()
+  if (!name) {
+    throw new Error(
+      "GCP_BUCKET_NAME (env) is required: it names the bucket holding course drafts"
+    )
+  }
+
+  return name
+}

package/src/utils/packageManifest.ts

@@ -13,6 +13,14 @@ const frontMatter = require("front-matter")
 export const PACKAGE_MANIFEST_FILENAME = "package-manifest.json"
 export const PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json"
 export const SCHEMA_VERSION = 1
+// Version of the Rigobot prompt/model used to generate step descriptions. It
+// travels inside the completion inputs, so bumping it both busts Rigobot's
+// cache and marks every description stored at a lower version for regeneration.
+// Keep in sync with the prompt body provisioned in Rigobot.
+export const DESCRIPTION_PROMPT_VERSION = 1
+
+// Target length, in words, of a generated step description.
+export const DESCRIPTION_TARGET_WORD_COUNT = 25
 
 export type LessonType = "READ" | "CODE" | "QUIZ";
 
@@ -86,7 +94,14 @@ function findSyllabusLesson(
   })
 }
 
-function deriveLessonId(exerciseSlug: string): string {
+/**
+ * Lesson id when no syllabus entry matches: the numeric prefix of the folder.
+ *
+ * Exported because `legacyPackageRepair` has to synthesize syllabus ids with
+ * exactly this rule — a repaired package whose ids differed from the ones its
+ * manifest already carries would look like a content change to every consumer.
+ */
+export function deriveLessonId(exerciseSlug: string): string {
   const match = exerciseSlug.match(/^(\d+(?:\.\d+)?)/)
   return match ? match[1] : exerciseSlug
 }
@@ -167,13 +182,18 @@ function resolveLessonTitle(
 }
 
 function resolveLessonDescriptions(
-  exercise: Exercise
+  exercise: Exercise,
+  syllabusLesson: Lesson | undefined
 ): Record<string, string | null> {
   const description: Record<string, string | null> = {}
 
+  // Descriptions are a projection: the source of truth lives in the syllabus
+  // lesson (translations[lang].description), populated by the sweep/backfill.
+  // When absent we emit null (the frontend hides the area) rather than a
+  // low-quality placeholder.
   for (const lang of Object.keys(exercise.translations)) {
-    // TODO: add description content
-    description[lang] = null
+    description[lang] =
+      syllabusLesson?.translations?.[lang]?.description ?? null
   }
 
   return description
@@ -240,7 +260,7 @@ export function buildPackageManifestFromSources(
         courseLang,
         readmeFrontmatters[exercise.slug] ?? {}
       ),
-      description: resolveLessonDescriptions(exercise),
+      description: resolveLessonDescriptions(exercise, syllabusLesson),
       video: resolveLessonVideos(exercise, readmeFrontmatters),
     }
   })
@@ -312,6 +332,43 @@ export function serializePackageManifest(manifest: PackageManifest): string {
   return JSON.stringify(manifest, null, 2)
 }
 
+/** The manifest as it travels inside a publication event. */
+export type PackageManifestEventView = Omit<PackageManifest, "generatedAt">;
+
+/**
+ * Project a manifest into the shape sent to breathecode.
+ *
+ * Today this is nearly the identity — only `generatedAt`, an internal detail of
+ * the projection, is dropped. It exists as an explicit pick so that fields
+ * added to the manifest later (hashes, generation flags) do not silently become
+ * part of an external contract: joining the event payload has to be a decision
+ * visible in a diff.
+ */
+export function serializePackageManifestForEvent(
+  manifest: PackageManifest
+): PackageManifestEventView {
+  return {
+    schemaVersion: manifest.schemaVersion,
+    publishedAt: manifest.publishedAt,
+    slug: manifest.slug,
+    title: manifest.title,
+    description: manifest.description,
+    preview: manifest.preview,
+    technologies: manifest.technologies,
+    difficulty: manifest.difficulty,
+    duration: manifest.duration,
+    lessons: manifest.lessons.map(lesson => ({
+      id: lesson.id,
+      slug: lesson.slug,
+      position: lesson.position,
+      type: lesson.type,
+      title: lesson.title,
+      description: lesson.description,
+      video: lesson.video,
+    })),
+  }
+}
+
 async function readReadmeFrontmatter(
   bucket: Bucket,
   exerciseSlug: string,