@learnpack/learnpack 5.0.353 → 5.0.354

package/src/utils/descriptions/s3Storage.ts

@@ -1,206 +1,214 @@
-import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"
-import { CloudFrontClient } from "@aws-sdk/client-cloudfront"
-import { Syllabus } from "../../models/creator"
-import { PACKAGE_MANIFEST_REL_PATH, PackageManifest } from "../packageManifest"
-import {
-  AwsClient,
-  fetchJsonObject,
-  fetchTextObject,
-  invalidatePackageManifestPaths,
-  listObjectKeys,
-  processPackage,
-  withRetry,
-} from "../s3/packageManifestBackfill"
-import { SyllabusSyncStorage } from "../syllabusSync"
-import { CourseDescriptionsStorage } from "./generateCourseDescriptions"
-import { CourseExercise } from "./workList"
-
-/**
- * Published-bucket (S3) adapter for the description flows.
- *
- * The published package is the snapshot descriptions are generated from: it is
- * immutable between publications, unlike the draft in GCS, which keeps moving
- * while the teacher edits.
- */
-
-export const syllabusKey = (slug: string) =>
-  `${slug}/.learn/initialSyllabus.json`
-
-export const manifestKey = (slug: string) =>
-  `${slug}/${PACKAGE_MANIFEST_REL_PATH}`
-
-const configKeys = (slug: string) => [
-  `${slug}/.learn/config.json`,
-  `${slug}/config.json`,
-]
-
-type ConfigShape = {
-  exercises?: CourseExercise[];
-};
-
-export async function fetchExercises(
-  s3: AwsClient,
-  bucket: string,
-  slug: string
-): Promise<CourseExercise[]> {
-  for (const key of configKeys(slug)) {
-    // eslint-disable-next-line no-await-in-loop -- the second key is a fallback
-    const config = await fetchJsonObject<ConfigShape>(s3, bucket, key)
-    if (config?.exercises && Array.isArray(config.exercises)) {
-      return config.exercises
-    }
-  }
-
-  return []
-}
-
-export function createS3SyllabusSyncStorage(
-  s3: AwsClient,
-  bucket: string
-): SyllabusSyncStorage {
-  return {
-    async listExerciseFolderFileCounts(courseSlug) {
-      const prefix = `${courseSlug}/exercises/`
-      const keys = await listObjectKeys(s3, bucket, prefix)
-      const counts = new Map<string, number>()
-      for (const key of keys) {
-        const segment = key.slice(prefix.length).split("/")[0]
-        if (segment) {
-          counts.set(segment, (counts.get(segment) || 0) + 1)
-        }
-      }
-
-      return counts
-    },
-    async translationLangsBySlug(courseSlug) {
-      const exercises = await fetchExercises(s3, bucket, courseSlug)
-      const map = new Map<string, string[]>()
-      for (const exercise of exercises) {
-        const langs = Object.keys(exercise.translations || {})
-          .map(lang => lang.toLowerCase())
-          .filter(Boolean)
-        if (langs.length > 0) {
-          map.set(exercise.slug, [...new Set(langs)])
-        }
-      }
-
-      return map
-    },
-  }
-}
-
-export type S3DescriptionsStorageOptions = {
-  /** CloudFront distribution to invalidate after re-projecting the manifest. */
-  cloudFront?: { client: AwsClient; distributionId: string };
-  /** Set false in the backfill, where manifests are projected in a later pass. */
-  reprojectManifest?: boolean;
-};
-
-/**
- * Storage for the published bucket, wired from the environment. Used by the
- * post-publish stage and the sweep, which have no CLI flags to read.
- */
-export function createS3DescriptionsStorageFromEnv(): CourseDescriptionsStorage {
-  const s3 = new S3Client({
-    region: process.env.AWS_REGION || "us-east-1",
-  }) as unknown as AwsClient
-
-  const distributionId = process.env.CLOUDFRONT_DISTRIBUTION_ID
-  const cloudFront = distributionId ?
-    {
-        client: new CloudFrontClient({
-          region: process.env.AWS_REGION || "us-east-1",
-        }) as unknown as AwsClient,
-        distributionId,
-      } :
-    undefined
-
-  if (!cloudFront) {
-    console.warn(
-      "[descriptions] CLOUDFRONT_DISTRIBUTION_ID is not set: the manifest will be re-projected but not invalidated"
-    )
-  }
-
-  return createS3DescriptionsStorage(
-    s3,
-    process.env.S3_PACKAGES_BUCKET || "learnpack-paquetes",
-    { cloudFront }
-  )
-}
-
-export function createS3DescriptionsStorage(
-  s3: AwsClient,
-  bucket: string,
-  options: S3DescriptionsStorageOptions = {}
-): CourseDescriptionsStorage {
-  const storage: CourseDescriptionsStorage = {
-    async readSyllabus(courseSlug) {
-      return fetchJsonObject<Syllabus>(s3, bucket, syllabusKey(courseSlug))
-    },
-    async writeSyllabus(courseSlug, syllabus) {
-      await withRetry(() =>
-        s3.send(
-          new PutObjectCommand({
-            Bucket: bucket,
-            Key: syllabusKey(courseSlug),
-            Body: JSON.stringify(syllabus, null, 2),
-            ContentType: "application/json",
-          })
-        )
-      )
-    },
-    async listExercises(courseSlug) {
-      return fetchExercises(s3, bucket, courseSlug)
-    },
-    async readReadme(courseSlug, exerciseSlug, fileName) {
-      return fetchTextObject(
-        s3,
-        bucket,
-        `${courseSlug}/exercises/${exerciseSlug}/${fileName}`
-      )
-    },
-    async readManifest(courseSlug) {
-      return fetchJsonObject<PackageManifest>(
-        s3,
-        bucket,
-        manifestKey(courseSlug)
-      )
-    },
-    syllabusSyncStorage: createS3SyllabusSyncStorage(s3, bucket),
-  }
-
-  if (options.reprojectManifest !== false) {
-    storage.reprojectManifest = async courseSlug => {
-      // Preserve the publication timestamp: this is a re-projection of an
-      // already published package, not a new publication.
-      const existing = await fetchJsonObject<PackageManifest>(
-        s3,
-        bucket,
-        manifestKey(courseSlug)
-      )
-
-      const result = await processPackage(s3, bucket, courseSlug, {
-        dryRun: false,
-        force: false,
-        skipExisting: false,
-        publishedAt: existing?.publishedAt ?? null,
-      })
-
-      if (result.status === "failed") {
-        throw new Error(result.error || "manifest projection failed")
-      }
-
-      if (result.status === "skipped" || !options.cloudFront) {
-        return
-      }
-
-      await invalidatePackageManifestPaths(
-        options.cloudFront.client,
-        options.cloudFront.distributionId,
-        [`/${manifestKey(courseSlug)}`]
-      )
-    }
-  }
-
-  return storage
-}
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"
+import { CloudFrontClient } from "@aws-sdk/client-cloudfront"
+import { Syllabus } from "../../models/creator"
+import { PACKAGE_MANIFEST_REL_PATH, PackageManifest } from "../packageManifest"
+import {
+  AwsClient,
+  fetchJsonObject,
+  fetchTextObject,
+  invalidatePackageManifestPaths,
+  listObjectKeys,
+  processPackage,
+  withRetry,
+} from "../s3/packageManifestBackfill"
+import { SyllabusSyncStorage } from "../syllabusSync"
+import {
+  awsRegion,
+  requireAwsCredentials,
+  requireS3PackagesBucket,
+} from "../awsCredentials"
+import { CourseDescriptionsStorage } from "./generateCourseDescriptions"
+import { CourseExercise } from "./workList"
+
+/**
+ * Published-bucket (S3) adapter for the description flows.
+ *
+ * The published package is the snapshot descriptions are generated from: it is
+ * immutable between publications, unlike the draft in GCS, which keeps moving
+ * while the teacher edits.
+ */
+
+export const syllabusKey = (slug: string) =>
+  `${slug}/.learn/initialSyllabus.json`
+
+export const manifestKey = (slug: string) =>
+  `${slug}/${PACKAGE_MANIFEST_REL_PATH}`
+
+const configKeys = (slug: string) => [
+  `${slug}/.learn/config.json`,
+  `${slug}/config.json`,
+]
+
+type ConfigShape = {
+  exercises?: CourseExercise[];
+};
+
+export async function fetchExercises(
+  s3: AwsClient,
+  bucket: string,
+  slug: string
+): Promise<CourseExercise[]> {
+  for (const key of configKeys(slug)) {
+    // eslint-disable-next-line no-await-in-loop -- the second key is a fallback
+    const config = await fetchJsonObject<ConfigShape>(s3, bucket, key)
+    if (config?.exercises && Array.isArray(config.exercises)) {
+      return config.exercises
+    }
+  }
+
+  return []
+}
+
+export function createS3SyllabusSyncStorage(
+  s3: AwsClient,
+  bucket: string
+): SyllabusSyncStorage {
+  return {
+    async listExerciseFolderFileCounts(courseSlug) {
+      const prefix = `${courseSlug}/exercises/`
+      const keys = await listObjectKeys(s3, bucket, prefix)
+      const counts = new Map<string, number>()
+      for (const key of keys) {
+        const segment = key.slice(prefix.length).split("/")[0]
+        if (segment) {
+          counts.set(segment, (counts.get(segment) || 0) + 1)
+        }
+      }
+
+      return counts
+    },
+    async translationLangsBySlug(courseSlug) {
+      const exercises = await fetchExercises(s3, bucket, courseSlug)
+      const map = new Map<string, string[]>()
+      for (const exercise of exercises) {
+        const langs = Object.keys(exercise.translations || {})
+          .map(lang => lang.toLowerCase())
+          .filter(Boolean)
+        if (langs.length > 0) {
+          map.set(exercise.slug, [...new Set(langs)])
+        }
+      }
+
+      return map
+    },
+  }
+}
+
+export type S3DescriptionsStorageOptions = {
+  /** CloudFront distribution to invalidate after re-projecting the manifest. */
+  cloudFront?: { client: AwsClient; distributionId: string };
+  /** Set false in the backfill, where manifests are projected in a later pass. */
+  reprojectManifest?: boolean;
+};
+
+/**
+ * Storage for the published bucket, wired from the environment. Used by the
+ * post-publish stage and the sweep, which have no CLI flags to read.
+ */
+export function createS3DescriptionsStorageFromEnv(): CourseDescriptionsStorage {
+  // Validated here so a missing variable fails with its own name, instead of
+  // surfacing minutes later as "Could not load credentials from any providers"
+  // from inside a background job.
+  const credentials = requireAwsCredentials()
+  const region = awsRegion()
+
+  const s3 = new S3Client({ region, credentials }) as unknown as AwsClient
+
+  const distributionId = (process.env.CLOUDFRONT_DISTRIBUTION_ID || "").trim()
+  const cloudFront = distributionId ?
+    {
+        client: new CloudFrontClient({
+          region,
+          credentials,
+        }) as unknown as AwsClient,
+        distributionId,
+      } :
+    undefined
+
+  if (!cloudFront) {
+    console.warn(
+      "[descriptions] CLOUDFRONT_DISTRIBUTION_ID is not set: the manifest will be re-projected but not invalidated"
+    )
+  }
+
+  return createS3DescriptionsStorage(s3, requireS3PackagesBucket(), {
+    cloudFront,
+  })
+}
+
+export function createS3DescriptionsStorage(
+  s3: AwsClient,
+  bucket: string,
+  options: S3DescriptionsStorageOptions = {}
+): CourseDescriptionsStorage {
+  const storage: CourseDescriptionsStorage = {
+    async readSyllabus(courseSlug) {
+      return fetchJsonObject<Syllabus>(s3, bucket, syllabusKey(courseSlug))
+    },
+    async writeSyllabus(courseSlug, syllabus) {
+      await withRetry(() =>
+        s3.send(
+          new PutObjectCommand({
+            Bucket: bucket,
+            Key: syllabusKey(courseSlug),
+            Body: JSON.stringify(syllabus, null, 2),
+            ContentType: "application/json",
+          })
+        )
+      )
+    },
+    async listExercises(courseSlug) {
+      return fetchExercises(s3, bucket, courseSlug)
+    },
+    async readReadme(courseSlug, exerciseSlug, fileName) {
+      return fetchTextObject(
+        s3,
+        bucket,
+        `${courseSlug}/exercises/${exerciseSlug}/${fileName}`
+      )
+    },
+    async readManifest(courseSlug) {
+      return fetchJsonObject<PackageManifest>(
+        s3,
+        bucket,
+        manifestKey(courseSlug)
+      )
+    },
+    syllabusSyncStorage: createS3SyllabusSyncStorage(s3, bucket),
+  }
+
+  if (options.reprojectManifest !== false) {
+    storage.reprojectManifest = async courseSlug => {
+      // Preserve the publication timestamp: this is a re-projection of an
+      // already published package, not a new publication.
+      const existing = await fetchJsonObject<PackageManifest>(
+        s3,
+        bucket,
+        manifestKey(courseSlug)
+      )
+
+      const result = await processPackage(s3, bucket, courseSlug, {
+        dryRun: false,
+        force: false,
+        skipExisting: false,
+        publishedAt: existing?.publishedAt ?? null,
+      })
+
+      if (result.status === "failed") {
+        throw new Error(result.error || "manifest projection failed")
+      }
+
+      if (result.status === "skipped" || !options.cloudFront) {
+        return
+      }
+
+      await invalidatePackageManifestPaths(
+        options.cloudFront.client,
+        options.cloudFront.distributionId,
+        [`/${manifestKey(courseSlug)}`]
+      )
+    }
+  }
+
+  return storage
+}

package/src/utils/packageManifest.ts

@@ -94,7 +94,14 @@ function findSyllabusLesson(
   })
 }
 
-function deriveLessonId(exerciseSlug: string): string {
+/**
+ * Lesson id when no syllabus entry matches: the numeric prefix of the folder.
+ *
+ * Exported because `legacyPackageRepair` has to synthesize syllabus ids with
+ * exactly this rule — a repaired package whose ids differed from the ones its
+ * manifest already carries would look like a content change to every consumer.
+ */
+export function deriveLessonId(exerciseSlug: string): string {
   const match = exerciseSlug.match(/^(\d+(?:\.\d+)?)/)
   return match ? match[1] : exerciseSlug
 }