@learnpack/learnpack 5.0.353 → 5.0.354

package/src/utils/repair/repairStorage.ts

@@ -0,0 +1,168 @@
+import { Bucket } from "@google-cloud/storage"
+import { PutObjectCommand } from "@aws-sdk/client-s3"
+import {
+  AwsClient,
+  fetchJsonObject,
+  fetchTextObject,
+  listObjectKeys,
+  withRetry,
+} from "../s3/packageManifestBackfill"
+
+/**
+ * The two buckets a package lives in, behind one interface.
+ *
+ * A legacy package is repaired **twice, independently** — once against the
+ * published snapshot in S3 and once against the draft in GCS — never by copying
+ * one into the other. The two hold different content (the draft keeps moving
+ * after a publication) and the synthesized structure is a projection of that
+ * content: a syllabus derived from the snapshot could name steps the draft does
+ * not have. This is the same rule `mirrorDescriptions` enforces with its content
+ * hash; deriving from each bucket needs no guard at all.
+ *
+ * What crosses between them is only the descriptions, through
+ * `descriptionsS3Backfill --mirror-draft`, which is hash-guarded per step.
+ */
+
+export type RepairTarget = "s3" | "gcs";
+
+export const ALL_REPAIR_TARGETS: RepairTarget[] = ["s3", "gcs"]
+
+/**
+ * Where each input lives. The two buckets do not agree on this, and the
+ * disagreement is load-bearing:
+ *
+ * - S3 reads `{slug}/config.json` first (`fetchPackageSources`), which is where
+ *   the publish writes it into the zip.
+ * - GCS reads `courses/{slug}/.learn/config.json` — the publish route downloads
+ *   that one before anything else, so a config written anywhere else in the
+ *   draft would not make the package publishable.
+ */
+export type PackageLayout = {
+  learnJsonKey(slug: string): string;
+  /** Config locations in resolution order; the first is the write target. */
+  configKeys(slug: string): string[];
+  syllabusKey(slug: string): string;
+  /** Sidebar locations in resolution order; the first is the write target. */
+  sidebarKeys(slug: string): string[];
+  exercisesPrefix(slug: string): string;
+  /**
+   * Whether `config.json` is the authority on the exercise list, or the folder
+   * listing is.
+   *
+   * This decides the **language keys** of the synthesized syllabus, and the two
+   * buckets genuinely disagree:
+   *
+   * - S3 reads the published `config.json` (`fetchPackageSources`), where these
+   *   legacy packages carry `us` for the unsuffixed README.
+   * - GCS has no such reader. Both the publish-time manifest and the
+   *   descriptions pipeline go through `configBuilder.buildConfig`, which
+   *   derives exercises from the listing and maps an unsuffixed `README.md` to
+   *   `en` — regardless of what the draft's `.learn/config.json` says.
+   *
+   * A syllabus keyed the other way is not read at all: the manifest would look
+   * up `translations["en"]`, find nothing, and publish `description: null` for
+   * every step. So each side is built the way its own reader will read it.
+   */
+  exercisesFromConfig: boolean;
+};
+
+export type RepairStorage = {
+  target: RepairTarget;
+  layout: PackageLayout;
+  readJson<T>(key: string): Promise<T | null>;
+  readText(key: string): Promise<string | null>;
+  listKeys(prefix: string): Promise<string[]>;
+  writeJson(key: string, value: unknown): Promise<void>;
+};
+
+export const s3Layout: PackageLayout = {
+  learnJsonKey: slug => `${slug}/learn.json`,
+  configKeys: slug => [`${slug}/config.json`, `${slug}/.learn/config.json`],
+  syllabusKey: slug => `${slug}/.learn/initialSyllabus.json`,
+  // 25 of the published packages keep their sidebar at the root. It has to be
+  // resolved before writing: `fetchPackageSources` prefers the `.learn/` copy,
+  // so a synthesized one would shadow the published titles.
+  sidebarKeys: slug => [
+    `${slug}/.learn/sidebar.json`,
+    `${slug}/sidebar.json`,
+  ],
+  exercisesPrefix: slug => `${slug}/exercises/`,
+  exercisesFromConfig: true,
+}
+
+export const gcsLayout: PackageLayout = {
+  learnJsonKey: slug => `courses/${slug}/learn.json`,
+  configKeys: slug => [
+    `courses/${slug}/.learn/config.json`,
+    `courses/${slug}/config.json`,
+  ],
+  syllabusKey: slug => `courses/${slug}/.learn/initialSyllabus.json`,
+  sidebarKeys: slug => [`courses/${slug}/.learn/sidebar.json`],
+  exercisesPrefix: slug => `courses/${slug}/exercises/`,
+  exercisesFromConfig: false,
+}
+
+export function createS3RepairStorage(
+  s3: AwsClient,
+  bucket: string
+): RepairStorage {
+  return {
+    target: "s3",
+    layout: s3Layout,
+    readJson: key => fetchJsonObject(s3, bucket, key),
+    readText: key => fetchTextObject(s3, bucket, key),
+    listKeys: prefix => listObjectKeys(s3, bucket, prefix),
+    async writeJson(key, value) {
+      await withRetry(() =>
+        s3.send(
+          new PutObjectCommand({
+            Bucket: bucket,
+            Key: key,
+            Body: JSON.stringify(value, null, 2),
+            ContentType: "application/json",
+          })
+        )
+      )
+    },
+  }
+}
+
+export function createGcsRepairStorage(bucket: Bucket): RepairStorage {
+  const download = async (key: string): Promise<string | null> => {
+    try {
+      const [buf] = await bucket.file(key).download()
+      return buf.toString()
+    } catch {
+      return null
+    }
+  }
+
+  return {
+    target: "gcs",
+    layout: gcsLayout,
+    async readJson<T>(key: string): Promise<T | null> {
+      const content = await download(key)
+      if (content === null) {
+        return null
+      }
+
+      try {
+        return JSON.parse(content) as T
+      } catch {
+        return null
+      }
+    },
+    readText: download,
+    async listKeys(prefix) {
+      const [files] = await bucket.getFiles({ prefix })
+      return files.map(file => file.name)
+    },
+    async writeJson(key, value) {
+      await bucket
+        .file(key)
+        .save(Buffer.from(JSON.stringify(value, null, 2), "utf8"), {
+          contentType: "application/json",
+        })
+    },
+  }
+}