@learnpack/learnpack 5.0.351 → 5.0.353

package/src/utils/s3/packageManifestBackfill.ts

@@ -0,0 +1,776 @@
+import {
+  CreateInvalidationCommand,
+  CloudFrontClient,
+} from "@aws-sdk/client-cloudfront"
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  S3Client,
+} from "@aws-sdk/client-s3"
+import { Exercise } from "../configBuilder"
+import { Syllabus } from "../../models/creator"
+import {
+  buildPackageManifestFromSources,
+  comparePackageManifests,
+  PACKAGE_MANIFEST_REL_PATH,
+  parseReadmeContent,
+  PackageManifest,
+  ReadmeFrontmatter,
+  serializePackageManifest,
+} from "../packageManifest"
+
+export type PackageResultStatus = "created" | "updated" | "skipped" | "failed";
+
+export type PackageResult = {
+  slug: string;
+  status: PackageResultStatus;
+  durationMs: number;
+  error?: string;
+  step?: string;
+  sources?: {
+    learnJson: boolean;
+    configJson: boolean;
+    sidebar: boolean;
+    syllabus: boolean;
+    readmes: number;
+  };
+};
+
+export type BackfillOptions = {
+  dryRun: boolean;
+  force: boolean;
+  skipExisting: boolean;
+  noInvalidate: boolean;
+  concurrency: number;
+  publishedAt: string | null;
+  resumeFrom?: string;
+  reportOut: string;
+  slug?: string;
+  bucket: string;
+  region: string;
+  cloudFrontDistributionId?: string;
+};
+
+export type BatchReport = {
+  total: number;
+  created: number;
+  updated: number;
+  skipped: number;
+  failed: number;
+  startedAt: string;
+  finishedAt: string;
+  options: BackfillOptions;
+  results: PackageResult[];
+  invalidation?: {
+    attempted: boolean;
+    skippedReason?: string;
+    id?: string;
+    paths?: string[];
+    error?: string;
+  };
+};
+
+export type AwsClient = {
+  send(command: unknown): Promise<unknown>;
+};
+
+type ConfigJsonShape = {
+  config?: Record<string, unknown>;
+  exercises?: Exercise[];
+};
+
+type PackageSources = {
+  learnJson: Record<string, unknown>;
+  exercises: Exercise[];
+  sidebar: Record<string, Record<string, string>>;
+  syllabus: Syllabus | null;
+  readmeFrontmatters: Record<string, Record<string, ReadmeFrontmatter>>;
+  meta: PackageResult["sources"];
+};
+
+const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i
+const INDEX_HTML_PATTERN = /^[^/]+\/index\.html$/
+const TRANSIENT_ERROR_CODES = new Set([
+  "SlowDown",
+  "ServiceUnavailable",
+  "RequestTimeout",
+  "Throttling",
+  "ThrottlingException",
+  "TooManyRequestsException",
+])
+const CLOUDFRONT_INVALIDATION_CHUNK_SIZE = 3000
+
+export async function streamToString(body: unknown): Promise<string> {
+  if (!body) {
+    return ""
+  }
+
+  if (typeof body === "string") {
+    return body
+  }
+
+  if (Buffer.isBuffer(body)) {
+    return body.toString("utf-8")
+  }
+
+  if (body instanceof Uint8Array) {
+    return Buffer.from(body).toString("utf-8")
+  }
+
+  const chunks: Buffer[] = []
+  for await (const chunk of body as AsyncIterable<Buffer | string>) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
+  }
+
+  return Buffer.concat(chunks).toString("utf-8")
+}
+
+function isTransientError(error: unknown): boolean {
+  const err = error as {
+    name?: string;
+    Code?: string;
+    $metadata?: { httpStatusCode?: number };
+  }
+  const code = err.name || err.Code || ""
+  if (TRANSIENT_ERROR_CODES.has(code)) {
+    return true
+  }
+
+  const status = err.$metadata?.httpStatusCode
+  return status === 429 || status === 503
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => {
+    setTimeout(resolve, ms)
+  })
+}
+
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  attempts = 3,
+  attempt = 1
+): Promise<T> {
+  try {
+    return await fn()
+  } catch (error) {
+    if (!isTransientError(error) || attempt === attempts) {
+      throw error
+    }
+
+    const multiplier = 2 ** (attempt - 1)
+    const delayMs = 250 * multiplier
+    await sleep(delayMs)
+    return withRetry(fn, attempts, attempt + 1)
+  }
+}
+
+export async function mapWithConcurrency<T, R>(
+  items: T[],
+  limit: number,
+  fn: (item: T) => Promise<R>
+): Promise<R[]> {
+  const results: R[] = Array.from({ length: items.length })
+  let nextIndex = 0
+
+  async function worker(): Promise<void> {
+    while (nextIndex < items.length) {
+      const currentIndex = nextIndex
+      nextIndex += 1
+      // eslint-disable-next-line no-await-in-loop -- each worker processes items sequentially
+      results[currentIndex] = await fn(items[currentIndex])
+    }
+  }
+
+  const workers = Array.from({ length: Math.min(limit, items.length) }, () =>
+    worker()
+  )
+  await Promise.all(workers)
+  return results
+}
+
+function manifestKey(slug: string): string {
+  return `${slug}/${PACKAGE_MANIFEST_REL_PATH}`
+}
+
+function cloudFrontManifestPath(slug: string): string {
+  return `/${slug}/${PACKAGE_MANIFEST_REL_PATH}`
+}
+
+function isNotFoundError(error: unknown): boolean {
+  const err = error as { name?: string; Code?: string }
+  const code = err.name || err.Code || ""
+  return code === "NoSuchKey" || code === "NotFound" || code === "NoSuchBucket"
+}
+
+export async function listPublishedSlugs(
+  s3: AwsClient,
+  bucket: string
+): Promise<string[]> {
+  const slugs = new Set<string>()
+  let continuationToken: string | undefined
+
+  do {
+    // eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
+    const response = (await withRetry(() =>
+      s3.send(
+        new ListObjectsV2Command({
+          Bucket: bucket,
+          ContinuationToken: continuationToken,
+        })
+      )
+    )) as {
+      Contents?: Array<{ Key?: string }>;
+      NextContinuationToken?: string;
+    }
+
+    for (const item of response.Contents || []) {
+      const key = item.Key
+      if (key && INDEX_HTML_PATTERN.test(key)) {
+        slugs.add(key.split("/")[0])
+      }
+    }
+
+    continuationToken = response.NextContinuationToken
+  } while (continuationToken)
+
+  return [...slugs].sort()
+}
+
+export async function fetchJsonObject<T>(
+  s3: AwsClient,
+  bucket: string,
+  key: string
+): Promise<T | null> {
+  try {
+    const response = (await withRetry(() =>
+      s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }))
+    )) as { Body?: unknown }
+
+    const content = await streamToString(response.Body)
+    return JSON.parse(content) as T
+  } catch (error) {
+    if (isNotFoundError(error)) {
+      return null
+    }
+
+    throw error
+  }
+}
+
+export async function fetchTextObject(
+  s3: AwsClient,
+  bucket: string,
+  key: string
+): Promise<string | null> {
+  try {
+    const response = (await withRetry(() =>
+      s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }))
+    )) as { Body?: unknown }
+
+    return await streamToString(response.Body)
+  } catch (error) {
+    if (isNotFoundError(error)) {
+      return null
+    }
+
+    throw error
+  }
+}
+
+export async function listObjectKeys(
+  s3: AwsClient,
+  bucket: string,
+  prefix: string
+): Promise<string[]> {
+  const keys: string[] = []
+  let continuationToken: string | undefined
+
+  do {
+    // eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
+    const response = (await withRetry(() =>
+      s3.send(
+        new ListObjectsV2Command({
+          Bucket: bucket,
+          Prefix: prefix,
+          ContinuationToken: continuationToken,
+        })
+      )
+    )) as {
+      Contents?: Array<{ Key?: string }>;
+      NextContinuationToken?: string;
+    }
+
+    for (const item of response.Contents || []) {
+      if (item.Key) {
+        keys.push(item.Key)
+      }
+    }
+
+    continuationToken = response.NextContinuationToken
+  } while (continuationToken)
+
+  return keys
+}
+
+function resolveReadmeLang(
+  filename: string,
+  exercise: Exercise
+): string | null {
+  const lower = filename.toLowerCase()
+  for (const [lang, readmeFile] of Object.entries(exercise.translations)) {
+    if (readmeFile.toLowerCase() === lower) {
+      return lang
+    }
+  }
+
+  return null
+}
+
+function normalizeLearnAndExercises(
+  slug: string,
+  learnJson: Record<string, unknown> | null,
+  configJson: ConfigJsonShape | null
+): { learnJson: Record<string, unknown>; exercises: Exercise[] } {
+  const exercises = configJson?.exercises
+  if (!Array.isArray(exercises) || exercises.length === 0) {
+    throw new Error(`Missing or empty exercises array in ${slug}/config.json`)
+  }
+
+  const resolvedLearn =
+    learnJson ||
+    (configJson?.config as Record<string, unknown> | undefined) ||
+    null
+
+  if (!resolvedLearn) {
+    throw new Error(
+      `Missing learn metadata: no ${slug}/learn.json and no config.config`
+    )
+  }
+
+  return { learnJson: resolvedLearn, exercises }
+}
+
+export async function fetchPackageSources(
+  s3: AwsClient,
+  bucket: string,
+  slug: string
+): Promise<PackageSources> {
+  const [
+    learnJsonRaw,
+    configJsonRoot,
+    configJsonLearn,
+    sidebarLearn,
+    sidebarRoot,
+    syllabus,
+  ] = await Promise.all([
+    fetchJsonObject<Record<string, unknown>>(s3, bucket, `${slug}/learn.json`),
+    fetchJsonObject<ConfigJsonShape>(s3, bucket, `${slug}/config.json`),
+    fetchJsonObject<ConfigJsonShape>(s3, bucket, `${slug}/.learn/config.json`),
+    fetchJsonObject<Record<string, Record<string, string>>>(
+      s3,
+      bucket,
+      `${slug}/.learn/sidebar.json`
+    ),
+    fetchJsonObject<Record<string, Record<string, string>>>(
+      s3,
+      bucket,
+      `${slug}/sidebar.json`
+    ),
+    fetchJsonObject<Syllabus>(
+      s3,
+      bucket,
+      `${slug}/.learn/initialSyllabus.json`
+    ),
+  ])
+
+  const configJson = configJsonRoot || configJsonLearn
+  if (!configJson) {
+    throw new Error(
+      `Missing required file: ${slug}/config.json or ${slug}/.learn/config.json`
+    )
+  }
+
+  const { learnJson, exercises } = normalizeLearnAndExercises(
+    slug,
+    learnJsonRaw,
+    configJson
+  )
+
+  const exerciseSlugs = new Set(exercises.map(exercise => exercise.slug))
+  const readmeKeys = (await listObjectKeys(s3, bucket, `${slug}/exercises/`))
+    .filter(key => {
+      const filename = key.split("/").pop() || ""
+      return README_FILENAME_PATTERN.test(filename)
+    })
+    .filter(key => {
+      const parts = key.split("/")
+      const exerciseSlug = parts[2]
+      return Boolean(exerciseSlug && exerciseSlugs.has(exerciseSlug))
+    })
+
+  const readmeFrontmatters: Record<
+    string,
+    Record<string, ReadmeFrontmatter>
+  > = {}
+
+  await Promise.all(
+    readmeKeys.map(async key => {
+      const parts = key.split("/")
+      const exerciseSlug = parts[2]
+      const filename = parts[parts.length - 1]
+      const exercise = exercises.find(item => item.slug === exerciseSlug)
+      if (!exercise) {
+        return
+      }
+
+      const lang = resolveReadmeLang(filename, exercise)
+      if (!lang) {
+        return
+      }
+
+      const content = await fetchTextObject(s3, bucket, key)
+      if (!content) {
+        return
+      }
+
+      if (!readmeFrontmatters[exerciseSlug]) {
+        readmeFrontmatters[exerciseSlug] = {}
+      }
+
+      readmeFrontmatters[exerciseSlug][lang] = parseReadmeContent(content)
+    })
+  )
+
+  const readmeCount = Object.values(readmeFrontmatters).reduce(
+    (total, byLang) => total + Object.keys(byLang).length,
+    0
+  )
+
+  return {
+    learnJson,
+    exercises,
+    sidebar: sidebarLearn || sidebarRoot || {},
+    syllabus,
+    readmeFrontmatters,
+    meta: {
+      learnJson: Boolean(learnJsonRaw || configJson.config),
+      configJson: true,
+      sidebar: Boolean(sidebarLearn || sidebarRoot),
+      syllabus: Boolean(syllabus),
+      readmes: readmeCount,
+    },
+  }
+}
+
+export function buildManifestFromS3Sources(
+  slug: string,
+  sources: PackageSources,
+  options: { publishedAt: string | null; generatedAt?: string }
+): PackageManifest {
+  return buildPackageManifestFromSources({
+    learnJson: sources.learnJson,
+    syllabus: sources.syllabus,
+    sidebar: sources.sidebar,
+    exercises: sources.exercises,
+    readmeFrontmatters: sources.readmeFrontmatters,
+    options: {
+      slug,
+      generatedAt: options.generatedAt || new Date().toISOString(),
+      publishedAt: options.publishedAt,
+    },
+  })
+}
+
+export async function processPackage(
+  s3: AwsClient,
+  bucket: string,
+  slug: string,
+  options: Pick<
+    BackfillOptions,
+    "dryRun" | "force" | "skipExisting" | "publishedAt"
+  >
+): Promise<PackageResult> {
+  const started = Date.now()
+
+  try {
+    const existing = await fetchJsonObject<PackageManifest>(
+      s3,
+      bucket,
+      manifestKey(slug)
+    )
+
+    if (options.skipExisting && existing) {
+      return {
+        slug,
+        status: "skipped",
+        durationMs: Date.now() - started,
+        sources: {
+          learnJson: false,
+          configJson: false,
+          sidebar: false,
+          syllabus: false,
+          readmes: 0,
+        },
+      }
+    }
+
+    const sources = await fetchPackageSources(s3, bucket, slug)
+    const next = buildManifestFromS3Sources(slug, sources, {
+      publishedAt: options.publishedAt,
+    })
+
+    let status = comparePackageManifests(existing, next)
+
+    if (options.force && status === "skipped") {
+      status = "updated"
+    }
+
+    if (!options.dryRun && (status === "created" || status === "updated")) {
+      await withRetry(() =>
+        s3.send(
+          new PutObjectCommand({
+            Bucket: bucket,
+            Key: manifestKey(slug),
+            Body: serializePackageManifest(next),
+            ContentType: "application/json",
+          })
+        )
+      )
+    }
+
+    return {
+      slug,
+      status,
+      durationMs: Date.now() - started,
+      sources: sources.meta,
+    }
+  } catch (error) {
+    const err = error as Error & { step?: string }
+    return {
+      slug,
+      status: "failed",
+      durationMs: Date.now() - started,
+      error: err.message,
+      step: err.step || inferFailureStep(err.message),
+    }
+  }
+}
+
+function inferFailureStep(message: string): string {
+  if (message.includes("config.json") || message.includes("learn")) {
+    return "fetch_sources"
+  }
+
+  if (message.includes("PutObject")) {
+    return "put_object"
+  }
+
+  return "build"
+}
+
+export async function invalidatePackageManifestPaths(
+  cf: AwsClient,
+  distributionId: string,
+  paths: string[]
+): Promise<{ id: string }> {
+  if (paths.length === 0) {
+    throw new Error("No paths to invalidate")
+  }
+
+  const callerReference = `package-manifest-backfill-${Date.now()}`
+  let lastId = ""
+
+  for (
+    let index = 0;
+    index < paths.length;
+    index += CLOUDFRONT_INVALIDATION_CHUNK_SIZE
+  ) {
+    const chunk = paths.slice(
+      index,
+      index + CLOUDFRONT_INVALIDATION_CHUNK_SIZE
+    )
+    // eslint-disable-next-line no-await-in-loop -- sequential invalidation chunks to avoid rate limits
+    const response = (await withRetry(() =>
+      cf.send(
+        new CreateInvalidationCommand({
+          DistributionId: distributionId,
+          InvalidationBatch: {
+            CallerReference: `${callerReference}-${index}`,
+            Paths: {
+              Quantity: chunk.length,
+              Items: chunk,
+            },
+          },
+        })
+      )
+    )) as { Invalidation?: { Id?: string } }
+
+    lastId = response.Invalidation?.Id || lastId
+  }
+
+  return { id: lastId }
+}
+
+function loadResumeSlugs(reportPath: string): Set<string> {
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const fs = require("fs") as typeof import("fs")
+  const raw = fs.readFileSync(reportPath, "utf-8")
+  const report = JSON.parse(raw) as BatchReport
+  const skipStatuses = new Set<PackageResultStatus>([
+    "created",
+    "updated",
+    "skipped",
+  ])
+
+  return new Set(
+    report.results
+      .filter(result => skipStatuses.has(result.status))
+      .map(result => result.slug)
+  )
+}
+
+function summarizeResults(
+  results: PackageResult[]
+): Pick<BatchReport, "created" | "updated" | "skipped" | "failed"> {
+  return {
+    created: results.filter(result => result.status === "created").length,
+    updated: results.filter(result => result.status === "updated").length,
+    skipped: results.filter(result => result.status === "skipped").length,
+    failed: results.filter(result => result.status === "failed").length,
+  }
+}
+
+export async function runBatch(
+  options: BackfillOptions,
+  clients?: {
+    s3?: AwsClient;
+    cf?: AwsClient;
+    onProgress?: (index: number, total: number, result: PackageResult) => void;
+  }
+): Promise<BatchReport> {
+  const startedAt = new Date().toISOString()
+  const s3 =
+    clients?.s3 ||
+    new S3Client({
+      region: options.region,
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
+      },
+    })
+
+  const cf =
+    clients?.cf ||
+    new CloudFrontClient({
+      region: "us-east-1",
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
+      },
+    })
+
+  let slugs = options.slug ?
+    [options.slug] :
+    await listPublishedSlugs(s3, options.bucket)
+
+  if (options.resumeFrom) {
+    const resumeSlugs = loadResumeSlugs(options.resumeFrom)
+    slugs = slugs.filter(slug => !resumeSlugs.has(slug))
+  }
+
+  let completed = 0
+  const results = await mapWithConcurrency(
+    slugs,
+    options.concurrency,
+    async slug => {
+      const result = await processPackage(s3, options.bucket, slug, options)
+      completed += 1
+      clients?.onProgress?.(completed, slugs.length, result)
+      return result
+    }
+  )
+
+  const summary = summarizeResults(results)
+  const invalidationPaths = results
+    .filter(
+      result => result.status === "created" || result.status === "updated"
+    )
+    .map(result => cloudFrontManifestPath(result.slug))
+
+  let invalidation: BatchReport["invalidation"]
+
+  if (options.dryRun) {
+    invalidation = {
+      attempted: false,
+      skippedReason: "dry-run",
+    }
+  } else if (options.noInvalidate) {
+    invalidation = {
+      attempted: false,
+      skippedReason: "no-invalidate",
+    }
+  } else if (invalidationPaths.length === 0) {
+    invalidation = {
+      attempted: false,
+      skippedReason: "no-paths",
+    }
+  } else if (!options.cloudFrontDistributionId) {
+    invalidation = {
+      attempted: true,
+      paths: invalidationPaths,
+      error: "Missing CLOUDFRONT_DISTRIBUTION_ID",
+    }
+  } else {
+    try {
+      const result = await invalidatePackageManifestPaths(
+        cf,
+        options.cloudFrontDistributionId,
+        invalidationPaths
+      )
+      invalidation = {
+        attempted: true,
+        id: result.id,
+        paths: invalidationPaths,
+      }
+    } catch (error) {
+      invalidation = {
+        attempted: true,
+        paths: invalidationPaths,
+        error: (error as Error).message,
+      }
+    }
+  }
+
+  const report: BatchReport = {
+    total: slugs.length,
+    ...summary,
+    startedAt,
+    finishedAt: new Date().toISOString(),
+    options,
+    results,
+    invalidation,
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const fs = require("fs") as typeof import("fs")
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const path = require("path") as typeof import("path")
+  fs.mkdirSync(path.dirname(options.reportOut), { recursive: true })
+  fs.writeFileSync(options.reportOut, JSON.stringify(report, null, 2), "utf-8")
+
+  return report
+}
+
+export function getBatchExitCode(report: BatchReport): number {
+  if (report.failed > 0) {
+    return 1
+  }
+
+  if (report.invalidation?.attempted && report.invalidation.error) {
+    return 1
+  }
+
+  return 0
+}