@learnpack/learnpack 5.0.350 → 5.0.352

package/src/utils/export/README.md

@@ -4,8 +4,9 @@ This directory contains utilities for exporting courses to different formats.
 
 ## Supported Formats
 
-- **SCORM**: Learning Management System compatible package
-- **EPUB**: E-book format for offline reading
+- **SCORM**: Learning Management System compatible package (includes `.learn/package-manifest.json`)
+- **ZIP**: Full course mirror from GCS (includes `.learn/package-manifest.json`)
+- **EPUB**: E-book format for offline reading (does not include `package-manifest.json`)
 
 ## Usage
 

package/src/utils/export/scorm.ts

@@ -12,6 +12,7 @@ import {
   getCourseMetadata,
   escapeXml,
 } from "./shared";
+import { generateAndPersistPackageManifest } from "../packageManifest";
 
 export async function exportToScorm(options: ExportOptions): Promise<string> {
   const { courseSlug, bucket, outDir } = options;
@@ -67,6 +68,11 @@ export async function exportToScorm(options: ExportOptions): Promise<string> {
     path.join(scormOutDir, ".learn")
   );
 
+  await generateAndPersistPackageManifest(bucket, courseSlug, {
+    publishedAt: null,
+    localDir: scormOutDir,
+  });
+
   // 4. Read learn.json for course info
   const learnJson = await getCourseMetadata(bucket, courseSlug);
 

package/src/utils/export/zip.ts

@@ -5,7 +5,8 @@ import * as mkdirp from "mkdirp";
 import * as rimraf from "rimraf";
 import { v4 as uuidv4 } from "uuid";
 import { ExportOptions } from "./types";
-import { downloadS3Folder, getCourseMetadata } from "./shared";
+import { downloadS3Folder } from "./shared";
+import { generateAndPersistPackageManifest } from "../packageManifest";
 
 export async function exportToZip(options: ExportOptions): Promise<string> {
   const { courseSlug, bucket, outDir } = options;
@@ -16,6 +17,10 @@ export async function exportToZip(options: ExportOptions): Promise<string> {
   rimraf.sync(zipOutDir);
   mkdirp.sync(zipOutDir);
 
+  await generateAndPersistPackageManifest(bucket, courseSlug, {
+    publishedAt: null,
+  });
+
   // 2. Download all course files from bucket to temporary directory
   await downloadS3Folder(bucket, `courses/${courseSlug}/`, zipOutDir);
 

package/src/utils/packageManifest.ts

@@ -0,0 +1,433 @@
+import { Bucket } from "@google-cloud/storage"
+import * as fs from "fs"
+import * as mkdirp from "mkdirp"
+import * as path from "path"
+import { Lesson, Syllabus } from "../models/creator"
+import { buildConfig, Exercise } from "./configBuilder"
+import { getReadmeExtension, slugify } from "./creatorUtilities"
+import { formatSidebarTitle } from "./titlefy"
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const frontMatter = require("front-matter")
+
+export const PACKAGE_MANIFEST_FILENAME = "package-manifest.json"
+export const PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json"
+export const SCHEMA_VERSION = 1
+
+export type LessonType = "READ" | "CODE" | "QUIZ";
+
+export type PackageManifestLesson = {
+  id: string;
+  slug: string;
+  position: number;
+  type: LessonType;
+  title: Record<string, string>;
+  description: Record<string, string | null>;
+  video: Record<string, string | null>;
+};
+
+export type PackageManifest = {
+  schemaVersion: number;
+  generatedAt: string;
+  publishedAt: string | null;
+  slug: string;
+  title: Record<string, string>;
+  description: Record<string, string>;
+  preview: string;
+  technologies?: string[];
+  difficulty?: string;
+  duration?: number;
+  lessons: PackageManifestLesson[];
+};
+
+export type ReadmeFrontmatter = {
+  tutorial?: string;
+  intro?: string;
+  title?: string;
+};
+
+export type PackageManifestBuildInput = {
+  learnJson: Record<string, unknown>;
+  syllabus: Syllabus | null;
+  sidebar: Record<string, Record<string, string>>;
+  exercises: Exercise[];
+  readmeFrontmatters: Record<string, Record<string, ReadmeFrontmatter>>;
+  options: {
+    slug: string;
+    generatedAt: string;
+    publishedAt: string | null;
+  };
+};
+
+const VALID_LESSON_TYPES = new Set<LessonType>(["READ", "CODE", "QUIZ"])
+
+export function packageManifestGcsPath(courseSlug: string): string {
+  return `courses/${courseSlug}/${PACKAGE_MANIFEST_REL_PATH}`
+}
+
+export function resolvePreviewUrl(slug: string, preview?: string): string {
+  if (preview && preview.startsWith("http")) {
+    return preview
+  }
+
+  return `https://${slug}.learn-pack.com/preview.png`
+}
+
+function findSyllabusLesson(
+  exerciseSlug: string,
+  lessons: Lesson[]
+): Lesson | undefined {
+  return lessons.find(lesson => {
+    const candidates = [
+      slugify(lesson.id + "-" + lesson.title),
+      lesson.uid,
+    ].filter(Boolean)
+    return candidates.includes(exerciseSlug)
+  })
+}
+
+function deriveLessonId(exerciseSlug: string): string {
+  const match = exerciseSlug.match(/^(\d+(?:\.\d+)?)/)
+  return match ? match[1] : exerciseSlug
+}
+
+function coerceLessonType(type: string | undefined): LessonType {
+  if (type && VALID_LESSON_TYPES.has(type as LessonType)) {
+    return type as LessonType
+  }
+
+  if (type) {
+    console.warn(
+      `[package-manifest] Invalid lesson type "${type}", defaulting to READ`
+    )
+  }
+
+  return "READ"
+}
+
+export function extractReadmeH1(body: string): string | null {
+  const match = body.match(/^#\s+(.+)$/m)
+  if (!match) {
+    return null
+  }
+
+  const title = match[1].replace(/\r/g, "").trim()
+  return title || null
+}
+
+export function normalizeReadmeTitle(raw: string): string | null {
+  const normalized = raw
+    .trim()
+    .replace(/^\d+(?:\.\d+)?\s*[.:-]?\s*/, "")
+    .trim()
+  return normalized || null
+}
+
+function firstNonEmpty(
+  ...candidates: Array<string | null | undefined>
+): string | null {
+  for (const candidate of candidates) {
+    const value = candidate?.trim()
+    if (value) {
+      return value
+    }
+  }
+
+  return null
+}
+
+function resolveLessonTitle(
+  exercise: Exercise,
+  sidebar: Record<string, Record<string, string>>,
+  syllabusLesson: Lesson | undefined,
+  courseLang: string,
+  readmeByLang: Record<string, ReadmeFrontmatter>
+): Record<string, string> {
+  const sidebarEntry = sidebar[exercise.slug] ?? {}
+  const languages = new Set([
+    ...Object.keys(exercise.translations),
+    ...Object.keys(sidebarEntry),
+  ])
+
+  const titles: Record<string, string> = {}
+
+  for (const lang of languages) {
+    const title = firstNonEmpty(
+      readmeByLang[lang]?.title,
+      sidebarEntry[lang] ? formatSidebarTitle(sidebarEntry[lang]) : null,
+      lang === courseLang ? syllabusLesson?.title : null
+    )
+
+    if (title) {
+      titles[lang] = title
+    }
+  }
+
+  return titles
+}
+
+function resolveLessonDescriptions(
+  exercise: Exercise
+): Record<string, string | null> {
+  const description: Record<string, string | null> = {}
+
+  for (const lang of Object.keys(exercise.translations)) {
+    // TODO: add description content
+    description[lang] = null
+  }
+
+  return description
+}
+
+function resolveLessonVideos(
+  exercise: Exercise,
+  readmeFrontmatters: Record<string, Record<string, ReadmeFrontmatter>>
+): Record<string, string | null> {
+  const video: Record<string, string | null> = {}
+  const frontmatters = readmeFrontmatters[exercise.slug] || {}
+
+  for (const lang of Object.keys(exercise.translations)) {
+    const attrs = frontmatters[lang] || {}
+    const url = attrs.tutorial || attrs.intro || null
+    video[lang] = url && url.trim().length > 0 ? url.trim() : null
+  }
+
+  return video
+}
+
+export function buildPackageManifestFromSources(
+  input: PackageManifestBuildInput
+): PackageManifest {
+  const {
+    learnJson,
+    syllabus,
+    sidebar,
+    exercises,
+    readmeFrontmatters,
+    options,
+  } = input
+
+  const syllabusLessons = syllabus?.lessons ?? []
+  const courseLang = syllabus?.courseInfo?.language || "en"
+
+  const sortedExercises = [...exercises].sort(
+    (a, b) => a.position - b.position
+  )
+
+  const lessons: PackageManifestLesson[] = sortedExercises.map(exercise => {
+    const syllabusLesson = findSyllabusLesson(exercise.slug, syllabusLessons)
+
+    const id = syllabusLesson?.id ?? deriveLessonId(exercise.slug)
+    const type = syllabusLesson ?
+      coerceLessonType(syllabusLesson.type) :
+      "READ"
+
+    if (!syllabusLesson) {
+      console.warn(
+        `[package-manifest] No syllabus match for exercise "${exercise.slug}", using derived id "${id}"`
+      )
+    }
+
+    return {
+      id,
+      slug: exercise.slug,
+      position: exercise.position,
+      type,
+      title: resolveLessonTitle(
+        exercise,
+        sidebar,
+        syllabusLesson,
+        courseLang,
+        readmeFrontmatters[exercise.slug] ?? {}
+      ),
+      description: resolveLessonDescriptions(exercise),
+      video: resolveLessonVideos(exercise, readmeFrontmatters),
+    }
+  })
+
+  const title = (learnJson.title as Record<string, string>) || {}
+  const description = (learnJson.description as Record<string, string>) || {}
+
+  return {
+    schemaVersion: SCHEMA_VERSION,
+    generatedAt: options.generatedAt,
+    publishedAt: options.publishedAt,
+    slug: options.slug,
+    title,
+    description,
+    preview: resolvePreviewUrl(
+      options.slug,
+      learnJson.preview as string | undefined
+    ),
+    technologies: learnJson.technologies as string[] | undefined,
+    difficulty: learnJson.difficulty as string | undefined,
+    duration: learnJson.duration as number | undefined,
+    lessons,
+  }
+}
+
+async function downloadJsonFile<T>(
+  bucket: Bucket,
+  filePath: string
+): Promise<T | null> {
+  try {
+    const [buf] = await bucket.file(filePath).download()
+    return JSON.parse(buf.toString()) as T
+  } catch {
+    return null
+  }
+}
+
+export function parseReadmeContent(content: string): ReadmeFrontmatter {
+  const parsed = frontMatter(content)
+  const attributes = parsed.attributes || {}
+  const h1 = extractReadmeH1(parsed.body || "")
+  const title = h1 ? normalizeReadmeTitle(h1) : null
+  return {
+    ...(attributes.tutorial ? { tutorial: attributes.tutorial } : {}),
+    ...(attributes.intro ? { intro: attributes.intro } : {}),
+    ...(title ? { title } : {}),
+  }
+}
+
+export function comparePackageManifests(
+  existing: PackageManifest | null,
+  next: PackageManifest
+): "created" | "updated" | "skipped" {
+  if (existing === null) {
+    return "created"
+  }
+
+  const { generatedAt: _existingGeneratedAt, ...existingRest } = existing
+  const { generatedAt: _nextGeneratedAt, ...nextRest } = next
+
+  if (JSON.stringify(existingRest) === JSON.stringify(nextRest)) {
+    return "skipped"
+  }
+
+  return "updated"
+}
+
+export function serializePackageManifest(manifest: PackageManifest): string {
+  return JSON.stringify(manifest, null, 2)
+}
+
+async function readReadmeFrontmatter(
+  bucket: Bucket,
+  exerciseSlug: string,
+  lang: string,
+  readmePath: string
+): Promise<{
+  exerciseSlug: string;
+  lang: string;
+  frontmatter: ReadmeFrontmatter;
+}> {
+  try {
+    const [buf] = await bucket.file(readmePath).download()
+    return {
+      exerciseSlug,
+      lang,
+      frontmatter: parseReadmeContent(buf.toString()),
+    }
+  } catch (error) {
+    console.warn(
+      `[package-manifest] Could not read README for ${exerciseSlug} (${lang}):`,
+      (error as Error).message
+    )
+    return { exerciseSlug, lang, frontmatter: {} }
+  }
+}
+
+async function collectReadmeFrontmatters(
+  bucket: Bucket,
+  courseSlug: string,
+  exercises: Exercise[]
+): Promise<Record<string, Record<string, ReadmeFrontmatter>>> {
+  const readmeTasks = exercises.flatMap(exercise =>
+    Object.keys(exercise.translations).map(lang => {
+      const readmeFilename = `README${getReadmeExtension(lang)}`
+      const readmePath = `courses/${courseSlug}/exercises/${exercise.slug}/${readmeFilename}`
+      return readReadmeFrontmatter(bucket, exercise.slug, lang, readmePath)
+    })
+  )
+
+  const readmeEntries = await Promise.all(readmeTasks)
+  const result: Record<string, Record<string, ReadmeFrontmatter>> = {}
+
+  for (const { exerciseSlug, lang, frontmatter } of readmeEntries) {
+    if (!result[exerciseSlug]) {
+      result[exerciseSlug] = {}
+    }
+
+    result[exerciseSlug][lang] = frontmatter
+  }
+
+  return result
+}
+
+function writeManifestJson(
+  manifest: PackageManifest,
+  localDir?: string
+): string {
+  const json = serializePackageManifest(manifest)
+
+  if (localDir) {
+    const learnDir = path.join(localDir, ".learn")
+    mkdirp.sync(learnDir)
+    const localPath = path.join(learnDir, PACKAGE_MANIFEST_FILENAME)
+    fs.writeFileSync(localPath, json, "utf-8")
+  }
+
+  return json
+}
+
+export async function generateAndPersistPackageManifest(
+  bucket: Bucket,
+  courseSlug: string,
+  options: { publishedAt: string | null; localDir?: string }
+): Promise<PackageManifest> {
+  const generatedAt = new Date().toISOString()
+
+  const { config: learnJson, exercises } = await buildConfig(
+    bucket,
+    courseSlug
+  )
+
+  const syllabus = await downloadJsonFile<Syllabus>(
+    bucket,
+    `courses/${courseSlug}/.learn/initialSyllabus.json`
+  )
+
+  const sidebar =
+    (await downloadJsonFile<Record<string, Record<string, string>>>(
+      bucket,
+      `courses/${courseSlug}/.learn/sidebar.json`
+    )) || {}
+
+  const readmeFrontmatters = await collectReadmeFrontmatters(
+    bucket,
+    courseSlug,
+    exercises
+  )
+
+  const manifest = buildPackageManifestFromSources({
+    learnJson,
+    syllabus,
+    sidebar,
+    exercises,
+    readmeFrontmatters,
+    options: {
+      slug: courseSlug,
+      generatedAt,
+      publishedAt: options.publishedAt,
+    },
+  })
+
+  const json = writeManifestJson(manifest, options.localDir)
+
+  await bucket
+    .file(packageManifestGcsPath(courseSlug))
+    .save(Buffer.from(json, "utf8"))
+
+  return manifest
+}