@learnpack/learnpack 5.0.275 → 5.0.277

package/src/utils/export/README.md

@@ -0,0 +1,178 @@
+# Export Utilities
+
+This directory contains utilities for exporting courses to different formats.
+
+## Supported Formats
+
+- **SCORM**: Learning Management System compatible package
+- **EPUB**: E-book format for offline reading
+
+## Usage
+
+### API Endpoints
+
+The export functionality is available through the `/export/:course_slug/:format` endpoint:
+
+- `GET /export/{course_slug}/scorm` - Export course to SCORM format
+- `GET /export/{course_slug}/epub` - Export course to EPUB format
+
+### Example Usage
+
+```bash
+# Export course to SCORM
+curl -O "http://localhost:3000/export/my-course/scorm"
+
+# Export course to EPUB (default English)
+curl -O "http://localhost:3000/export/my-course/epub"
+
+# Export course to EPUB in Spanish
+curl -O "http://localhost:3000/export/my-course/epub?language=es"
+
+# Export course to EPUB in French
+curl -O "http://localhost:3000/export/my-course/epub?language=fr"
+```
+
+## Requirements
+
+### SCORM Export
+
+- No additional requirements
+
+### EPUB Export
+
+- **Pandoc** must be installed on the system
+- Pandoc is used to convert markdown to HTML and generate EPUB files
+- **Language Support**: Use the `language` query parameter to specify the language
+  - Default: `en` (English)
+  - The system will prioritize language-specific files (e.g., `README.es.md` for Spanish)
+  - Falls back to `README.md` if no language-specific file is found
+
+### Installing Pandoc
+
+#### Windows
+
+```bash
+# Using Chocolatey
+choco install pandoc
+
+# Using Scoop
+scoop install pandoc
+
+# Or download from https://pandoc.org/installing.html
+```
+
+#### macOS
+
+```bash
+# Using Homebrew
+brew install pandoc
+
+# Using MacPorts
+sudo port install pandoc
+```
+
+#### Linux
+
+```bash
+# Ubuntu/Debian
+sudo apt-get install pandoc
+
+# CentOS/RHEL
+sudo yum install pandoc
+
+# Arch Linux
+sudo pacman -S pandoc
+```
+
+## Architecture
+
+The export system is modular and consists of:
+
+- **`types.ts`**: Type definitions for export options and course metadata
+- **`shared.ts`**: Common utility functions used by all export formats
+- **`scorm.ts`**: SCORM export implementation
+- **`epub.ts`**: EPUB export implementation using Pandoc
+- **`index.ts`**: Main export interface
+
+## Adding New Export Formats
+
+To add a new export format:
+
+1. Create a new file in the export directory (e.g., `pdf.ts`)
+2. Implement the export function following the same pattern
+3. Add the new format to the `ExportFormat` type in `types.ts`
+4. Update the main export endpoint in `serve.ts`
+5. Add the new format to the export index
+
+### Example New Format
+
+```typescript
+// pdf.ts
+export async function exportToPdf(options: ExportOptions): Promise<string> {
+  // Implementation here
+  return outputPath;
+}
+
+// types.ts
+export type ExportFormat = 'scorm' | 'epub' | 'pdf';
+
+// serve.ts
+} else if (format === "pdf") {
+  outputPath = await exportToPdf({
+    courseSlug: course_slug,
+    format: "pdf",
+    bucket,
+    outDir: path.join(__dirname, "../output/directory")
+  });
+  filename = `${course_slug}.pdf`;
+}
+```
+
+## File Structure
+
+```
+src/utils/export/
+├── index.ts          # Main export interface
+├── types.ts          # Type definitions
+├── shared.ts         # Shared utilities
+├── scorm.ts          # SCORM export
+├── epub.ts           # EPUB export
+├── templates/        # Export templates
+│   ├── scorm/        # SCORM template files
+│   └── epub/         # EPUB template files
+└── README.md         # This file
+```
+
+## Templates
+
+### SCORM Template
+
+Contains the necessary files for a SCORM package:
+
+- `imsmanifest.xml` - Course manifest
+- `config/` - Configuration files
+- XSD schema files for validation
+
+### EPUB Template
+
+Contains styling and assets for EPUB generation:
+
+- `epub.css` - Basic EPUB styling
+- `cover.png` - Default cover image (placeholder)
+
+## Error Handling
+
+The export system includes comprehensive error handling:
+
+- Invalid format validation
+- File system operation errors
+- Pandoc execution errors (for EPUB)
+- Network/download errors
+- Cleanup on failure
+
+## Performance Considerations
+
+- Temporary files are automatically cleaned up
+- Large courses may take time to process
+- Consider implementing progress indicators for long-running exports
+- Memory usage scales with course size

package/src/utils/export/epub.ts

@@ -0,0 +1,400 @@
+import * as path from "path";
+import * as fs from "fs";
+import * as mkdirp from "mkdirp";
+import * as rimraf from "rimraf";
+import { v4 as uuidv4 } from "uuid";
+import { spawn } from "child_process";
+import { ExportOptions, CourseMetadata } from "./types";
+import { downloadS3Folder, getCourseMetadata } from "./shared";
+
+// Convert markdown to HTML using pandoc
+async function convertMarkdownToHtml(
+  markdownPath: string,
+  outputPath: string
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    let stdout = "";
+    let stderr = "";
+
+    const pandocArgs = [
+      markdownPath,
+      "-o",
+      outputPath,
+      "--from",
+      "markdown",
+      "--to",
+      "html",
+      "--standalone",
+      "--css",
+      path.join(__dirname, "../templates/epub/epub.css"),
+    ];
+
+    console.log("Executing pandoc command:", "pandoc", pandocArgs.join(" "));
+    console.log("Input file exists:", fs.existsSync(markdownPath));
+    console.log(
+      "Output directory exists:",
+      fs.existsSync(path.dirname(outputPath))
+    );
+
+    const pandoc = spawn("pandoc", pandocArgs);
+
+    pandoc.stdout.on("data", (data) => {
+      stdout += data.toString();
+    });
+
+    pandoc.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+
+    pandoc.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        const errorMessage = `Pandoc process exited with code ${code}\n\nSTDOUT:\n${stdout}\n\nSTDERR:\n${stderr}`;
+        console.error("Pandoc Error Details:", errorMessage);
+        reject(new Error(errorMessage));
+      }
+    });
+
+    pandoc.on("error", (err) => {
+      console.error("Pandoc spawn error:", err);
+      reject(err);
+    });
+  });
+}
+
+// Generate EPUB using pandoc
+async function generateEpub(
+  htmlFiles: string[],
+  outputPath: string,
+  metadata: CourseMetadata
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    let stdout = "";
+    let stderr = "";
+
+    // Include all HTML files in the EPUB
+    const pandocArgs = [
+      ...htmlFiles, // All HTML files
+      "-o",
+      outputPath,
+      "--from",
+      "html",
+      "--to",
+      "epub",
+      "--metadata",
+      `title=${metadata.title}`,
+      "--metadata",
+      `language=${metadata.language || "en"}`,
+      "--metadata",
+      `creator=FourGeeksAcademy`,
+      "--metadata",
+      `description=${metadata.description || ""}`,
+      "--metadata",
+      `subject=${metadata.technologies?.join(", ") || "Programming"}`,
+      "--css",
+      path.join(__dirname, "../templates/epub/epub.css"),
+    ];
+
+    console.log(
+      "Executing pandoc EPUB command:",
+      "pandoc",
+      pandocArgs.join(" ")
+    );
+    console.log("HTML files to include:", htmlFiles);
+    console.log(
+      "All input files exist:",
+      htmlFiles.every((file) => fs.existsSync(file))
+    );
+    console.log(
+      "Output directory exists:",
+      fs.existsSync(path.dirname(outputPath))
+    );
+
+    const pandoc = spawn("pandoc", pandocArgs);
+
+    pandoc.stdout.on("data", (data) => {
+      stdout += data.toString();
+    });
+
+    pandoc.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+
+    pandoc.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        const errorMessage = `Pandoc process exited with code ${code}\n\nSTDOUT:\n${stdout}\n\nSTDERR:\n${stderr}`;
+        console.error("Pandoc Error Details:", errorMessage);
+        reject(new Error(errorMessage));
+      }
+    });
+
+    pandoc.on("error", (err) => {
+      console.error("Pandoc spawn error:", err);
+      reject(err);
+    });
+  });
+}
+
+// Process exercises and convert to HTML
+async function processExercises(
+  exercisesDir: string,
+  outputDir: string,
+  language: string = "en"
+): Promise<string[]> {
+  const htmlFiles: string[] = [];
+
+  console.log("Processing exercises directory:", exercisesDir);
+  console.log("Output directory:", outputDir);
+  console.log("Language:", language);
+
+  if (!fs.existsSync(exercisesDir)) {
+    console.log("Exercises directory does not exist:", exercisesDir);
+    return htmlFiles;
+  }
+
+  console.log("Exercises directory exists, scanning for files...");
+
+  const processDirectory = async (dir: string, relativePath: string = "") => {
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+    // Sort entries to prioritize README files first
+    entries.sort((a, b) => {
+      const aName = a.name.toLowerCase();
+      const bName = b.name.toLowerCase();
+
+      // Prioritize README files
+      if (aName === "readme.md" && bName !== "readme.md") return -1;
+      if (bName === "readme.md" && aName !== "readme.md") return 1;
+
+      return aName.localeCompare(bName);
+    });
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      const relativeFilePath = path.join(relativePath, entry.name);
+
+      if (entry.isDirectory()) {
+        console.log("Processing subdirectory:", relativeFilePath);
+        // esling-disable-next-line
+        await processDirectory(fullPath, relativeFilePath);
+      } else if (
+        entry.name.endsWith(".md") ||
+        entry.name.endsWith(".markdown")
+      ) {
+        // Apply language filtering for all markdown files
+        const baseName = entry.name.replace(/\.(md|markdown)$/, "");
+
+        if (language === "en") {
+          // For English: exclude files with language suffixes
+          if (
+            baseName.includes(".") &&
+            /\.(es|fr|de|it|pt|ja|ko|zh|ar|ru)$/i.test(baseName)
+          ) {
+            console.log(
+              "Skipping language-specific file for English:",
+              relativeFilePath
+            );
+            continue;
+          }
+        } else {
+          // For other languages: only include files with the correct language suffix
+          if (!baseName.endsWith(`.${language}`)) {
+            console.log(
+              "Skipping file with different language:",
+              relativeFilePath
+            );
+            continue;
+          }
+        }
+
+        // Convert markdown to HTML
+        console.log("Converting markdown file:", relativeFilePath);
+        const htmlFileName = entry.name.replace(/\.(md|markdown)$/, ".html");
+        const htmlPath = path.join(
+          outputDir,
+          relativeFilePath.replace(/\.(md|markdown)$/, ".html")
+        );
+
+        console.log("HTML output path:", htmlPath);
+        mkdirp.sync(path.dirname(htmlPath));
+        // esling-disable-next-line
+        await convertMarkdownToHtml(fullPath, htmlPath);
+
+        htmlFiles.push(htmlPath);
+        console.log(
+          "Successfully converted:",
+          relativeFilePath,
+          "->",
+          htmlPath
+        );
+      } else if (entry.name.endsWith(".html")) {
+        // Copy HTML files as-is
+        console.log("Copying HTML file:", relativeFilePath);
+        const destPath = path.join(outputDir, relativeFilePath);
+        mkdirp.sync(path.dirname(destPath));
+        // esling-disable-next-line
+        fs.copyFileSync(fullPath, destPath);
+        htmlFiles.push(destPath);
+        console.log("Successfully copied:", relativeFilePath, "->", destPath);
+      } else {
+        console.log(
+          "Skipping file:",
+          relativeFilePath,
+          "(not markdown or HTML)"
+        );
+      }
+    }
+  };
+
+  await processDirectory(exercisesDir);
+  return htmlFiles;
+}
+
+// Create EPUB table of contents
+function createTocHtml(htmlFiles: string[], metadata: CourseMetadata): string {
+  let tocContent = `
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Table of Contents - ${metadata.title}</title>
+    <link rel="stylesheet" href="epub.css">
+</head>
+<body>
+    <h1>${metadata.title}</h1>
+    <nav id="toc">
+        <h2>Table of Contents</h2>
+        <ol>
+            <li><a href="main.html">Course Overview</a></li>
+`;
+
+  htmlFiles.forEach((file) => {
+    const fileName = path.basename(file, ".html");
+    const displayName = fileName
+      .replace(/[-_]/g, " ")
+      .replace(/\b\w/g, (l) => l.toUpperCase());
+    // Use the full relative path for the href to ensure proper navigation
+    const relativePath = file.replace(/.*\/html\//, "").replace(/\\/g, "/");
+    tocContent += `            <li><a href="${relativePath}">${displayName}</a></li>\n`;
+  });
+
+  tocContent += `
+        </ol>
+    </nav>
+</body>
+</html>`;
+
+  return tocContent;
+}
+
+export async function exportToEpub(options: ExportOptions): Promise<string> {
+  const { courseSlug, bucket, outDir, language = "en" } = options;
+
+  console.log("Starting EPUB export for course:", courseSlug);
+  console.log("Output directory:", outDir);
+  console.log("Language:", language);
+
+  // 1. Create temporary folder
+  const tmpName = uuidv4();
+  const epubOutDir = path.join(outDir, tmpName);
+  rimraf.sync(epubOutDir);
+  mkdirp.sync(epubOutDir);
+
+  console.log("Created temporary directory:", epubOutDir);
+
+  // 2. Create EPUB template directory
+  const epubTemplateDir = path.join(epubOutDir, "template");
+  mkdirp.sync(epubTemplateDir);
+
+  // 3. Download exercises and .learn from bucket
+  console.log("Downloading exercises...");
+  await downloadS3Folder(
+    bucket,
+    `courses/${courseSlug}/exercises/`,
+    path.join(epubOutDir, "exercises")
+  );
+  console.log("Downloading .learn files...");
+  await downloadS3Folder(
+    bucket,
+    `courses/${courseSlug}/.learn/`,
+    path.join(epubOutDir, ".learn")
+  );
+
+  // 4. Read learn.json for course info
+  console.log("Reading course metadata...");
+  const learnJson = await getCourseMetadata(bucket, courseSlug);
+
+  const metadata: CourseMetadata = {
+    title: learnJson.title?.en || learnJson.title || courseSlug,
+    description: learnJson.description?.en || learnJson.description,
+    language: learnJson.language || "en",
+    technologies: learnJson.technologies || [],
+    difficulty: learnJson.difficulty || "beginner",
+  };
+
+  console.log("Course metadata:", metadata);
+
+  // 5. Process exercises and convert to HTML
+  console.log("Processing exercises and converting to HTML...");
+  const htmlFiles = await processExercises(
+    path.join(epubOutDir, "exercises"),
+    path.join(epubOutDir, "html"),
+    language
+  );
+
+  console.log("Generated HTML files:", htmlFiles.length);
+  console.log("HTML files:", htmlFiles);
+
+  // 6. Create table of contents
+  const tocHtml = createTocHtml(htmlFiles, metadata);
+  const tocPath = path.join(epubOutDir, "html", "toc.html");
+  mkdirp.sync(path.dirname(tocPath));
+  fs.writeFileSync(tocPath, tocHtml, "utf-8");
+
+  // 7. Create main content file
+  const mainContent = `
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>${metadata.title}</title>
+    <link rel="stylesheet" href="epub.css">
+</head>
+<body>
+    <h1>${metadata.title}</h1>
+    <p>${metadata.description || ""}</p>
+    <p><strong>Technologies:</strong> ${
+      metadata.technologies?.join(", ") || "Programming"
+    }</p>
+    <p><strong>Difficulty:</strong> ${metadata.difficulty}</p>
+    <p><strong>Language:</strong> ${language}</p>
+    <p><a href="toc.html">Go to Table of Contents</a></p>
+</body>
+</html>`;
+
+  const mainContentPath = path.join(epubOutDir, "html", "main.html");
+  fs.writeFileSync(mainContentPath, mainContent, "utf-8");
+
+  // 8. Prepare all HTML files for EPUB generation
+  const allHtmlFiles = [mainContentPath, tocPath, ...htmlFiles];
+  console.log("All HTML files for EPUB:", allHtmlFiles);
+
+  // 9. Generate EPUB using pandoc
+  console.log("Generating EPUB file...");
+  const epubPath = path.join(epubOutDir, `${courseSlug}.epub`);
+  await generateEpub(allHtmlFiles, epubPath, metadata);
+
+  console.log("EPUB generated successfully:", epubPath);
+
+  // Clean up temporary files
+  console.log("Cleaning up temporary files...");
+  rimraf.sync(path.join(epubOutDir, "html"));
+  rimraf.sync(path.join(epubOutDir, "exercises"));
+  rimraf.sync(path.join(epubOutDir, ".learn"));
+
+  console.log("EPUB export completed successfully");
+  return epubPath;
+}

package/src/utils/export/index.ts

@@ -0,0 +1,3 @@
+export { exportToScorm } from "./scorm";
+export { exportToEpub } from "./epub";
+export { ExportFormat, ExportOptions } from "./types";

package/src/utils/export/scorm.ts

@@ -0,0 +1,121 @@
+import * as path from "path";
+import * as fs from "fs";
+import * as archiver from "archiver";
+import * as mkdirp from "mkdirp";
+import * as rimraf from "rimraf";
+import { v4 as uuidv4 } from "uuid";
+import { ExportOptions, CourseMetadata } from "./types";
+import {
+  copyDir,
+  downloadS3Folder,
+  getFilesList,
+  getCourseMetadata,
+} from "./shared";
+
+export async function exportToScorm(options: ExportOptions): Promise<string> {
+  const { courseSlug, bucket, outDir } = options;
+
+  // 1. Create temporary folder
+  const tmpName = uuidv4();
+  const scormOutDir = path.join(outDir, tmpName);
+  rimraf.sync(scormOutDir);
+  mkdirp.sync(scormOutDir);
+
+  // 2. Copy SCORM template
+  const scormTpl = path.join(__dirname, "../templates/scorm");
+  copyDir(scormTpl, scormOutDir);
+
+  // Paths
+  const appDir = path.resolve(__dirname, "../../ui/_app");
+  const configDir = path.join(scormOutDir, "config");
+
+  // Check if _app exists and has content
+  if (fs.existsSync(appDir) && fs.readdirSync(appDir).length > 0) {
+    // Copy app.css and app.js to config/
+    for (const file of ["app.css", "app.js"]) {
+      const src = path.join(appDir, file);
+      const dest = path.join(configDir, file);
+      if (fs.existsSync(src)) {
+        fs.copyFileSync(src, dest);
+      }
+    }
+
+    // Copy logo-192.png and logo-512.png to SCORM build root
+    for (const file of ["logo-192.png", "logo-512.png"]) {
+      const src = path.join(appDir, file);
+      const dest = path.join(scormOutDir, file);
+      if (fs.existsSync(src)) {
+        fs.copyFileSync(src, dest);
+      }
+    }
+  }
+
+  // 3. Download exercises and .learn from bucket
+  await downloadS3Folder(
+    bucket,
+    `courses/${courseSlug}/exercises/`,
+    path.join(scormOutDir, "exercises")
+  );
+  await downloadS3Folder(
+    bucket,
+    `courses/${courseSlug}/.learn/`,
+    path.join(scormOutDir, ".learn")
+  );
+
+  // 4. Read learn.json for course info
+  const learnJson = await getCourseMetadata(bucket, courseSlug);
+
+  // 5. Replace imsmanifest.xml
+  const manifestPath = path.join(scormOutDir, "imsmanifest.xml");
+  let manifest = fs.readFileSync(manifestPath, "utf-8");
+  manifest = manifest.replace(/{{organization_name}}/g, "FourGeeksAcademy");
+  manifest = manifest.replace(
+    /{{course_title}}/g,
+    learnJson.title?.en || learnJson.title || courseSlug
+  );
+
+  // Generate exercises list
+  const exercisesList = getFilesList(
+    path.join(scormOutDir, "exercises"),
+    "exercises"
+  ).join("\n      ");
+  manifest = manifest.replace(/{{exercises_list}}/g, exercisesList);
+
+  // Generate images list (optional, if you have assets)
+  let imagesList = "";
+  const assetsDir = path.join(scormOutDir, ".learn", "assets");
+  if (fs.existsSync(assetsDir)) {
+    imagesList = getFilesList(assetsDir, ".learn/assets").join("\n      ");
+  }
+  manifest = manifest.replace(/{{images_list}}/g, imagesList);
+
+  fs.writeFileSync(manifestPath, manifest, "utf-8");
+
+  // 6. Replace title in config/index.html
+  const configIndexPath = path.join(scormOutDir, "config", "index.html");
+  let indexHtml = fs.readFileSync(configIndexPath, "utf-8");
+  indexHtml = indexHtml.replace(
+    /{{title}}/g,
+    learnJson.title?.en || learnJson.title || courseSlug
+  );
+  fs.writeFileSync(configIndexPath, indexHtml, "utf-8");
+
+  // 7. Compress as ZIP
+  const zipPath = `${scormOutDir}.zip`;
+  await new Promise((resolve: any, reject: any) => {
+    const output = fs.createWriteStream(zipPath);
+    const archive = archiver("zip", { zlib: { level: 9 } });
+
+    output.on("close", resolve);
+    archive.on("error", reject);
+
+    archive.pipe(output);
+    archive.directory(scormOutDir, false);
+    archive.finalize();
+  });
+
+  // Clean up temporary directory
+  rimraf.sync(scormOutDir);
+
+  return zipPath;
+}