@learnpack/learnpack 5.0.276 → 5.0.278

package/src/utils/export/epub.ts

@@ -0,0 +1,612 @@
+import * as path from "path";
+import * as fs from "fs";
+import * as mkdirp from "mkdirp";
+import * as rimraf from "rimraf";
+import { v4 as uuidv4 } from "uuid";
+import { spawn } from "child_process";
+import { ExportOptions, CourseMetadata, EpubMetadata } from "./types";
+import { downloadS3Folder, getCourseMetadata } from "./shared";
+
+const replaceQuestionsFromMarkdown = (markdown: string) => {
+  // Search for all code blocks like this (No breaking lines sensitive)
+  // ```question some parameters
+  // some text
+  // ```
+  const questionRegex = /```question([\s\S]*?)```/g;
+  return markdown.replace(questionRegex, "");
+};
+
+const replaceMermaidFromMarkdown = (markdown: string) => {
+  // Search for all code blocks like this (No breaking lines sensitive)
+  // ```mermaid
+  // some graphs in mermaid syntax
+  // ```
+  const mermaidRegex = /```mermaid([\s\S]*?)```/g;
+  return markdown.replace(mermaidRegex, "");
+};
+
+// Download cover image from bucket
+async function downloadCoverImage(
+  bucket: any,
+  courseSlug: string,
+  outputDir: string
+): Promise<string | undefined> {
+  try {
+    const coverImagePath = path.join(outputDir, "cover.png");
+    const [coverBuf] = await bucket
+      .file(`courses/${courseSlug}/preview.png`)
+      .download();
+    fs.writeFileSync(coverImagePath, coverBuf);
+    console.log("Cover image downloaded:", coverImagePath);
+    return coverImagePath;
+  } catch (error) {
+    console.log("No cover image found, skipping...");
+    return undefined;
+  }
+}
+
+// Convert markdown to HTML using pandoc
+async function convertMarkdownToHtml(
+  markdownPath: string,
+  outputPath: string,
+  learnAssetsDir?: string
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    let stdout = "";
+    let stderr = "";
+
+    // Process markdown content to fix image paths
+    let markdownContent = fs.readFileSync(markdownPath, "utf-8");
+
+    if (learnAssetsDir && fs.existsSync(learnAssetsDir)) {
+      // Find all image references in markdown
+      const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
+      let match;
+      let imageCount = 0;
+
+      while ((match = imageRegex.exec(markdownContent)) !== null) {
+        const [fullMatch, altText, imagePath] = match;
+
+        // Check if it's a relative path starting with /learn/assets/
+        if (imagePath.startsWith("/.learn/assets/")) {
+          const imageName = path.basename(imagePath);
+          const absoluteImagePath = path.join(learnAssetsDir, imageName);
+
+          console.log(`📸 Looking for image: ${imageName}`);
+          console.log(`📁 Learn assets dir: ${learnAssetsDir}`);
+          console.log(`📁 Absolute path: ${absoluteImagePath}`);
+          console.log(`📁 Path exists: ${fs.existsSync(absoluteImagePath)}`);
+
+          if (fs.existsSync(absoluteImagePath)) {
+            // Use absolute path for the image
+            console.log(
+              `✅ Found image, replacing path: ${imagePath} -> ${absoluteImagePath}`
+            );
+            markdownContent = markdownContent.replace(
+              fullMatch,
+              `![${altText}](${absoluteImagePath})`
+            );
+            imageCount++;
+          } else {
+            console.log(`❌ Image not found: ${absoluteImagePath}`);
+            // List files in the assets directory for debugging
+            if (fs.existsSync(learnAssetsDir)) {
+              const files = fs.readdirSync(learnAssetsDir);
+              console.log(`📁 Files in assets directory: ${files.join(", ")}`);
+            } else {
+              console.log(
+                `❌ Assets directory does not exist: ${learnAssetsDir}`
+              );
+            }
+          }
+        }
+      }
+    }
+
+    markdownContent = replaceQuestionsFromMarkdown(markdownContent);
+    markdownContent = replaceMermaidFromMarkdown(markdownContent);
+
+    // Write processed markdown to a temporary file
+    const tempMarkdownPath = markdownPath + ".processed";
+    fs.writeFileSync(tempMarkdownPath, markdownContent, "utf-8");
+
+    const pandocArgs = [
+      tempMarkdownPath,
+      "-o",
+      outputPath,
+      "--from",
+      "markdown",
+      "--to",
+      "html",
+      "--standalone",
+      "--css",
+      path.join(__dirname, "../templates/epub/epub.css"),
+    ];
+
+    const pandoc = spawn("pandoc", pandocArgs);
+
+    pandoc.stdout.on("data", (data) => {
+      stdout += data.toString();
+    });
+
+    pandoc.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+
+    pandoc.on("close", (code) => {
+      // Clean up temporary file
+      // if (fs.existsSync(tempMarkdownPath)) {
+      //   fs.unlinkSync(tempMarkdownPath);
+      // }
+
+      if (code === 0) {
+        console.log("Successfully converted:", markdownPath, "->", outputPath);
+        resolve();
+      } else {
+        const errorMessage = `Pandoc process exited with code ${code}\n\nSTDOUT:\n${stdout}\n\nSTDERR:\n${stderr}`;
+        console.error("Pandoc Error Details:", errorMessage);
+        reject(new Error(errorMessage));
+      }
+    });
+
+    pandoc.on("error", (err) => {
+      console.error("Pandoc spawn error:", err);
+      reject(err);
+    });
+  });
+}
+
+// Generate EPUB using pandoc
+async function generateEpub(
+  htmlFiles: string[],
+  outputPath: string,
+  metadata: CourseMetadata,
+  epubMetadata: EpubMetadata,
+  coverImagePath?: string
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    let stdout = "";
+    let stderr = "";
+
+    const yamlPath = path.join(path.dirname(outputPath), "metadata.yaml");
+
+    const escapeYaml = (str: string) => {
+      if (!str) return "";
+      return str.replace(/"/g, '\\"').replace(/\n/g, "\\n");
+    };
+
+    const yamlContent = `---
+title: "${epubMetadata.title}"
+creator: "${epubMetadata.creator}"
+publisher: "${epubMetadata.publisher}"
+rights: "${epubMetadata.rights}"
+lang: "${epubMetadata.lang}"
+description: "${escapeYaml(metadata.description || "")}"
+subject: "${metadata.technologies?.join(", ") || "Programming"}"
+---`;
+
+    fs.writeFileSync(yamlPath, yamlContent, "utf-8");
+
+    // Include all HTML files in the EPUB
+    const pandocArgs = [
+      ...htmlFiles, // All HTML files
+      "-o",
+      outputPath,
+      "--from",
+      "html",
+      "--to",
+      "epub",
+      "--metadata-file",
+      yamlPath,
+      "--css",
+      path.join(__dirname, "../templates/epub/epub.css"),
+    ];
+
+    // Add cover image if available - use absolute path
+    if (coverImagePath) {
+      pandocArgs.push("--epub-cover-image", coverImagePath);
+    }
+
+    console.log("🔧 Executing pandoc EPUB command:");
+    console.log("Command:", "pandoc", pandocArgs.join(" "));
+
+    const pandoc = spawn("pandoc", pandocArgs);
+
+    pandoc.stdout.on("data", (data) => {
+      const output = data.toString();
+      stdout += output;
+      console.log("📤 Pandoc STDOUT:", output);
+    });
+
+    pandoc.stderr.on("data", (data) => {
+      const output = data.toString();
+      stderr += output;
+      console.log("⚠️ Pandoc STDERR:", output);
+    });
+
+    pandoc.on("close", (code) => {
+      console.log(`🏁 Pandoc process finished with exit code: ${code}`);
+      if (code === 0) {
+        console.log("✅ EPUB generation successful!");
+        resolve();
+      } else {
+        const errorMessage = `Pandoc process exited with code ${code}\n\nSTDOUT:\n${stdout}\n\nSTDERR:\n${stderr}`;
+        console.error("❌ Pandoc Error Details:", errorMessage);
+        reject(new Error(errorMessage));
+      }
+    });
+
+    pandoc.on("error", (err) => {
+      console.error("❌ Pandoc spawn error:", err);
+      reject(err);
+    });
+  });
+}
+
+// Process exercises and convert to HTML
+async function processExercises(
+  exercisesDir: string,
+  outputDir: string,
+  language: string = "en",
+  learnAssetsDir?: string
+): Promise<string[]> {
+  const htmlFiles: string[] = [];
+
+  console.log("Processing exercises directory:", exercisesDir);
+  console.log("Output directory:", outputDir);
+  console.log("Language:", language);
+  console.log("Learn assets directory:", learnAssetsDir);
+
+  if (!fs.existsSync(exercisesDir)) {
+    console.log("Exercises directory does not exist:", exercisesDir);
+    return htmlFiles;
+  }
+
+  console.log("Exercises directory exists, scanning for files...");
+
+  const processDirectory = async (dir: string, relativePath: string = "") => {
+    console.log(`🔍 Processing directory: ${dir} (relative: ${relativePath})`);
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    console.log(
+      `📁 Found ${entries.length} entries in ${dir}:`,
+      entries.map((e) => e.name)
+    );
+
+    // Sort entries to prioritize README files first
+    entries.sort((a, b) => {
+      const aName = a.name.toLowerCase();
+      const bName = b.name.toLowerCase();
+
+      // Prioritize README files
+      if (aName === "readme.md" && bName !== "readme.md") return -1;
+      if (bName === "readme.md" && aName !== "readme.md") return 1;
+
+      return aName.localeCompare(bName);
+    });
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      const relativeFilePath = path.join(relativePath, entry.name);
+
+      if (entry.isDirectory()) {
+        console.log("Processing subdirectory:", relativeFilePath);
+        // esling-disable-next-line
+        await processDirectory(fullPath, relativeFilePath);
+      } else if (
+        entry.name.endsWith(".md") ||
+        entry.name.endsWith(".markdown")
+      ) {
+        // Apply language filtering for all markdown files
+        const baseName = entry.name.replace(/\.(md|markdown)$/, "");
+
+        if (language === "en") {
+          // For English: only include README.md and files without language suffixes
+          if (entry.name.toLowerCase() === "readme.md") {
+            console.log(`✅ Including English README: ${relativeFilePath}`);
+          } else if (
+            baseName.includes(".") &&
+            /\.(es|fr|de|it|pt|ja|ko|zh|ar|ru)$/i.test(baseName)
+          ) {
+            console.log(
+              `❌ Skipping language-specific file for English: ${relativeFilePath}`
+            );
+            continue;
+          } else {
+            console.log(`✅ Including English file: ${relativeFilePath}`);
+          }
+        } else {
+          // For other languages: only include files with the correct language suffix
+          if (entry.name.toLowerCase() === "readme.md") {
+            console.log(
+              `❌ Skipping English README for language ${language}: ${relativeFilePath}`
+            );
+            continue;
+          } else if (!baseName.endsWith(`.${language}`)) {
+            console.log(
+              `❌ Skipping file with different language: ${relativeFilePath}`
+            );
+            continue;
+          } else {
+            console.log(`✅ Including ${language} file: ${relativeFilePath}`);
+          }
+        }
+
+        // Convert markdown to HTML
+        console.log("Converting markdown file:", relativeFilePath);
+        
+        // Use the directory name as the HTML filename instead of README.html
+        const exerciseDirName = path.basename(path.dirname(fullPath));
+        const htmlFileName = `${exerciseDirName}.html`;
+        const htmlPath = path.join(outputDir, htmlFileName);
+
+        console.log("HTML output path:", htmlPath);
+        mkdirp.sync(path.dirname(htmlPath));
+        // esling-disable-next-line
+        await convertMarkdownToHtml(fullPath, htmlPath, learnAssetsDir);
+
+        htmlFiles.push(htmlPath);
+        console.log(
+          "Successfully converted:",
+          relativeFilePath,
+          "->",
+          htmlPath
+        );
+      } else if (entry.name.endsWith(".html")) {
+        // Copy HTML files as-is
+        console.log("Copying HTML file:", relativeFilePath);
+        const destPath = path.join(outputDir, relativeFilePath);
+        mkdirp.sync(path.dirname(destPath));
+        // esling-disable-next-line
+        fs.copyFileSync(fullPath, destPath);
+        htmlFiles.push(destPath);
+        console.log("Successfully copied:", relativeFilePath, "->", destPath);
+      } else {
+        console.log(
+          "Skipping file:",
+          relativeFilePath,
+          "(not markdown or HTML)"
+        );
+      }
+    }
+  };
+
+  await processDirectory(exercisesDir);
+  return htmlFiles;
+}
+
+const titlelify = (title: string) => {
+  return title.replace(/[-_]/g, " ").replace(/\b\w/g, (l) => l.toUpperCase());
+};
+
+// Read sidebar.json to get exercise titles
+function readSidebarData(learnDir: string): any {
+  try {
+    const sidebarPath = path.join(learnDir, "sidebar.json");
+    if (fs.existsSync(sidebarPath)) {
+      const sidebarContent = fs.readFileSync(sidebarPath, "utf-8");
+      return JSON.parse(sidebarContent);
+    }
+  } catch (error) {
+    console.log("⚠️ Could not read sidebar.json:", error);
+  }
+  return null;
+}
+
+// Create EPUB table of contents
+function createTocHtml(
+  htmlFiles: string[],
+  metadata: CourseMetadata,
+  sidebarData: any,
+  language: string
+): string {
+  let tocContent = `
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Table of Contents - ${metadata.title}</title>
+    <link rel="stylesheet" href="epub.css">
+</head>
+<body>
+    <h1>${metadata.title}</h1>
+    <nav id="toc">
+        <h2>Table of Contents</h2>
+        <ol>
+            <li><a href="main.html">Course Overview</a></li>
+`;
+
+  htmlFiles.forEach((file) => {
+    const fileName = path.basename(file, ".html");
+    const relativePath = fileName + ".html";
+
+    // Get the parent directory name (exercise slug) to look up in sidebar
+    const parentDir = path.basename(path.dirname(file));
+
+    // Try to get the title from sidebar.json using the parent directory name
+    let displayName = fileName;
+    if (sidebarData && sidebarData[parentDir]) {
+      console.log(
+        "🔍 Found title in sidebar.json:",
+        sidebarData[parentDir],
+        "; language: ",
+        language
+      );
+      // Get the title for the current language, fallback to English, then to the key itself
+      displayName =
+        sidebarData[parentDir][language] ||
+        sidebarData[parentDir]["en"] ||
+        titlelify(parentDir);
+    } else {
+      console.log(
+        "🔍 No title found in sidebar.json for parent directory:",
+        parentDir
+      );
+      // Fallback to titlelify if not found in sidebar
+      displayName = titlelify(parentDir);
+    }
+
+    tocContent += `            <li><a href="${relativePath}">${displayName}</a></li>\n`;
+  });
+
+  tocContent += `
+        </ol>
+    </nav>
+</body>
+</html>`;
+
+  return tocContent;
+}
+
+export async function exportToEpub(
+  options: ExportOptions,
+  epubMetadata: EpubMetadata
+): Promise<string> {
+  const { courseSlug, bucket, outDir, language = "en" } = options;
+
+  console.log("🚀 Starting EPUB export for course:", courseSlug);
+  console.log("📁 Output directory:", outDir);
+  console.log("🌍 Language:", language);
+  console.log("📋 EPUB Metadata:", epubMetadata);
+
+  // 1. Create temporary folder
+  const tmpName = uuidv4();
+  const epubOutDir = path.join(outDir, tmpName);
+  rimraf.sync(epubOutDir);
+  mkdirp.sync(epubOutDir);
+
+  console.log("Created temporary directory:", epubOutDir);
+
+  // 2. Create EPUB template directory
+  const epubTemplateDir = path.join(epubOutDir, "template");
+  mkdirp.sync(epubTemplateDir);
+
+  // 3. Download exercises, .learn, and cover image from bucket
+  console.log("📥 Downloading exercises...");
+  await downloadS3Folder(
+    bucket,
+    `courses/${courseSlug}/exercises/`,
+    path.join(epubOutDir, "exercises")
+  );
+  console.log("✅ Exercises downloaded successfully");
+
+  console.log("📥 Downloading .learn files...");
+  await downloadS3Folder(
+    bucket,
+    `courses/${courseSlug}/.learn/`,
+    path.join(epubOutDir, "learn")
+  );
+  console.log("✅ .learn files downloaded successfully");
+
+  console.log("📥 Downloading cover image...");
+  const coverImagePath = await downloadCoverImage(
+    bucket,
+    courseSlug,
+    epubOutDir
+  );
+  console.log(
+    "✅ Cover image download completed:",
+    coverImagePath ? "Found" : "Not found"
+  );
+
+  // 4. Read learn.json for course info
+  console.log("Reading course metadata...");
+  const learnJson = await getCourseMetadata(bucket, courseSlug);
+
+  const metadata: CourseMetadata = {
+    title: learnJson.title?.[epubMetadata.lang] || courseSlug,
+    description:
+      learnJson.description?.[epubMetadata.lang] || learnJson.description,
+    language: epubMetadata.lang,
+    technologies: learnJson.technologies || [],
+    difficulty: learnJson.difficulty || "beginner",
+  };
+
+  console.log("Course metadata:", metadata);
+
+  // 5. Process exercises and convert to HTML
+  console.log("🔄 Processing exercises and converting to HTML...");
+
+  // First, let's see what files were downloaded
+  const exercisesPath = path.join(epubOutDir, "exercises");
+  console.log("📂 Exercises directory contents:");
+  if (fs.existsSync(exercisesPath)) {
+    const exerciseDirs = fs.readdirSync(exercisesPath, { withFileTypes: true });
+    console.log(`Found ${exerciseDirs.length} exercise directories:`);
+    exerciseDirs.forEach((dir) => {
+      if (dir.isDirectory()) {
+        const dirPath = path.join(exercisesPath, dir.name);
+        const files = fs.readdirSync(dirPath);
+        console.log(
+          `  📁 ${dir.name}: ${files.length} files - ${files.join(", ")}`
+        );
+      }
+    });
+  } else {
+    console.log("❌ Exercises directory does not exist!");
+  }
+
+  const htmlFiles = await processExercises(
+    path.join(epubOutDir, "exercises"),
+    path.join(epubOutDir, "html"),
+    language,
+    path.join(epubOutDir, "learn", "assets")
+  );
+
+  // 6. Read sidebar data and create table of contents
+  const sidebarData = readSidebarData(path.join(epubOutDir, "learn"));
+  console.log("📋 Sidebar data loaded:", sidebarData ? "Success" : "Not found");
+
+  const tocHtml = createTocHtml(htmlFiles, metadata, sidebarData, language);
+  const tocPath = path.join(epubOutDir, "html", "toc.html");
+  mkdirp.sync(path.dirname(tocPath));
+  fs.writeFileSync(tocPath, tocHtml, "utf-8");
+
+  // 7. Create main content file
+  const mainContent = `
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>${metadata.title}</title>
+    <link rel="stylesheet" href="epub.css">
+</head>
+<body>
+    <h1>${metadata.title}</h1>
+    <p>${metadata.description || ""}</p>
+    <p><strong>Technologies:</strong> ${
+      metadata.technologies?.join(", ") || "Programming"
+    }</p>
+    <p><strong>Difficulty:</strong> ${metadata.difficulty}</p>
+    <p><strong>Language:</strong> ${language}</p>
+    <p><a href="toc.html">Go to Table of Contents</a></p>
+</body>
+</html>`;
+
+  const mainContentPath = path.join(epubOutDir, "html", "main.html");
+  fs.writeFileSync(mainContentPath, mainContent, "utf-8");
+
+  // 8. Prepare all HTML files for EPUB generation
+  const allHtmlFiles = [mainContentPath, tocPath, ...htmlFiles];
+
+  // 9. Generate EPUB using pandoc
+  console.log("📖 Generating EPUB file...");
+  const epubPath = path.join(epubOutDir, `${courseSlug}.epub`);
+  await generateEpub(
+    allHtmlFiles,
+    epubPath,
+    metadata,
+    epubMetadata,
+    coverImagePath
+  );
+
+  console.log("EPUB generated successfully:", epubPath);
+
+  // Clean up temporary files
+  console.log("Cleaning up temporary files...");
+  rimraf.sync(path.join(epubOutDir, "html"));
+  rimraf.sync(path.join(epubOutDir, "exercises"));
+  rimraf.sync(path.join(epubOutDir, "learn"));
+
+  console.log("EPUB export completed successfully");
+  return epubPath;
+}

package/src/utils/export/index.ts

@@ -0,0 +1,3 @@
+export { exportToScorm } from "./scorm";
+export { exportToEpub } from "./epub";
+export { ExportFormat, ExportOptions, EpubMetadata } from "./types";