offcourse 0.0.1 → 1.0.0

package/src/scraper/highlevel/schemas.ts

@@ -0,0 +1,183 @@
+/**
+ * Zod schemas for HighLevel API responses.
+ * These provide runtime validation and type inference.
+ */
+
+import { z } from "zod";
+
+// Re-export Firebase auth types (Firebase is used by HighLevel for auth)
+export {
+  FirebaseAuthTokenSchema,
+  type FirebaseAuthToken,
+  type FirebaseAuthRaw,
+} from "../../shared/firebase.js";
+
+// ============================================================================
+// Portal Settings API
+// ============================================================================
+
+export const PortalSettingsResponseSchema = z.object({
+  locationId: z.string(),
+  portalName: z.string().optional(),
+  name: z.string().optional(),
+});
+
+export type PortalSettingsResponse = z.infer<typeof PortalSettingsResponseSchema>;
+
+// ============================================================================
+// Video License API
+// ============================================================================
+
+export const VideoLicenseResponseSchema = z.object({
+  url: z.string(),
+  token: z.string(),
+});
+
+export type VideoLicenseResponse = z.infer<typeof VideoLicenseResponseSchema>;
+
+// ============================================================================
+// Post Details API
+// ============================================================================
+
+const VideoAssetSchema = z.object({
+  id: z.string().optional(),
+  assetId: z.string().optional(),
+  assetsLicenseId: z.string().optional(),
+  url: z.string().optional(),
+});
+
+const PosterImageSchema = z.object({
+  assetId: z.string().optional(),
+  url: z.string().optional(),
+});
+
+const ContentBlockSchema = z.object({
+  type: z.string(),
+  id: z.string().optional(),
+  assetId: z.string().optional(),
+  assetsLicenseId: z.string().optional(),
+  url: z.string().optional(),
+});
+
+const MaterialSchema = z.object({
+  id: z.string().optional(),
+  name: z.string().optional(),
+  url: z.string().optional(),
+  type: z.string().optional(),
+});
+
+export const PostDetailsSchema = z.object({
+  title: z.string().optional(),
+  description: z.string().nullable().optional(),
+  video: VideoAssetSchema.nullable().optional(),
+  posterImage: PosterImageSchema.nullable().optional(),
+  contentBlock: z.array(ContentBlockSchema).optional(),
+  materials: z.array(MaterialSchema).optional(),
+  post_materials: z.array(MaterialSchema).optional(),
+});
+
+// Response can have data directly or wrapped in "post"
+export const PostDetailsResponseSchema = z.object({
+  post: PostDetailsSchema.optional(),
+  // Also allow all post fields directly on root
+  title: z.string().optional(),
+  description: z.string().nullable().optional(),
+  video: VideoAssetSchema.nullable().optional(),
+  posterImage: PosterImageSchema.nullable().optional(),
+  contentBlock: z.array(ContentBlockSchema).optional(),
+  materials: z.array(MaterialSchema).optional(),
+  post_materials: z.array(MaterialSchema).optional(),
+});
+
+export type PostDetailsResponse = z.infer<typeof PostDetailsResponseSchema>;
+
+// ============================================================================
+// Categories API
+// ============================================================================
+
+export const CategorySchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  description: z.string().nullable().optional(),
+  position: z.number().optional(),
+  postCount: z.number().optional(),
+  visibility: z.string().optional(),
+});
+
+export const CategoriesResponseSchema = z.object({
+  categories: z.array(CategorySchema),
+});
+
+export type CategoriesResponse = z.infer<typeof CategoriesResponseSchema>;
+export type Category = z.infer<typeof CategorySchema>;
+
+// ============================================================================
+// Posts (Lessons) API
+// ============================================================================
+
+export const PostSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  indexPosition: z.number().optional(),
+  visibility: z.string().optional(),
+});
+
+export const PostsResponseSchema = z.object({
+  category: z
+    .object({
+      posts: z.array(PostSchema),
+    })
+    .optional(),
+});
+
+export type PostsResponse = z.infer<typeof PostsResponseSchema>;
+export type Post = z.infer<typeof PostSchema>;
+
+// ============================================================================
+// Product (Course) API
+// ============================================================================
+
+export const ProductSchema = z.object({
+  id: z.string().optional(),
+  title: z.string(),
+  description: z.string().optional(),
+  posterImage: z.string().nullable().optional(),
+  instructor: z.string().nullable().optional(),
+  postCount: z.number().optional(),
+});
+
+export const ProductResponseSchema = z.object({
+  product: ProductSchema.optional(),
+  // Also allow fields directly on root
+  id: z.string().optional(),
+  title: z.string().optional(),
+  description: z.string().optional(),
+  posterImage: z.string().nullable().optional(),
+  instructor: z.string().nullable().optional(),
+  postCount: z.number().optional(),
+});
+
+export type ProductResponse = z.infer<typeof ProductResponseSchema>;
+export type Product = z.infer<typeof ProductSchema>;
+
+// ============================================================================
+// Helper: Safe parse with logging
+// ============================================================================
+
+/**
+ * Safely parses data with a Zod schema.
+ * Returns the parsed data or null if validation fails.
+ * Logs validation errors for debugging.
+ */
+export function safeParse<T>(schema: z.ZodType<T>, data: unknown, context?: string): T | null {
+  const result = schema.safeParse(data);
+  if (result.success) {
+    return result.data;
+  }
+
+  if (context) {
+    console.warn(`[${context}] Validation failed:`, z.treeifyError(result.error));
+  }
+
+  return null;
+}

package/src/scraper/navigator.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, it } from "vitest";
+import { createFolderName, slugify, isModuleUrl, getClassroomBaseUrl } from "./navigator.js";
+
+describe("slugify", () => {
+  it("converts to lowercase", () => {
+    expect(slugify("Hello World")).toBe("hello-world");
+  });
+
+  it("replaces spaces with hyphens", () => {
+    expect(slugify("hello world test")).toBe("hello-world-test");
+  });
+
+  it("removes special characters", () => {
+    expect(slugify("Hello! World? Test.")).toBe("hello-world-test");
+  });
+
+  it("handles German umlauts", () => {
+    expect(slugify("Größe")).toBe("groesse");
+    expect(slugify("Über")).toBe("ueber");
+    expect(slugify("Änderung")).toBe("aenderung");
+    expect(slugify("Straße")).toBe("strasse");
+  });
+
+  it("collapses multiple hyphens", () => {
+    expect(slugify("hello   world")).toBe("hello-world");
+    expect(slugify("hello---world")).toBe("hello-world");
+  });
+
+  it("removes leading and trailing hyphens", () => {
+    expect(slugify("  hello world  ")).toBe("hello-world");
+    expect(slugify("---hello---")).toBe("hello");
+  });
+
+  it("truncates to 100 characters", () => {
+    const longString = "a".repeat(150);
+    const result = slugify(longString);
+    expect(result.length).toBeLessThanOrEqual(100);
+  });
+
+  it("handles numbers", () => {
+    expect(slugify("Lesson 1: Introduction")).toBe("lesson-1-introduction");
+  });
+
+  it("handles empty string", () => {
+    expect(slugify("")).toBe("");
+  });
+
+  it("handles string with only special characters", () => {
+    // @sindresorhus/slugify converts & to "and"
+    expect(slugify("!@#$%^&*()")).toBe("and");
+    // Pure symbols without & become empty
+    expect(slugify("!@#$%^*()")).toBe("");
+  });
+});
+
+describe("createFolderName", () => {
+  it("creates folder name with zero-padded index", () => {
+    expect(createFolderName(0, "Introduction")).toBe("01-introduction");
+    expect(createFolderName(9, "Advanced Topics")).toBe("10-advanced-topics");
+  });
+
+  it("handles double-digit indices", () => {
+    expect(createFolderName(99, "Last Module")).toBe("100-last-module");
+  });
+
+  it("applies slugify to name", () => {
+    expect(createFolderName(0, "Hello World!")).toBe("01-hello-world");
+  });
+
+  it("handles German characters in name", () => {
+    expect(createFolderName(0, "Einführung")).toBe("01-einfuehrung");
+  });
+
+  it("handles empty name", () => {
+    expect(createFolderName(0, "")).toBe("01-");
+  });
+});
+
+describe("isModuleUrl", () => {
+  it("detects module URL with 8-char hex slug", () => {
+    const result = isModuleUrl("https://www.skool.com/community/classroom/a1b2c3d4");
+    expect(result).toEqual({ isModule: true, moduleSlug: "a1b2c3d4" });
+  });
+
+  it("detects module URL with query params", () => {
+    const result = isModuleUrl("https://www.skool.com/community/classroom/deadbeef?md=abc");
+    expect(result).toEqual({ isModule: true, moduleSlug: "deadbeef" });
+  });
+
+  it("returns false for classroom root", () => {
+    const result = isModuleUrl("https://www.skool.com/community/classroom");
+    expect(result).toEqual({ isModule: false, moduleSlug: null });
+  });
+
+  it("returns false for non-classroom URLs", () => {
+    const result = isModuleUrl("https://www.skool.com/community/about");
+    expect(result).toEqual({ isModule: false, moduleSlug: null });
+  });
+
+  it("only matches valid hex slugs", () => {
+    // "zzzzzzzz" is not hex
+    const result = isModuleUrl("https://www.skool.com/community/classroom/zzzzzzzz");
+    expect(result).toEqual({ isModule: false, moduleSlug: null });
+  });
+});
+
+describe("getClassroomBaseUrl", () => {
+  it("removes module slug from URL", () => {
+    const result = getClassroomBaseUrl("https://www.skool.com/community/classroom/a1b2c3d4");
+    expect(result).toBe("https://www.skool.com/community/classroom");
+  });
+
+  it("removes module slug and query params", () => {
+    const result = getClassroomBaseUrl("https://www.skool.com/community/classroom/a1b2c3d4?md=xyz");
+    expect(result).toBe("https://www.skool.com/community/classroom");
+  });
+
+  it("keeps URL unchanged if no module slug", () => {
+    const result = getClassroomBaseUrl("https://www.skool.com/community/classroom");
+    expect(result).toBe("https://www.skool.com/community/classroom");
+  });
+});

package/src/scraper/navigator.ts

@@ -0,0 +1,355 @@
+import type { Page } from "playwright";
+import {
+  parseNextData,
+  extractModulesFromNextData,
+  extractLessonAccessFromNextData,
+} from "./schemas.js";
+
+export interface CourseModule {
+  name: string;
+  slug: string;
+  url: string;
+  isLocked: boolean;
+}
+
+export interface Lesson {
+  name: string;
+  slug: string;
+  url: string;
+  index: number;
+  isLocked: boolean;
+}
+
+export interface CourseStructure {
+  name: string;
+  url: string;
+  modules: (CourseModule & { lessons: Lesson[] })[];
+}
+
+// Browser automation - requires Playwright
+/* v8 ignore start */
+
+/**
+ * Extracts the course/community name from page data.
+ */
+export async function extractCourseName(page: Page): Promise<string> {
+  const title = await page.title();
+  // Title format: "Classroom · Community Name"
+  const match = /·\s*(.+)$/.exec(title);
+  return (match?.[1]?.trim() ?? title.replace("Classroom", "").trim()) || "Unknown Course";
+}
+
+/**
+ * Extracts module data from the embedded JSON in the page.
+ * Skool embeds course structure as JSON in a script tag.
+ */
+export async function extractModulesFromJson(page: Page): Promise<CourseModule[]> {
+  // Get the raw JSON from the page
+  const nextDataJson = await page.evaluate(() => {
+    const nextDataScript = document.getElementById("__NEXT_DATA__");
+    return nextDataScript?.textContent ?? null;
+  });
+
+  // Parse and validate with Zod schema (in Node context)
+  if (nextDataJson) {
+    const parsed = parseNextData(nextDataJson);
+    if (parsed) {
+      const skoolModules = extractModulesFromNextData(parsed);
+      if (skoolModules.length > 0) {
+        const baseUrl = page.url().split("/classroom")[0];
+        return skoolModules.map((m) => ({
+          name: m.title,
+          slug: m.slug,
+          url: `${baseUrl}/classroom/${m.slug}`,
+          isLocked: !m.hasAccess,
+        }));
+      }
+    }
+  }
+
+  // Fallback: Find script tags that contain course data (regex approach)
+  const modules = await page.evaluate(() => {
+    const scripts = Array.from(document.querySelectorAll("script"));
+    const results: CourseModule[] = [];
+
+    for (const script of scripts) {
+      const content = script.textContent ?? "";
+
+      // Look for module data pattern in the JSON
+      // Structure: "id":"...","name":"SLUG","metadata":{..."title":"TITLE"...}
+      // Pattern: "name":"8-char-hex" followed by "title":"..." within metadata
+      const modulePattern = /"name":"([a-f0-9]{8})","metadata":\{[^}]*"title":"([^"]+)"/g;
+      let match;
+
+      while ((match = modulePattern.exec(content)) !== null) {
+        const slug = match[1];
+        const title = match[2];
+
+        // Skip if already added
+        if (slug && title && !results.some((m) => m.slug === slug)) {
+          // Decode unicode escapes (e.g., \u0026 -> &)
+          const decodedTitle = title.replace(/\\u([0-9a-fA-F]{4})/g, (_, code: string) =>
+            String.fromCharCode(parseInt(code, 16))
+          );
+
+          results.push({
+            name: decodedTitle,
+            slug,
+            url: "", // Will be set later
+            isLocked: false,
+          });
+        }
+      }
+    }
+
+    return results;
+  });
+
+  // Build URLs for each module
+  const baseUrl = page.url().split("/classroom")[0];
+
+  return modules.map((module) => ({
+    ...module,
+    url: `${baseUrl}/classroom/${module.slug}`,
+  }));
+}
+
+/**
+ * Extracts lessons from a module page.
+ * Lessons are listed in the sidebar with links.
+ */
+export async function extractLessons(page: Page, moduleUrl: string): Promise<Lesson[]> {
+  const currentUrl = page.url();
+
+  const moduleBasePath = moduleUrl.split("?")[0] ?? moduleUrl;
+  if (!currentUrl.includes(moduleBasePath)) {
+    await page.goto(moduleUrl, { timeout: 30000 });
+    await page.waitForLoadState("domcontentloaded");
+    await page.waitForTimeout(2000);
+  }
+
+  // Get __NEXT_DATA__ and parse it in Node context
+  const nextDataJson = await page.evaluate(() => {
+    const nextDataScript = document.getElementById("__NEXT_DATA__");
+    return nextDataScript?.textContent ?? null;
+  });
+
+  // Build access map from validated data
+  let accessMap = new Map<string, boolean>();
+  if (nextDataJson) {
+    const parsed = parseNextData(nextDataJson);
+    if (parsed) {
+      accessMap = extractLessonAccessFromNextData(parsed);
+    }
+  }
+
+  // Extract lesson links from DOM
+  const lessonData = await page.evaluate(() => {
+    const results: { name: string; slug: string; href: string }[] = [];
+
+    // Skool uses styled-components with "ChildrenLink" in the class name
+    const lessonLinks = document.querySelectorAll('a[class*="ChildrenLink"]');
+
+    lessonLinks.forEach((link, index) => {
+      const anchor = link as HTMLAnchorElement;
+      const href = anchor.href;
+      const name = anchor.textContent?.trim() ?? `Lesson ${index + 1}`;
+
+      // Extract lesson ID from URL (?md=...)
+      const urlParams = new URL(href).searchParams;
+      const lessonId = urlParams.get("md") ?? "";
+
+      if (lessonId && !results.some((l) => l.slug === lessonId)) {
+        results.push({ name, slug: lessonId, href });
+      }
+    });
+
+    return results;
+  });
+
+  // Build final lesson list with access info
+  return lessonData.map((lesson, index) => {
+    let isLocked = false;
+
+    // Check access map from __NEXT_DATA__
+    if (accessMap.has(lesson.slug)) {
+      isLocked = !accessMap.get(lesson.slug);
+    }
+
+    return {
+      name: lesson.name,
+      slug: lesson.slug,
+      url: lesson.href,
+      index,
+      isLocked,
+    };
+  });
+}
+
+/**
+ * Alternative: Extract modules from the classroom overview page links.
+ */
+export async function extractModulesFromPage(page: Page): Promise<CourseModule[]> {
+  await page.waitForTimeout(1000);
+
+  const modules = await page.evaluate(() => {
+    // Look for module cards - they're usually divs/links with course images
+    const moduleCards = document.querySelectorAll('a[href*="/classroom/"]');
+    const results: CourseModule[] = [];
+    const seen = new Set<string>();
+
+    moduleCards.forEach((card) => {
+      const anchor = card as HTMLAnchorElement;
+      const href = anchor.href;
+
+      // Extract slug from URL (8 character hex string)
+      const slugMatch = /\/classroom\/([a-f0-9]{8})(?:\?|$)/.exec(href);
+      if (!slugMatch?.[1]) return;
+
+      const slug = slugMatch[1];
+      if (seen.has(slug)) return;
+      seen.add(slug);
+
+      // Find title - could be in various child elements
+      const titleEl =
+        card.querySelector("h3, h4, [class*='title'], [class*='Title']") ??
+        card.querySelector("div > div > div");
+      const name = titleEl?.textContent?.trim() ?? `Module ${results.length + 1}`;
+
+      // Check for lock icon
+      const isLocked = card.querySelector('[class*="lock"], [class*="Lock"]') !== null;
+
+      results.push({
+        name,
+        slug,
+        url: href,
+        isLocked,
+      });
+    });
+
+    return results;
+  });
+
+  return modules;
+}
+/* v8 ignore stop */
+
+/**
+ * Checks if a URL points to a specific module (has 8-char hex slug).
+ */
+export function isModuleUrl(url: string): { isModule: boolean; moduleSlug: string | null } {
+  const match = /\/classroom\/([a-f0-9]{8})(?:\?|$)/.exec(url);
+  return {
+    isModule: !!match,
+    moduleSlug: match?.[1] ?? null,
+  };
+}
+
+/**
+ * Gets the classroom base URL (without module slug).
+ */
+export function getClassroomBaseUrl(url: string): string {
+  // Remove module slug and query params
+  return url.replace(/\/classroom\/[a-f0-9]{8}.*$/, "/classroom");
+}
+
+/**
+ * Progress callback for buildCourseStructure.
+ */
+export interface ScanProgress {
+  phase: "init" | "modules" | "lessons" | "done";
+  courseName?: string;
+  totalModules?: number;
+  currentModule?: string;
+  currentModuleIndex?: number;
+  lessonsFound?: number;
+  skippedLocked?: boolean;
+}
+
+/* v8 ignore start */
+/**
+ * Builds the complete course structure by crawling all modules and lessons.
+ */
+export async function buildCourseStructure(
+  page: Page,
+  classroomUrl: string,
+  onProgress?: (progress: ScanProgress) => void
+): Promise<CourseStructure> {
+  const { isModule, moduleSlug } = isModuleUrl(classroomUrl);
+
+  // If URL points to a specific module, get the base classroom URL first
+  const baseClassroomUrl = isModule ? getClassroomBaseUrl(classroomUrl) : classroomUrl;
+
+  // Navigate to the classroom overview to get all modules
+  await page.goto(baseClassroomUrl, { timeout: 30000 });
+  await page.waitForLoadState("domcontentloaded");
+  await page.waitForTimeout(2000);
+
+  const courseName = await extractCourseName(page);
+  onProgress?.({ phase: "init", courseName });
+
+  // Try JSON extraction first (more reliable), fall back to page scraping
+  let modules = await extractModulesFromJson(page);
+
+  if (modules.length === 0) {
+    modules = await extractModulesFromPage(page);
+  }
+
+  // If user specified a specific module, filter to just that one
+  if (isModule && moduleSlug) {
+    const targetModule = modules.find((m) => m.slug === moduleSlug);
+    if (targetModule) {
+      modules = [targetModule];
+    }
+  }
+
+  onProgress?.({ phase: "modules", totalModules: modules.length });
+
+  const modulesWithLessons: CourseStructure["modules"] = [];
+
+  for (const [i, module] of modules.entries()) {
+    if (module.isLocked) {
+      onProgress?.({
+        phase: "lessons",
+        currentModule: module.name,
+        currentModuleIndex: i,
+        skippedLocked: true,
+      });
+      continue;
+    }
+
+    onProgress?.({
+      phase: "lessons",
+      currentModule: module.name,
+      currentModuleIndex: i,
+    });
+
+    if (module.url) {
+      const lessons = await extractLessons(page, module.url);
+
+      onProgress?.({
+        phase: "lessons",
+        currentModule: module.name,
+        currentModuleIndex: i,
+        lessonsFound: lessons.length,
+      });
+
+      modulesWithLessons.push({
+        ...module,
+        lessons,
+      });
+    }
+  }
+
+  onProgress?.({ phase: "done" });
+
+  return {
+    name: courseName,
+    url: baseClassroomUrl,
+    modules: modulesWithLessons,
+  };
+}
+/* v8 ignore stop */
+
+// Re-export shared utilities for backwards compatibility
+export { slugify, createFolderName } from "../shared/slug.js";