offcourse 1.0.0 → 1.0.1

package/src/scraper/highlevel/schemas.ts

@@ -1,183 +0,0 @@
-/**
- * Zod schemas for HighLevel API responses.
- * These provide runtime validation and type inference.
- */
-
-import { z } from "zod";
-
-// Re-export Firebase auth types (Firebase is used by HighLevel for auth)
-export {
-  FirebaseAuthTokenSchema,
-  type FirebaseAuthToken,
-  type FirebaseAuthRaw,
-} from "../../shared/firebase.js";
-
-// ============================================================================
-// Portal Settings API
-// ============================================================================
-
-export const PortalSettingsResponseSchema = z.object({
-  locationId: z.string(),
-  portalName: z.string().optional(),
-  name: z.string().optional(),
-});
-
-export type PortalSettingsResponse = z.infer<typeof PortalSettingsResponseSchema>;
-
-// ============================================================================
-// Video License API
-// ============================================================================
-
-export const VideoLicenseResponseSchema = z.object({
-  url: z.string(),
-  token: z.string(),
-});
-
-export type VideoLicenseResponse = z.infer<typeof VideoLicenseResponseSchema>;
-
-// ============================================================================
-// Post Details API
-// ============================================================================
-
-const VideoAssetSchema = z.object({
-  id: z.string().optional(),
-  assetId: z.string().optional(),
-  assetsLicenseId: z.string().optional(),
-  url: z.string().optional(),
-});
-
-const PosterImageSchema = z.object({
-  assetId: z.string().optional(),
-  url: z.string().optional(),
-});
-
-const ContentBlockSchema = z.object({
-  type: z.string(),
-  id: z.string().optional(),
-  assetId: z.string().optional(),
-  assetsLicenseId: z.string().optional(),
-  url: z.string().optional(),
-});
-
-const MaterialSchema = z.object({
-  id: z.string().optional(),
-  name: z.string().optional(),
-  url: z.string().optional(),
-  type: z.string().optional(),
-});
-
-export const PostDetailsSchema = z.object({
-  title: z.string().optional(),
-  description: z.string().nullable().optional(),
-  video: VideoAssetSchema.nullable().optional(),
-  posterImage: PosterImageSchema.nullable().optional(),
-  contentBlock: z.array(ContentBlockSchema).optional(),
-  materials: z.array(MaterialSchema).optional(),
-  post_materials: z.array(MaterialSchema).optional(),
-});
-
-// Response can have data directly or wrapped in "post"
-export const PostDetailsResponseSchema = z.object({
-  post: PostDetailsSchema.optional(),
-  // Also allow all post fields directly on root
-  title: z.string().optional(),
-  description: z.string().nullable().optional(),
-  video: VideoAssetSchema.nullable().optional(),
-  posterImage: PosterImageSchema.nullable().optional(),
-  contentBlock: z.array(ContentBlockSchema).optional(),
-  materials: z.array(MaterialSchema).optional(),
-  post_materials: z.array(MaterialSchema).optional(),
-});
-
-export type PostDetailsResponse = z.infer<typeof PostDetailsResponseSchema>;
-
-// ============================================================================
-// Categories API
-// ============================================================================
-
-export const CategorySchema = z.object({
-  id: z.string(),
-  title: z.string(),
-  description: z.string().nullable().optional(),
-  position: z.number().optional(),
-  postCount: z.number().optional(),
-  visibility: z.string().optional(),
-});
-
-export const CategoriesResponseSchema = z.object({
-  categories: z.array(CategorySchema),
-});
-
-export type CategoriesResponse = z.infer<typeof CategoriesResponseSchema>;
-export type Category = z.infer<typeof CategorySchema>;
-
-// ============================================================================
-// Posts (Lessons) API
-// ============================================================================
-
-export const PostSchema = z.object({
-  id: z.string(),
-  title: z.string(),
-  indexPosition: z.number().optional(),
-  visibility: z.string().optional(),
-});
-
-export const PostsResponseSchema = z.object({
-  category: z
-    .object({
-      posts: z.array(PostSchema),
-    })
-    .optional(),
-});
-
-export type PostsResponse = z.infer<typeof PostsResponseSchema>;
-export type Post = z.infer<typeof PostSchema>;
-
-// ============================================================================
-// Product (Course) API
-// ============================================================================
-
-export const ProductSchema = z.object({
-  id: z.string().optional(),
-  title: z.string(),
-  description: z.string().optional(),
-  posterImage: z.string().nullable().optional(),
-  instructor: z.string().nullable().optional(),
-  postCount: z.number().optional(),
-});
-
-export const ProductResponseSchema = z.object({
-  product: ProductSchema.optional(),
-  // Also allow fields directly on root
-  id: z.string().optional(),
-  title: z.string().optional(),
-  description: z.string().optional(),
-  posterImage: z.string().nullable().optional(),
-  instructor: z.string().nullable().optional(),
-  postCount: z.number().optional(),
-});
-
-export type ProductResponse = z.infer<typeof ProductResponseSchema>;
-export type Product = z.infer<typeof ProductSchema>;
-
-// ============================================================================
-// Helper: Safe parse with logging
-// ============================================================================
-
-/**
- * Safely parses data with a Zod schema.
- * Returns the parsed data or null if validation fails.
- * Logs validation errors for debugging.
- */
-export function safeParse<T>(schema: z.ZodType<T>, data: unknown, context?: string): T | null {
-  const result = schema.safeParse(data);
-  if (result.success) {
-    return result.data;
-  }
-
-  if (context) {
-    console.warn(`[${context}] Validation failed:`, z.treeifyError(result.error));
-  }
-
-  return null;
-}

package/src/scraper/navigator.test.ts

@@ -1,122 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { createFolderName, slugify, isModuleUrl, getClassroomBaseUrl } from "./navigator.js";
-
-describe("slugify", () => {
-  it("converts to lowercase", () => {
-    expect(slugify("Hello World")).toBe("hello-world");
-  });
-
-  it("replaces spaces with hyphens", () => {
-    expect(slugify("hello world test")).toBe("hello-world-test");
-  });
-
-  it("removes special characters", () => {
-    expect(slugify("Hello! World? Test.")).toBe("hello-world-test");
-  });
-
-  it("handles German umlauts", () => {
-    expect(slugify("Größe")).toBe("groesse");
-    expect(slugify("Über")).toBe("ueber");
-    expect(slugify("Änderung")).toBe("aenderung");
-    expect(slugify("Straße")).toBe("strasse");
-  });
-
-  it("collapses multiple hyphens", () => {
-    expect(slugify("hello   world")).toBe("hello-world");
-    expect(slugify("hello---world")).toBe("hello-world");
-  });
-
-  it("removes leading and trailing hyphens", () => {
-    expect(slugify("  hello world  ")).toBe("hello-world");
-    expect(slugify("---hello---")).toBe("hello");
-  });
-
-  it("truncates to 100 characters", () => {
-    const longString = "a".repeat(150);
-    const result = slugify(longString);
-    expect(result.length).toBeLessThanOrEqual(100);
-  });
-
-  it("handles numbers", () => {
-    expect(slugify("Lesson 1: Introduction")).toBe("lesson-1-introduction");
-  });
-
-  it("handles empty string", () => {
-    expect(slugify("")).toBe("");
-  });
-
-  it("handles string with only special characters", () => {
-    // @sindresorhus/slugify converts & to "and"
-    expect(slugify("!@#$%^&*()")).toBe("and");
-    // Pure symbols without & become empty
-    expect(slugify("!@#$%^*()")).toBe("");
-  });
-});
-
-describe("createFolderName", () => {
-  it("creates folder name with zero-padded index", () => {
-    expect(createFolderName(0, "Introduction")).toBe("01-introduction");
-    expect(createFolderName(9, "Advanced Topics")).toBe("10-advanced-topics");
-  });
-
-  it("handles double-digit indices", () => {
-    expect(createFolderName(99, "Last Module")).toBe("100-last-module");
-  });
-
-  it("applies slugify to name", () => {
-    expect(createFolderName(0, "Hello World!")).toBe("01-hello-world");
-  });
-
-  it("handles German characters in name", () => {
-    expect(createFolderName(0, "Einführung")).toBe("01-einfuehrung");
-  });
-
-  it("handles empty name", () => {
-    expect(createFolderName(0, "")).toBe("01-");
-  });
-});
-
-describe("isModuleUrl", () => {
-  it("detects module URL with 8-char hex slug", () => {
-    const result = isModuleUrl("https://www.skool.com/community/classroom/a1b2c3d4");
-    expect(result).toEqual({ isModule: true, moduleSlug: "a1b2c3d4" });
-  });
-
-  it("detects module URL with query params", () => {
-    const result = isModuleUrl("https://www.skool.com/community/classroom/deadbeef?md=abc");
-    expect(result).toEqual({ isModule: true, moduleSlug: "deadbeef" });
-  });
-
-  it("returns false for classroom root", () => {
-    const result = isModuleUrl("https://www.skool.com/community/classroom");
-    expect(result).toEqual({ isModule: false, moduleSlug: null });
-  });
-
-  it("returns false for non-classroom URLs", () => {
-    const result = isModuleUrl("https://www.skool.com/community/about");
-    expect(result).toEqual({ isModule: false, moduleSlug: null });
-  });
-
-  it("only matches valid hex slugs", () => {
-    // "zzzzzzzz" is not hex
-    const result = isModuleUrl("https://www.skool.com/community/classroom/zzzzzzzz");
-    expect(result).toEqual({ isModule: false, moduleSlug: null });
-  });
-});
-
-describe("getClassroomBaseUrl", () => {
-  it("removes module slug from URL", () => {
-    const result = getClassroomBaseUrl("https://www.skool.com/community/classroom/a1b2c3d4");
-    expect(result).toBe("https://www.skool.com/community/classroom");
-  });
-
-  it("removes module slug and query params", () => {
-    const result = getClassroomBaseUrl("https://www.skool.com/community/classroom/a1b2c3d4?md=xyz");
-    expect(result).toBe("https://www.skool.com/community/classroom");
-  });
-
-  it("keeps URL unchanged if no module slug", () => {
-    const result = getClassroomBaseUrl("https://www.skool.com/community/classroom");
-    expect(result).toBe("https://www.skool.com/community/classroom");
-  });
-});

package/src/scraper/navigator.ts

@@ -1,355 +0,0 @@
-import type { Page } from "playwright";
-import {
-  parseNextData,
-  extractModulesFromNextData,
-  extractLessonAccessFromNextData,
-} from "./schemas.js";
-
-export interface CourseModule {
-  name: string;
-  slug: string;
-  url: string;
-  isLocked: boolean;
-}
-
-export interface Lesson {
-  name: string;
-  slug: string;
-  url: string;
-  index: number;
-  isLocked: boolean;
-}
-
-export interface CourseStructure {
-  name: string;
-  url: string;
-  modules: (CourseModule & { lessons: Lesson[] })[];
-}
-
-// Browser automation - requires Playwright
-/* v8 ignore start */
-
-/**
- * Extracts the course/community name from page data.
- */
-export async function extractCourseName(page: Page): Promise<string> {
-  const title = await page.title();
-  // Title format: "Classroom · Community Name"
-  const match = /·\s*(.+)$/.exec(title);
-  return (match?.[1]?.trim() ?? title.replace("Classroom", "").trim()) || "Unknown Course";
-}
-
-/**
- * Extracts module data from the embedded JSON in the page.
- * Skool embeds course structure as JSON in a script tag.
- */
-export async function extractModulesFromJson(page: Page): Promise<CourseModule[]> {
-  // Get the raw JSON from the page
-  const nextDataJson = await page.evaluate(() => {
-    const nextDataScript = document.getElementById("__NEXT_DATA__");
-    return nextDataScript?.textContent ?? null;
-  });
-
-  // Parse and validate with Zod schema (in Node context)
-  if (nextDataJson) {
-    const parsed = parseNextData(nextDataJson);
-    if (parsed) {
-      const skoolModules = extractModulesFromNextData(parsed);
-      if (skoolModules.length > 0) {
-        const baseUrl = page.url().split("/classroom")[0];
-        return skoolModules.map((m) => ({
-          name: m.title,
-          slug: m.slug,
-          url: `${baseUrl}/classroom/${m.slug}`,
-          isLocked: !m.hasAccess,
-        }));
-      }
-    }
-  }
-
-  // Fallback: Find script tags that contain course data (regex approach)
-  const modules = await page.evaluate(() => {
-    const scripts = Array.from(document.querySelectorAll("script"));
-    const results: CourseModule[] = [];
-
-    for (const script of scripts) {
-      const content = script.textContent ?? "";
-
-      // Look for module data pattern in the JSON
-      // Structure: "id":"...","name":"SLUG","metadata":{..."title":"TITLE"...}
-      // Pattern: "name":"8-char-hex" followed by "title":"..." within metadata
-      const modulePattern = /"name":"([a-f0-9]{8})","metadata":\{[^}]*"title":"([^"]+)"/g;
-      let match;
-
-      while ((match = modulePattern.exec(content)) !== null) {
-        const slug = match[1];
-        const title = match[2];
-
-        // Skip if already added
-        if (slug && title && !results.some((m) => m.slug === slug)) {
-          // Decode unicode escapes (e.g., \u0026 -> &)
-          const decodedTitle = title.replace(/\\u([0-9a-fA-F]{4})/g, (_, code: string) =>
-            String.fromCharCode(parseInt(code, 16))
-          );
-
-          results.push({
-            name: decodedTitle,
-            slug,
-            url: "", // Will be set later
-            isLocked: false,
-          });
-        }
-      }
-    }
-
-    return results;
-  });
-
-  // Build URLs for each module
-  const baseUrl = page.url().split("/classroom")[0];
-
-  return modules.map((module) => ({
-    ...module,
-    url: `${baseUrl}/classroom/${module.slug}`,
-  }));
-}
-
-/**
- * Extracts lessons from a module page.
- * Lessons are listed in the sidebar with links.
- */
-export async function extractLessons(page: Page, moduleUrl: string): Promise<Lesson[]> {
-  const currentUrl = page.url();
-
-  const moduleBasePath = moduleUrl.split("?")[0] ?? moduleUrl;
-  if (!currentUrl.includes(moduleBasePath)) {
-    await page.goto(moduleUrl, { timeout: 30000 });
-    await page.waitForLoadState("domcontentloaded");
-    await page.waitForTimeout(2000);
-  }
-
-  // Get __NEXT_DATA__ and parse it in Node context
-  const nextDataJson = await page.evaluate(() => {
-    const nextDataScript = document.getElementById("__NEXT_DATA__");
-    return nextDataScript?.textContent ?? null;
-  });
-
-  // Build access map from validated data
-  let accessMap = new Map<string, boolean>();
-  if (nextDataJson) {
-    const parsed = parseNextData(nextDataJson);
-    if (parsed) {
-      accessMap = extractLessonAccessFromNextData(parsed);
-    }
-  }
-
-  // Extract lesson links from DOM
-  const lessonData = await page.evaluate(() => {
-    const results: { name: string; slug: string; href: string }[] = [];
-
-    // Skool uses styled-components with "ChildrenLink" in the class name
-    const lessonLinks = document.querySelectorAll('a[class*="ChildrenLink"]');
-
-    lessonLinks.forEach((link, index) => {
-      const anchor = link as HTMLAnchorElement;
-      const href = anchor.href;
-      const name = anchor.textContent?.trim() ?? `Lesson ${index + 1}`;
-
-      // Extract lesson ID from URL (?md=...)
-      const urlParams = new URL(href).searchParams;
-      const lessonId = urlParams.get("md") ?? "";
-
-      if (lessonId && !results.some((l) => l.slug === lessonId)) {
-        results.push({ name, slug: lessonId, href });
-      }
-    });
-
-    return results;
-  });
-
-  // Build final lesson list with access info
-  return lessonData.map((lesson, index) => {
-    let isLocked = false;
-
-    // Check access map from __NEXT_DATA__
-    if (accessMap.has(lesson.slug)) {
-      isLocked = !accessMap.get(lesson.slug);
-    }
-
-    return {
-      name: lesson.name,
-      slug: lesson.slug,
-      url: lesson.href,
-      index,
-      isLocked,
-    };
-  });
-}
-
-/**
- * Alternative: Extract modules from the classroom overview page links.
- */
-export async function extractModulesFromPage(page: Page): Promise<CourseModule[]> {
-  await page.waitForTimeout(1000);
-
-  const modules = await page.evaluate(() => {
-    // Look for module cards - they're usually divs/links with course images
-    const moduleCards = document.querySelectorAll('a[href*="/classroom/"]');
-    const results: CourseModule[] = [];
-    const seen = new Set<string>();
-
-    moduleCards.forEach((card) => {
-      const anchor = card as HTMLAnchorElement;
-      const href = anchor.href;
-
-      // Extract slug from URL (8 character hex string)
-      const slugMatch = /\/classroom\/([a-f0-9]{8})(?:\?|$)/.exec(href);
-      if (!slugMatch?.[1]) return;
-
-      const slug = slugMatch[1];
-      if (seen.has(slug)) return;
-      seen.add(slug);
-
-      // Find title - could be in various child elements
-      const titleEl =
-        card.querySelector("h3, h4, [class*='title'], [class*='Title']") ??
-        card.querySelector("div > div > div");
-      const name = titleEl?.textContent?.trim() ?? `Module ${results.length + 1}`;
-
-      // Check for lock icon
-      const isLocked = card.querySelector('[class*="lock"], [class*="Lock"]') !== null;
-
-      results.push({
-        name,
-        slug,
-        url: href,
-        isLocked,
-      });
-    });
-
-    return results;
-  });
-
-  return modules;
-}
-/* v8 ignore stop */
-
-/**
- * Checks if a URL points to a specific module (has 8-char hex slug).
- */
-export function isModuleUrl(url: string): { isModule: boolean; moduleSlug: string | null } {
-  const match = /\/classroom\/([a-f0-9]{8})(?:\?|$)/.exec(url);
-  return {
-    isModule: !!match,
-    moduleSlug: match?.[1] ?? null,
-  };
-}
-
-/**
- * Gets the classroom base URL (without module slug).
- */
-export function getClassroomBaseUrl(url: string): string {
-  // Remove module slug and query params
-  return url.replace(/\/classroom\/[a-f0-9]{8}.*$/, "/classroom");
-}
-
-/**
- * Progress callback for buildCourseStructure.
- */
-export interface ScanProgress {
-  phase: "init" | "modules" | "lessons" | "done";
-  courseName?: string;
-  totalModules?: number;
-  currentModule?: string;
-  currentModuleIndex?: number;
-  lessonsFound?: number;
-  skippedLocked?: boolean;
-}
-
-/* v8 ignore start */
-/**
- * Builds the complete course structure by crawling all modules and lessons.
- */
-export async function buildCourseStructure(
-  page: Page,
-  classroomUrl: string,
-  onProgress?: (progress: ScanProgress) => void
-): Promise<CourseStructure> {
-  const { isModule, moduleSlug } = isModuleUrl(classroomUrl);
-
-  // If URL points to a specific module, get the base classroom URL first
-  const baseClassroomUrl = isModule ? getClassroomBaseUrl(classroomUrl) : classroomUrl;
-
-  // Navigate to the classroom overview to get all modules
-  await page.goto(baseClassroomUrl, { timeout: 30000 });
-  await page.waitForLoadState("domcontentloaded");
-  await page.waitForTimeout(2000);
-
-  const courseName = await extractCourseName(page);
-  onProgress?.({ phase: "init", courseName });
-
-  // Try JSON extraction first (more reliable), fall back to page scraping
-  let modules = await extractModulesFromJson(page);
-
-  if (modules.length === 0) {
-    modules = await extractModulesFromPage(page);
-  }
-
-  // If user specified a specific module, filter to just that one
-  if (isModule && moduleSlug) {
-    const targetModule = modules.find((m) => m.slug === moduleSlug);
-    if (targetModule) {
-      modules = [targetModule];
-    }
-  }
-
-  onProgress?.({ phase: "modules", totalModules: modules.length });
-
-  const modulesWithLessons: CourseStructure["modules"] = [];
-
-  for (const [i, module] of modules.entries()) {
-    if (module.isLocked) {
-      onProgress?.({
-        phase: "lessons",
-        currentModule: module.name,
-        currentModuleIndex: i,
-        skippedLocked: true,
-      });
-      continue;
-    }
-
-    onProgress?.({
-      phase: "lessons",
-      currentModule: module.name,
-      currentModuleIndex: i,
-    });
-
-    if (module.url) {
-      const lessons = await extractLessons(page, module.url);
-
-      onProgress?.({
-        phase: "lessons",
-        currentModule: module.name,
-        currentModuleIndex: i,
-        lessonsFound: lessons.length,
-      });
-
-      modulesWithLessons.push({
-        ...module,
-        lessons,
-      });
-    }
-  }
-
-  onProgress?.({ phase: "done" });
-
-  return {
-    name: courseName,
-    url: baseClassroomUrl,
-    modules: modulesWithLessons,
-  };
-}
-/* v8 ignore stop */
-
-// Re-export shared utilities for backwards compatibility
-export { slugify, createFolderName } from "../shared/slug.js";