offcourse 0.0.2 → 1.0.0

package/src/scraper/schemas.ts

@@ -0,0 +1,177 @@
+/**
+ * Zod schemas for Skool/Next.js __NEXT_DATA__ responses.
+ * These validate only the fields we actually use, ignoring everything else.
+ */
+
+import { z } from "zod";
+
+// ============================================================================
+// Skool Course Child (Module/Lesson)
+// ============================================================================
+
+const CourseMetadataSchema = z.looseObject({
+  title: z.string().optional(),
+  videoLink: z.string().optional(),
+});
+
+const CourseInfoSchema = z.looseObject({
+  id: z.string().optional(),
+  name: z.string().optional(), // 8-char hex slug
+  metadata: CourseMetadataSchema.optional(),
+});
+
+const CourseChildSchema = z.looseObject({
+  course: CourseInfoSchema.optional(),
+  hasAccess: z.boolean().optional(),
+});
+
+// ============================================================================
+// Skool __NEXT_DATA__ PageProps
+// ============================================================================
+
+const SkoolCourseSchema = z.looseObject({
+  children: z.array(CourseChildSchema).optional(),
+});
+
+const SkoolPagePropsSchema = z.looseObject({
+  course: SkoolCourseSchema.optional(),
+  selectedModule: z.string().optional(),
+});
+
+export const SkoolNextDataSchema = z.looseObject({
+  props: z
+    .looseObject({
+      pageProps: SkoolPagePropsSchema.optional(),
+    })
+    .optional(),
+});
+
+export type SkoolNextData = z.infer<typeof SkoolNextDataSchema>;
+
+// ============================================================================
+// Extracted Types (clean types for use in the app)
+// ============================================================================
+
+export interface SkoolModule {
+  slug: string; // 8-char hex
+  title: string;
+  hasAccess: boolean;
+}
+
+export interface SkoolLesson {
+  id: string;
+  hasAccess: boolean;
+}
+
+export interface SkoolVideoInfo {
+  url: string;
+  type: "loom" | "vimeo" | "youtube" | "wistia" | "unknown";
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/**
+ * Safely parses __NEXT_DATA__ JSON from a script element.
+ * Returns null if parsing fails.
+ */
+export function parseNextData(json: string): SkoolNextData | null {
+  try {
+    const data: unknown = JSON.parse(json);
+    const result = SkoolNextDataSchema.safeParse(data);
+    if (result.success) {
+      return result.data;
+    }
+    console.warn("[parseNextData] Validation failed:", z.treeifyError(result.error));
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Extracts modules from parsed __NEXT_DATA__.
+ */
+export function extractModulesFromNextData(data: SkoolNextData): SkoolModule[] {
+  const children = data.props?.pageProps?.course?.children;
+  if (!Array.isArray(children)) return [];
+
+  const modules: SkoolModule[] = [];
+  const seen = new Set<string>();
+
+  for (const child of children) {
+    const course = child.course;
+    if (!course?.name) continue;
+
+    // Skool module slugs are 8-char hex strings
+    if (!/^[a-f0-9]{8}$/.test(course.name)) continue;
+
+    if (seen.has(course.name)) continue;
+    seen.add(course.name);
+
+    modules.push({
+      slug: course.name,
+      title: course.metadata?.title ?? `Module ${modules.length + 1}`,
+      hasAccess: child.hasAccess !== false,
+    });
+  }
+
+  return modules;
+}
+
+/**
+ * Extracts lesson access info from parsed __NEXT_DATA__.
+ */
+export function extractLessonAccessFromNextData(data: SkoolNextData): Map<string, boolean> {
+  const accessMap = new Map<string, boolean>();
+  const children = data.props?.pageProps?.course?.children;
+
+  if (!Array.isArray(children)) return accessMap;
+
+  for (const child of children) {
+    const id = child.course?.id;
+    const hasAccess = child.hasAccess;
+    if (id && typeof hasAccess === "boolean") {
+      accessMap.set(id, hasAccess);
+    }
+  }
+
+  return accessMap;
+}
+
+/**
+ * Extracts video URL from parsed __NEXT_DATA__ for a specific module.
+ */
+export function extractVideoFromNextData(
+  data: SkoolNextData,
+  selectedModuleId: string
+): SkoolVideoInfo | null {
+  const children = data.props?.pageProps?.course?.children;
+  if (!Array.isArray(children)) return null;
+
+  for (const child of children) {
+    if (child.course?.id === selectedModuleId) {
+      const videoLink = child.course.metadata?.videoLink;
+      if (!videoLink) return null;
+
+      // Determine video type
+      if (videoLink.includes("loom.com")) {
+        const embedUrl = videoLink.replace("/share/", "/embed/").split("?")[0];
+        return { url: embedUrl ?? videoLink, type: "loom" };
+      }
+      if (videoLink.includes("vimeo.com")) {
+        return { url: videoLink, type: "vimeo" };
+      }
+      if (videoLink.includes("youtube.com") || videoLink.includes("youtu.be")) {
+        return { url: videoLink, type: "youtube" };
+      }
+      if (videoLink.includes("wistia")) {
+        return { url: videoLink, type: "wistia" };
+      }
+      return { url: videoLink, type: "unknown" };
+    }
+  }
+
+  return null;
+}

package/src/scraper/videoInterceptor.ts

@@ -0,0 +1,435 @@
+/**
+ * Browser-based video URL interception - requires Playwright.
+ * Excluded from coverage via vitest.config.ts.
+ */
+import type { Page } from "playwright";
+
+// ============================================================================
+// Type definitions for external browser APIs
+// ============================================================================
+
+/** Vimeo player configuration embedded in the page */
+interface VimeoPlayerConfig {
+  request?: {
+    files?: {
+      hls?: {
+        cdns?: Record<string, { url?: string }>;
+      };
+      progressive?: { url?: string; height?: number }[];
+    };
+  };
+}
+
+/** Vimeo-related window properties */
+interface VimeoWindow {
+  playerConfig?: VimeoPlayerConfig;
+  vimeo?: { config?: VimeoPlayerConfig };
+  __vimeo_player__?: { config?: VimeoPlayerConfig };
+}
+
+/** Loom video asset URLs */
+interface LoomAssetUrls {
+  hls_url?: string;
+}
+
+/** Loom video data */
+interface LoomVideo {
+  asset_urls?: LoomAssetUrls;
+}
+
+/** Loom SSR state embedded in the page */
+interface LoomSSRState {
+  video?: LoomVideo;
+}
+
+/** Loom-related window properties */
+interface LoomWindow {
+  __LOOM_SSR_STATE__?: LoomSSRState;
+}
+
+/** Next.js data for Loom pages */
+interface LoomNextData {
+  props?: {
+    pageProps?: {
+      video?: LoomVideo;
+    };
+  };
+}
+
+/**
+ * Captures Vimeo video URL by extracting it from the running player.
+ * The key insight: the video is ALREADY playing in the iframe - we just need to get the URL.
+ */
+export async function captureVimeoConfig(
+  page: Page,
+  _videoId: string,
+  timeoutMs = 20000
+): Promise<{ hlsUrl: string | null; progressiveUrl: string | null; error?: string }> {
+  try {
+    // Step 1: Make sure we have a Vimeo iframe or video wrapper
+    // Skool wraps videos in a VideoPlayerWrapper - click it to ensure video loads
+    const videoWrapper = await page.$(
+      '[class*="VideoPlayerWrapper"], [class*="video-wrapper"], [class*="VideoPlayer"]'
+    );
+    if (videoWrapper) {
+      await videoWrapper.click().catch(() => {});
+      await page.waitForTimeout(1000);
+    }
+
+    // Step 2: Wait for Vimeo iframe to appear
+    let vimeoFrame = null;
+    const startTime = Date.now();
+
+    while (!vimeoFrame && Date.now() - startTime < timeoutMs) {
+      // Try to find the iframe
+      const iframe = await page.$('iframe[src*="vimeo.com"], iframe[src*="player.vimeo"]');
+      if (iframe) {
+        vimeoFrame = await iframe.contentFrame();
+        if (vimeoFrame) break;
+      }
+      await page.waitForTimeout(500);
+    }
+
+    if (!vimeoFrame) {
+      return { hlsUrl: null, progressiveUrl: null, error: "Vimeo iframe not found after waiting" };
+    }
+
+    // Step 3: Mute the video before playing (we don't want audio!)
+    await vimeoFrame
+      .evaluate(() => {
+        const video = document.querySelector("video");
+        if (video) {
+          video.muted = true;
+          video.volume = 0;
+        }
+      })
+      .catch(() => {});
+
+    // Step 4: Click play button in the iframe to start video
+    try {
+      // Multiple selectors for Vimeo's play button
+      await vimeoFrame
+        .click(
+          '.vp-controls button, .play-icon, [aria-label="Play"], .vp-big-play-button, button',
+          {
+            timeout: 2000,
+          }
+        )
+        .catch(() => {});
+    } catch {
+      // Video might auto-play
+    }
+
+    // Ensure video stays muted
+    await vimeoFrame
+      .evaluate(() => {
+        const video = document.querySelector("video");
+        if (video) {
+          video.muted = true;
+          video.volume = 0;
+        }
+      })
+      .catch(() => {});
+
+    // Step 4: Wait for video to actually start playing and get the URL
+    let hlsUrl: string | null = null;
+    let progressiveUrl: string | null = null;
+
+    const extractionStart = Date.now();
+    while (!hlsUrl && !progressiveUrl && Date.now() - extractionStart < timeoutMs - 5000) {
+      const urls = await vimeoFrame.evaluate(() => {
+        const result = {
+          hlsUrl: null as string | null,
+          progressiveUrl: null as string | null,
+          debug: [] as string[],
+        };
+
+        // Method 1: Get URL directly from video element
+        const video = document.querySelector("video");
+        if (video) {
+          result.debug.push(`Video element found, src length: ${video.src?.length ?? 0}`);
+
+          // Check currentSrc (what's actually playing)
+          if (video.currentSrc) {
+            result.debug.push(`currentSrc: ${video.currentSrc.substring(0, 80)}`);
+            if (video.currentSrc.includes(".m3u8")) {
+              result.hlsUrl = video.currentSrc;
+            } else if (video.currentSrc.includes(".mp4")) {
+              result.progressiveUrl = video.currentSrc;
+            }
+          }
+
+          // Also check src attribute
+          if (!result.hlsUrl && !result.progressiveUrl && video.src) {
+            if (video.src.includes(".m3u8")) {
+              result.hlsUrl = video.src;
+            } else if (video.src.includes(".mp4")) {
+              result.progressiveUrl = video.src;
+            }
+          }
+        }
+
+        // Method 2: Check source elements
+        if (!result.hlsUrl && !result.progressiveUrl) {
+          const sources = document.querySelectorAll("video source");
+          result.debug.push(`Found ${sources.length} source elements`);
+          for (const source of Array.from(sources)) {
+            const src = (source as HTMLSourceElement).src;
+            if (src?.includes(".m3u8")) {
+              result.hlsUrl = src;
+              break;
+            } else if (src?.includes(".mp4") && !result.progressiveUrl) {
+              result.progressiveUrl = src;
+            }
+          }
+        }
+
+        // Method 3: Extract from Vimeo's internal player state
+        if (!result.hlsUrl && !result.progressiveUrl) {
+          try {
+            const win = window as unknown as VimeoWindow;
+
+            // Try various Vimeo internal variables
+            const configPaths = [
+              win.playerConfig?.request?.files,
+              win.vimeo?.config?.request?.files,
+              win.__vimeo_player__?.config?.request?.files,
+            ];
+
+            for (const files of configPaths) {
+              if (!files) continue;
+
+              // HLS
+              if (files.hls?.cdns) {
+                const cdns = files.hls.cdns;
+                for (const cdn of Object.keys(cdns)) {
+                  const cdnEntry = cdns[cdn];
+                  if (cdnEntry?.url) {
+                    result.hlsUrl = cdnEntry.url;
+                    result.debug.push(`Found HLS in playerConfig.${cdn}`);
+                    break;
+                  }
+                }
+              }
+
+              // Progressive MP4
+              if (!result.progressiveUrl && files.progressive && files.progressive.length > 0) {
+                const sorted = [...files.progressive].sort(
+                  (a, b) => (b.height ?? 0) - (a.height ?? 0)
+                );
+                result.progressiveUrl = sorted[0]?.url ?? null;
+                if (result.progressiveUrl) {
+                  result.debug.push("Found progressive in playerConfig");
+                }
+              }
+
+              if (result.hlsUrl || result.progressiveUrl) break;
+            }
+          } catch (e) {
+            result.debug.push(
+              `Config extraction error: ${e instanceof Error ? e.message : String(e)}`
+            );
+          }
+        }
+
+        // Method 4: Network request URLs might be in DOM attributes
+        if (!result.hlsUrl && !result.progressiveUrl) {
+          const allElements = document.querySelectorAll("*");
+          for (const el of Array.from(allElements)) {
+            for (const attr of Array.from(el.attributes)) {
+              if (attr.value.includes("vimeocdn.com") && attr.value.includes(".m3u8")) {
+                result.hlsUrl = /https:\/\/[^\s"']+\.m3u8[^\s"']*/.exec(attr.value)?.[0] ?? null;
+                if (result.hlsUrl) {
+                  result.debug.push("Found HLS in element attribute");
+                  break;
+                }
+              }
+            }
+            if (result.hlsUrl) break;
+          }
+        }
+
+        return result;
+      });
+
+      hlsUrl = urls.hlsUrl;
+      progressiveUrl = urls.progressiveUrl;
+
+      if (!hlsUrl && !progressiveUrl) {
+        // Wait and try again
+        await new Promise((r) => setTimeout(r, 1000));
+      }
+    }
+
+    if (hlsUrl || progressiveUrl) {
+      return { hlsUrl, progressiveUrl };
+    }
+
+    return {
+      hlsUrl: null,
+      progressiveUrl: null,
+      error: "Could not extract video URL from Vimeo player",
+    };
+  } catch (error) {
+    return {
+      hlsUrl: null,
+      progressiveUrl: null,
+      error: `Vimeo extraction failed: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}
+
+/**
+ * Captures Loom HLS URL by navigating directly to the embed page.
+ * This works better than CDP because we can intercept all requests on that page.
+ */
+export async function captureLoomHls(
+  page: Page,
+  videoId: string,
+  timeoutMs = 15000
+): Promise<{ hlsUrl: string | null; error?: string }> {
+  let capturedUrl: string | null = null;
+  const originalUrl = page.url();
+
+  try {
+    // Use CDP to intercept network responses
+    const client = await page.context().newCDPSession(page);
+    await client.send("Network.enable");
+
+    // Match HLS playlists from Loom's CDN
+    // Prefer master playlist (playlist.m3u8) over media playlists (mediaplaylist-*.m3u8)
+    const masterPattern = /luna\.loom\.com.*\/playlist\.m3u8/;
+    const anyHlsPattern = /luna\.loom\.com.*\.m3u8/;
+
+    // Set up listener before navigation
+    const responsePromise = new Promise<void>((resolve) => {
+      const timeout = setTimeout(() => {
+        resolve();
+      }, timeoutMs);
+      let hasMasterPlaylist = false;
+
+      client.on("Network.responseReceived", (event) => {
+        const url = event.response.url;
+
+        // Always prefer master playlist
+        if (masterPattern.test(url)) {
+          capturedUrl = url;
+          hasMasterPlaylist = true;
+          clearTimeout(timeout);
+          resolve();
+        } else if (!hasMasterPlaylist && anyHlsPattern.test(url)) {
+          // Capture any HLS as fallback, but keep listening for master
+          capturedUrl = url;
+        }
+      });
+    });
+
+    // Navigate directly to Loom embed with autoplay (muted)
+    const embedUrl = `https://www.loom.com/embed/${videoId}?autoplay=1`;
+    await page.goto(embedUrl, { waitUntil: "domcontentloaded", timeout: 10000 });
+
+    // Mute the video immediately
+    await page
+      .evaluate(() => {
+        const video = document.querySelector("video");
+        if (video) {
+          video.muted = true;
+          video.volume = 0;
+        }
+      })
+      .catch(() => {});
+
+    // Try to click play button if video doesn't autoplay
+    try {
+      await page.waitForTimeout(2000);
+      const playButton = await page.$(
+        '[data-testid="play-button"], .PlayButton, [aria-label="Play"], button[class*="play"]'
+      );
+      if (playButton) {
+        // Mute again before clicking play
+        await page
+          .evaluate(() => {
+            const video = document.querySelector("video");
+            if (video) {
+              video.muted = true;
+              video.volume = 0;
+            }
+          })
+          .catch(() => {});
+        await playButton.click();
+      }
+    } catch {
+      // No play button or click failed
+    }
+
+    // Ensure video stays muted after play
+    await page
+      .evaluate(() => {
+        const video = document.querySelector("video");
+        if (video) {
+          video.muted = true;
+          video.volume = 0;
+        }
+      })
+      .catch(() => {});
+
+    // Wait for HLS to be captured
+    await responsePromise;
+
+    // Also try to extract from page JS if not found via network
+    if (!capturedUrl) {
+      const jsUrl = await page.evaluate(() => {
+        const win = window as unknown as LoomWindow;
+
+        // Check __LOOM_SSR_STATE__
+        if (win.__LOOM_SSR_STATE__?.video?.asset_urls?.hls_url) {
+          return win.__LOOM_SSR_STATE__.video.asset_urls.hls_url;
+        }
+
+        // Check for Next.js data
+        const nextData = document.getElementById("__NEXT_DATA__");
+        if (nextData?.textContent) {
+          try {
+            const data = JSON.parse(nextData.textContent) as LoomNextData;
+            const hlsUrl = data?.props?.pageProps?.video?.asset_urls?.hls_url;
+            if (hlsUrl) return hlsUrl;
+
+            // Try regex match in full data
+            const videoData = /hls_url['":\s]+['"]([^'"]+)['"]/.exec(JSON.stringify(data));
+            if (videoData?.[1]) return videoData[1];
+          } catch {
+            /* ignore parse errors */
+          }
+        }
+
+        // Scan scripts for HLS URL
+        const scripts = Array.from(document.querySelectorAll("script"));
+        for (const script of scripts) {
+          const match = /https:\/\/luna\.loom\.com[^"'\s]+\.m3u8[^"'\s]*/.exec(
+            script.textContent ?? ""
+          );
+          if (match) return match[0];
+        }
+
+        return null;
+      });
+
+      if (jsUrl) {
+        capturedUrl = jsUrl;
+      }
+    }
+
+    await client.detach();
+  } catch {
+    // Error during capture
+  }
+
+  // Navigate back to original page
+  try {
+    await page.goto(originalUrl, { waitUntil: "domcontentloaded", timeout: 10000 });
+  } catch {
+    // Failed to navigate back
+  }
+
+  return capturedUrl ? { hlsUrl: capturedUrl } : { hlsUrl: null, error: "HLS URL not captured" };
+}

package/src/shared/auth.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, it } from "vitest";
+import { createLoginChecker, isSkoolLoginPage, isHighLevelLoginPage } from "./auth.js";
+
+describe("auth", () => {
+  describe("createLoginChecker", () => {
+    it("creates checker that matches patterns", () => {
+      const checker = createLoginChecker([/\/login/, /\/signin/]);
+
+      expect(checker("https://example.com/login")).toBe(true);
+      expect(checker("https://example.com/signin")).toBe(true);
+      expect(checker("https://example.com/dashboard")).toBe(false);
+    });
+
+    it("uses default patterns when none provided", () => {
+      const checker = createLoginChecker();
+
+      expect(checker("https://example.com/login")).toBe(true);
+      expect(checker("https://accounts.google.com/auth")).toBe(true);
+      expect(checker("https://example.com/dashboard")).toBe(false);
+    });
+  });
+
+  describe("isSkoolLoginPage", () => {
+    it("detects Skool login page", () => {
+      expect(isSkoolLoginPage("https://www.skool.com/login")).toBe(true);
+    });
+
+    it("detects Google OAuth redirect", () => {
+      expect(isSkoolLoginPage("https://accounts.google.com/o/oauth2/auth?...")).toBe(true);
+    });
+
+    it("returns false for non-login pages", () => {
+      expect(isSkoolLoginPage("https://www.skool.com/my-community")).toBe(false);
+      expect(isSkoolLoginPage("https://www.skool.com/my-community/classroom")).toBe(false);
+    });
+  });
+
+  describe("isHighLevelLoginPage", () => {
+    it("detects HighLevel SSO page", () => {
+      expect(isHighLevelLoginPage("https://sso.clientclub.net/login")).toBe(true);
+    });
+
+    it("detects various login paths", () => {
+      expect(isHighLevelLoginPage("https://portal.example.com/login")).toBe(true);
+      expect(isHighLevelLoginPage("https://portal.example.com/signin")).toBe(true);
+      expect(isHighLevelLoginPage("https://portal.example.com/auth/callback")).toBe(true);
+    });
+
+    it("detects Firebase auth", () => {
+      expect(isHighLevelLoginPage("https://example.firebaseapp.com/auth")).toBe(true);
+    });
+
+    it("returns false for content pages", () => {
+      expect(isHighLevelLoginPage("https://portal.example.com/courses")).toBe(false);
+      expect(isHighLevelLoginPage("https://portal.example.com/dashboard")).toBe(false);
+    });
+  });
+});