offcourse 1.0.0 → 1.1.0

package/src/scraper/highlevel/__snapshots__/extractor.test.ts.snap

@@ -1,41 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`parseHLSMasterPlaylist > with fixture file > snapshot: parsed playlist structure 1`] = `
-[
-  {
-    "bandwidth": 6000000,
-    "height": 1080,
-    "label": "1080p",
-    "url": "https://cdn.example.com/video/1080p/index.m3u8",
-    "width": 1920,
-  },
-  {
-    "bandwidth": 3200000,
-    "height": 720,
-    "label": "720p",
-    "url": "https://cdn.example.com/video/720p/index.m3u8",
-    "width": 1280,
-  },
-  {
-    "bandwidth": 1680000,
-    "height": 480,
-    "label": "480p",
-    "url": "https://cdn.example.com/video/480p/index.m3u8",
-    "width": 854,
-  },
-  {
-    "bandwidth": 796800,
-    "height": 360,
-    "label": "360p",
-    "url": "https://cdn.example.com/video/360p/index.m3u8",
-    "width": 640,
-  },
-  {
-    "bandwidth": 326400,
-    "height": 240,
-    "label": "240p",
-    "url": "https://cdn.example.com/video/240p/index.m3u8",
-    "width": 426,
-  },
-]
-`;

package/src/scraper/highlevel/extractor.test.ts

@@ -1,134 +0,0 @@
-import { readFileSync } from "node:fs";
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-import { describe, expect, it } from "vitest";
-import { parseHLSMasterPlaylist } from "./extractor.js";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const fixturesDir = join(__dirname, "../../__fixtures__");
-
-describe("parseHLSMasterPlaylist", () => {
-  const baseUrl = "https://cdn.example.com/video/";
-
-  it("parses master playlist with multiple qualities", () => {
-    const content = `#EXTM3U
-#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360
-360p.m3u8
-#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
-720p.m3u8
-#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=1920x1080
-1080p.m3u8`;
-
-    const result = parseHLSMasterPlaylist(content, baseUrl);
-
-    expect(result).toHaveLength(3);
-    // Should be sorted by bandwidth (highest first)
-    expect(result[0]!.label).toBe("1080p");
-    expect(result[0]!.bandwidth).toBe(5000000);
-    expect(result[0]!.height).toBe(1080);
-    expect(result[0]!.width).toBe(1920);
-  });
-
-  it("resolves relative URLs correctly", () => {
-    const content = `#EXTM3U
-#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
-720p/index.m3u8`;
-
-    const result = parseHLSMasterPlaylist(content, baseUrl);
-
-    expect(result[0]!.url).toBe("https://cdn.example.com/video/720p/index.m3u8");
-  });
-
-  it("preserves absolute URLs", () => {
-    const content = `#EXTM3U
-#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
-https://other-cdn.com/720p.m3u8`;
-
-    const result = parseHLSMasterPlaylist(content, baseUrl);
-
-    expect(result[0]!.url).toBe("https://other-cdn.com/720p.m3u8");
-  });
-
-  it("handles playlist without resolution (audio only)", () => {
-    const content = `#EXTM3U
-#EXT-X-STREAM-INF:BANDWIDTH=128000
-audio.m3u8`;
-
-    const result = parseHLSMasterPlaylist(content, baseUrl);
-
-    expect(result).toHaveLength(1);
-    expect(result[0]!.label).toBe("128k");
-    expect(result[0]!.height).toBeUndefined();
-  });
-
-  it("returns empty array for empty playlist", () => {
-    const content = `#EXTM3U
-#EXT-X-VERSION:3`;
-
-    const result = parseHLSMasterPlaylist(content, baseUrl);
-    expect(result).toHaveLength(0);
-  });
-
-  describe("with fixture file", () => {
-    it("parses real-world HLS master playlist", () => {
-      const content = readFileSync(join(fixturesDir, "hls-master-playlist.m3u8"), "utf-8");
-
-      const result = parseHLSMasterPlaylist(content, "https://cdn.example.com/video/");
-
-      expect(result).toHaveLength(5);
-
-      // Sorted by bandwidth (highest first)
-      expect(result[0]!.label).toBe("1080p");
-      expect(result[0]!.height).toBe(1080);
-      expect(result[0]!.bandwidth).toBe(6000000);
-
-      expect(result[4]!.label).toBe("240p");
-      expect(result[4]!.height).toBe(240);
-    });
-
-    it("snapshot: parsed playlist structure", () => {
-      const content = readFileSync(join(fixturesDir, "hls-master-playlist.m3u8"), "utf-8");
-      const result = parseHLSMasterPlaylist(content, "https://cdn.example.com/video/");
-
-      expect(result).toMatchSnapshot();
-    });
-  });
-});
-
-describe("HighLevel API Response parsing", () => {
-  it("snapshot: fixture structure", () => {
-    const response = JSON.parse(
-      readFileSync(join(fixturesDir, "highlevel-post-response.json"), "utf-8")
-    );
-
-    // Verify the structure we depend on
-    expect(response).toHaveProperty("id");
-    expect(response).toHaveProperty("title");
-    expect(response).toHaveProperty("video");
-    expect(response).toHaveProperty("post_materials");
-
-    // Video structure
-    expect(response.video).toHaveProperty("url");
-    expect(response.video).toHaveProperty("assetsLicenseId");
-    expect(response.video.url).toMatch(/\.mp4$/);
-
-    // Materials structure
-    expect(response.post_materials).toHaveLength(2);
-    expect(response.post_materials[0]).toHaveProperty("name");
-    expect(response.post_materials[0]).toHaveProperty("url");
-  });
-
-  it("extracts video info from response", () => {
-    const response = JSON.parse(
-      readFileSync(join(fixturesDir, "highlevel-post-response.json"), "utf-8")
-    );
-
-    // Simulate the extraction logic
-    const video = response.video;
-    const assetId = video?.assetsLicenseId ?? video?.assetId ?? video?.id;
-    const directUrl = video?.url;
-
-    expect(assetId).toBe("689a0d672ea246086539f453");
-    expect(directUrl).toContain("1080p.mp4");
-  });
-});

package/src/scraper/highlevel/extractor.ts

@@ -1,537 +0,0 @@
-import type { Page } from "playwright";
-import { parseHLSPlaylist } from "../../downloader/hlsDownloader.js";
-import {
-  FirebaseAuthTokenSchema,
-  PostDetailsResponseSchema,
-  VideoLicenseResponseSchema,
-  safeParse,
-  type FirebaseAuthRaw,
-} from "./schemas.js";
-
-// Alias for backwards compatibility and internal use
-const parseHLSMasterPlaylist = parseHLSPlaylist;
-
-export interface HighLevelVideoInfo {
-  type: "hls" | "vimeo" | "loom" | "youtube" | "custom";
-  url: string;
-  masterPlaylistUrl?: string;
-  qualities?: {
-    label: string;
-    url: string;
-    width?: number;
-    height?: number;
-  }[];
-  duration?: number;
-  thumbnailUrl?: string;
-  token?: string;
-}
-
-export interface HighLevelPostContent {
-  id: string;
-  title: string;
-  description: string | null;
-  htmlContent: string | null;
-  video: HighLevelVideoInfo | null;
-  attachments: {
-    id: string;
-    name: string;
-    url: string;
-    type: string;
-    size?: number;
-  }[];
-  categoryId: string;
-  productId: string;
-}
-
-// Browser/API automation - requires Playwright
-/* v8 ignore start */
-
-/**
- * Extracts the Firebase auth token from the page.
- */
-export async function getAuthToken(page: Page): Promise<string | null> {
-  const rawData = await page.evaluate((): FirebaseAuthRaw | null => {
-    const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
-    if (!tokenKey) return null;
-
-    try {
-      return JSON.parse(localStorage.getItem(tokenKey) ?? "{}") as FirebaseAuthRaw;
-    } catch {
-      return null;
-    }
-  });
-
-  if (!rawData) return null;
-
-  const parsed = safeParse(FirebaseAuthTokenSchema, rawData, "getAuthToken");
-  return parsed?.stsTokenManager.accessToken ?? null;
-}
-
-/**
- * Extracts video info from a HighLevel post page by intercepting network requests.
- */
-export async function extractVideoFromPage(page: Page): Promise<HighLevelVideoInfo | null> {
-  // First, check if there's an HLS video on the page
-  const hlsUrl = await page.evaluate(() => {
-    // Look for HLS master playlist URLs in the DOM
-    const videoElements = Array.from(document.querySelectorAll("video"));
-    for (const video of videoElements) {
-      const src = video.currentSrc ?? video.src;
-      if (src?.includes(".m3u8")) {
-        return src;
-      }
-    }
-
-    // Check for plyr or other players
-    const sources = Array.from(
-      document.querySelectorAll('source[type*="m3u8"], source[src*=".m3u8"]')
-    );
-    for (const source of sources) {
-      const src = (source as HTMLSourceElement).src;
-      if (src) return src;
-    }
-
-    return null;
-  });
-
-  if (hlsUrl) {
-    return {
-      type: "hls",
-      url: hlsUrl,
-      masterPlaylistUrl: hlsUrl,
-    };
-  }
-
-  // Check for Vimeo embed
-  const vimeoUrl = await page.evaluate(() => {
-    const iframe = document.querySelector('iframe[src*="vimeo.com"], iframe[src*="player.vimeo"]');
-    if (iframe) {
-      return (iframe as HTMLIFrameElement).src;
-    }
-    return null;
-  });
-
-  if (vimeoUrl) {
-    return {
-      type: "vimeo",
-      url: vimeoUrl,
-    };
-  }
-
-  // Check for Loom embed
-  const loomUrl = await page.evaluate(() => {
-    const iframe = document.querySelector('iframe[src*="loom.com"]');
-    if (iframe) {
-      return (iframe as HTMLIFrameElement).src;
-    }
-    return null;
-  });
-
-  if (loomUrl) {
-    return {
-      type: "loom",
-      url: loomUrl,
-    };
-  }
-
-  // Check for YouTube embed
-  const youtubeUrl = await page.evaluate(() => {
-    const iframe = document.querySelector(
-      'iframe[src*="youtube.com"], iframe[src*="youtube-nocookie.com"], iframe[src*="youtu.be"]'
-    );
-    if (iframe) {
-      return (iframe as HTMLIFrameElement).src;
-    }
-    return null;
-  });
-
-  if (youtubeUrl) {
-    return {
-      type: "youtube",
-      url: youtubeUrl,
-    };
-  }
-
-  return null;
-}
-
-/**
- * Extracts video info by intercepting network requests during page load.
- */
-export async function interceptVideoRequests(
-  page: Page,
-  postUrl: string
-): Promise<HighLevelVideoInfo | null> {
-  const hlsUrls: string[] = [];
-  const drmUrls: string[] = [];
-
-  // Set up request interception
-  const requestHandler = (request: { url: () => string }) => {
-    const url = request.url();
-
-    // Capture HLS master playlist requests
-    if (url.includes(".m3u8") || url.includes("master.m3u8")) {
-      hlsUrls.push(url);
-    }
-
-    // Capture DRM license requests
-    if (url.includes("assets-drm/assets-license")) {
-      drmUrls.push(url);
-    }
-  };
-
-  page.on("request", requestHandler);
-
-  // Navigate to the post page
-  await page.goto(postUrl, { timeout: 30000 });
-  await page.waitForLoadState("domcontentloaded");
-  await page.waitForTimeout(3000);
-
-  // Remove the handler
-  page.off("request", requestHandler);
-
-  // Get the HLS master playlist URL
-  const masterPlaylistUrl = hlsUrls.find((url) => url.includes("master.m3u8"));
-
-  if (masterPlaylistUrl) {
-    return {
-      type: "hls",
-      url: masterPlaylistUrl,
-      masterPlaylistUrl,
-    };
-  }
-
-  // Fallback to DOM extraction
-  return extractVideoFromPage(page);
-}
-
-/**
- * Fetches post details from the API.
- */
-export async function fetchPostDetails(
-  page: Page,
-  locationId: string,
-  postId: string
-): Promise<{
-  title: string;
-  description: string | null;
-  video: {
-    assetId: string;
-    url: string;
-  } | null;
-  materials: {
-    id: string;
-    name: string;
-    url: string;
-    type: string;
-  }[];
-} | null> {
-  // Fetch raw data from browser context
-  type FetchResult = { error: string; status?: number } | { data: unknown };
-
-  const rawData = await page.evaluate(
-    async ({ locationId, postId }): Promise<FetchResult> => {
-      try {
-        const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
-        const tokenData = tokenKey
-          ? (JSON.parse(localStorage.getItem(tokenKey) ?? "{}") as FirebaseAuthRaw)
-          : null;
-        const token = tokenData?.stsTokenManager?.accessToken;
-
-        if (!token) {
-          return { error: "No auth token" };
-        }
-
-        const res = await fetch(
-          `https://services.leadconnectorhq.com/membership/locations/${locationId}/posts/${postId}?source=courses`,
-          {
-            headers: {
-              Authorization: `Bearer ${token}`,
-            },
-          }
-        );
-
-        if (!res.ok) {
-          return { error: `HTTP ${res.status}`, status: res.status };
-        }
-
-        const data: unknown = await res.json();
-        return { data };
-      } catch (error) {
-        return { error: String(error) };
-      }
-    },
-    { locationId, postId }
-  );
-
-  // Debug: Log raw response in Node context
-  if ("error" in rawData) {
-    console.log(`[DEBUG] API Error: ${rawData.error}`);
-    return null;
-  }
-
-  const data = rawData.data;
-  if (!data) {
-    console.log("[DEBUG] No data in response");
-    return null;
-  }
-
-  // Validate response with Zod schema
-  const parsed = safeParse(PostDetailsResponseSchema, data, "fetchPostDetails");
-  if (!parsed) {
-    console.log("[DEBUG] Response validation failed");
-    return null;
-  }
-
-  // The API returns data directly (not nested under .post)
-  // Check both for backwards compatibility
-  const post = parsed.post ?? parsed;
-
-  let video: { assetId: string; url: string } | null = null;
-
-  // Check for video directly on post
-  // Video can have: id, assetId, assetsLicenseId, or direct url
-  if (post.video) {
-    const videoAssetId = post.video.assetsLicenseId ?? post.video.assetId ?? post.video.id;
-    if (videoAssetId || post.video.url) {
-      video = {
-        assetId: videoAssetId ?? "",
-        url: post.video.url ?? "",
-      };
-    }
-  }
-
-  // Check posterImage for video asset (older format)
-  if (!video && post.posterImage?.assetId) {
-    video = {
-      assetId: post.posterImage.assetId,
-      url: post.posterImage.url ?? "",
-    };
-  }
-
-  // Check for video in contentBlock
-  if (!video && post.contentBlock) {
-    for (const block of post.contentBlock) {
-      if (block.type === "video") {
-        const blockAssetId = block.assetsLicenseId ?? block.assetId ?? block.id;
-        if (blockAssetId || block.url) {
-          video = {
-            assetId: blockAssetId ?? "",
-            url: block.url ?? "",
-          };
-          break;
-        }
-      }
-    }
-  }
-
-  const materials: {
-    id: string;
-    name: string;
-    url: string;
-    type: string;
-  }[] = [];
-
-  // Materials can be under 'materials' or 'post_materials'
-  const materialsList = post.materials ?? post.post_materials ?? [];
-  for (const material of materialsList) {
-    materials.push({
-      id: material.id ?? crypto.randomUUID(),
-      name: material.name ?? "Attachment",
-      url: material.url ?? "",
-      type: material.type ?? "file",
-    });
-  }
-
-  return {
-    title: post.title ?? "",
-    description: post.description ?? null,
-    video,
-    materials,
-  };
-}
-
-/**
- * Fetches the DRM license (HLS token) for a video asset.
- */
-export async function fetchVideoLicense(
-  page: Page,
-  assetId: string
-): Promise<{ url: string; token: string } | null> {
-  // Get auth token first
-  const authToken = await getAuthToken(page);
-  if (!authToken) {
-    return null;
-  }
-
-  try {
-    // Use page.request to make the API call (bypasses CORS)
-    const response = await page.request.get(
-      `https://backend.leadconnectorhq.com/assets-drm/assets-license/${assetId}`,
-      {
-        headers: {
-          Authorization: `Bearer ${authToken}`,
-        },
-      }
-    );
-
-    if (!response.ok()) {
-      return null;
-    }
-
-    const data: unknown = await response.json();
-
-    // Validate response with Zod schema
-    const parsed = safeParse(VideoLicenseResponseSchema, data, "fetchVideoLicense");
-    if (!parsed) {
-      return null;
-    }
-
-    return {
-      url: parsed.url,
-      token: parsed.token,
-    };
-  } catch (error) {
-    console.error("Failed to fetch video license:", error);
-    return null;
-  }
-}
-
-/**
- * Extracts complete post content including video and attachments.
- */
-export async function extractHighLevelPostContent(
-  page: Page,
-  postUrl: string,
-  locationId: string,
-  productId: string,
-  postId: string,
-  categoryId: string
-): Promise<HighLevelPostContent | null> {
-  // Navigate to post page
-  await page.goto(postUrl, { timeout: 30000 });
-  await page.waitForLoadState("domcontentloaded");
-  await page.waitForTimeout(3000);
-
-  // Fetch post details from API
-  const postDetails = await fetchPostDetails(page, locationId, postId);
-
-  if (!postDetails) {
-    console.error("Could not fetch post details");
-    return null;
-  }
-
-  let video: HighLevelVideoInfo | null = null;
-
-  // Check if we have video data
-  if (postDetails.video) {
-    // Option 1: Direct MP4 URL (preferred - no DRM)
-    if (postDetails.video.url?.endsWith(".mp4")) {
-      video = {
-        type: "custom", // Direct download, not HLS
-        url: postDetails.video.url,
-      };
-    }
-    // Option 2: Get HLS license URL via assetId
-    else if (postDetails.video.assetId) {
-      const license = await fetchVideoLicense(page, postDetails.video.assetId);
-
-      if (license?.url) {
-        video = {
-          type: "hls",
-          url: license.url,
-          masterPlaylistUrl: license.url,
-          token: license.token,
-        };
-      }
-    }
-  }
-
-  // Fallback: try to extract video from page DOM
-  video ??= await extractVideoFromPage(page);
-
-  // Extract HTML content
-  const htmlContent = await page.evaluate(() => {
-    const contentEl = document.querySelector(
-      "[class*='post-content'], [class*='PostContent'], [class*='lesson-content'], article"
-    );
-    return contentEl?.innerHTML ?? null;
-  });
-
-  // Extract text description
-  const description = await page.evaluate(() => {
-    const descEl = document.querySelector(
-      "[class*='description'], [class*='Description'], p:first-of-type"
-    );
-    return descEl?.textContent?.trim() ?? null;
-  });
-
-  return {
-    id: postId,
-    title: postDetails.title,
-    description: description ?? postDetails.description,
-    htmlContent,
-    video,
-    attachments: postDetails.materials.map((m) => ({
-      id: m.id,
-      name: m.name,
-      url: m.url,
-      type: m.type,
-    })),
-    categoryId,
-    productId,
-  };
-}
-/* v8 ignore stop */
-
-// Re-export for backwards compatibility
-export { parseHLSMasterPlaylist };
-
-/* v8 ignore start */
-/**
- * Fetches and parses HLS playlist to get quality options.
- */
-export async function getHLSQualities(
-  page: Page,
-  masterPlaylistUrl: string
-): Promise<
-  {
-    label: string;
-    url: string;
-    bandwidth: number;
-    width?: number | undefined;
-    height?: number | undefined;
-  }[]
-> {
-  try {
-    const content = await page.evaluate(async (url) => {
-      const res = await fetch(url);
-      if (!res.ok) return null;
-      return res.text();
-    }, masterPlaylistUrl);
-
-    if (!content) return [];
-
-    return parseHLSMasterPlaylist(content, masterPlaylistUrl);
-  } catch {
-    return [];
-  }
-}
-
-/**
- * Gets the best quality URL from an HLS master playlist.
- */
-export async function getBestHLSQuality(
-  page: Page,
-  masterPlaylistUrl: string
-): Promise<string | null> {
-  const qualities = await getHLSQualities(page, masterPlaylistUrl);
-
-  if (qualities.length === 0) {
-    return masterPlaylistUrl;
-  }
-
-  // Return highest quality
-  return qualities[0]?.url ?? null;
-}
-/* v8 ignore stop */

package/src/scraper/highlevel/index.ts

@@ -1,2 +0,0 @@
-export * from "./navigator.js";
-export * from "./extractor.js";