offcourse 0.0.2 → 1.0.0

package/dist/scraper/highlevel/auth.js

@@ -0,0 +1,189 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { chromium } from "playwright";
+import { getSessionPath, SESSIONS_DIR } from "../../config/paths.js";
+/**
+ * Checks if a valid HighLevel session exists for the given domain.
+ */
+export function hasValidHighLevelSession(domain) {
+    const sessionPath = getSessionPath(domain);
+    return existsSync(sessionPath);
+}
+/**
+ * Loads an existing session from disk.
+ */
+async function loadSession(browser, domain) {
+    const sessionPath = getSessionPath(domain);
+    const storageState = JSON.parse(readFileSync(sessionPath, "utf-8"));
+    return browser.newContext({ storageState });
+}
+/**
+ * Saves the current session to disk.
+ */
+async function saveSession(context, domain) {
+    const sessionPath = getSessionPath(domain);
+    const dir = dirname(sessionPath);
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    const storageState = await context.storageState();
+    writeFileSync(sessionPath, JSON.stringify(storageState, null, 2), "utf-8");
+}
+/**
+ * Checks if the current page is on a HighLevel login page.
+ */
+function isHighLevelLoginPage(url) {
+    const loginPatterns = [
+        /sso\.clientclub\.net/,
+        /\/login/,
+        /\/signin/,
+        /\/auth/,
+        /accounts\.google\.com/,
+        /firebaseapp\.com/,
+    ];
+    return loginPatterns.some((p) => p.test(url));
+}
+/**
+ * Checks if the page has a valid Firebase auth token.
+ */
+async function hasValidFirebaseToken(page) {
+    try {
+        const hasToken = await page.evaluate(() => {
+            const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
+            if (!tokenKey)
+                return false;
+            const tokenData = JSON.parse(localStorage.getItem(tokenKey) ?? "{}");
+            const expirationTime = tokenData?.stsTokenManager?.expirationTime;
+            // Check if token exists and is not expired
+            if (expirationTime) {
+                return Date.now() < expirationTime;
+            }
+            return !!tokenData?.stsTokenManager?.accessToken;
+        });
+        return hasToken;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Performs interactive login for HighLevel by opening a browser window.
+ * The user logs in manually, and we capture the session.
+ */
+export async function performHighLevelInteractiveLogin(domain, portalUrl) {
+    // Ensure sessions directory exists
+    if (!existsSync(SESSIONS_DIR)) {
+        mkdirSync(SESSIONS_DIR, { recursive: true });
+    }
+    const browser = await chromium.launch({
+        headless: false, // Must be visible for user interaction
+    });
+    const context = await browser.newContext({
+        viewport: { width: 1280, height: 800 },
+    });
+    const page = await context.newPage();
+    await page.goto(portalUrl);
+    console.log("\n🔐 Browser opened. Please log in manually.");
+    console.log("   The window will close automatically after successful login.\n");
+    // Wait for either:
+    // 1. Navigation away from login page
+    // 2. Firebase token to appear in localStorage
+    let loggedIn = false;
+    const startTime = Date.now();
+    const timeout = 300000; // 5 minutes
+    while (!loggedIn && Date.now() - startTime < timeout) {
+        await page.waitForTimeout(1000);
+        const currentUrl = page.url();
+        // Check if we're still on a login page
+        if (!isHighLevelLoginPage(currentUrl)) {
+            // Might be logged in, check for Firebase token
+            const hasToken = await hasValidFirebaseToken(page);
+            if (hasToken) {
+                loggedIn = true;
+                break;
+            }
+            // Also check if we're on a course page (successful login)
+            if (currentUrl.includes("/courses/") || currentUrl.includes("/library")) {
+                loggedIn = true;
+                break;
+            }
+        }
+    }
+    if (!loggedIn) {
+        await browser.close();
+        throw new Error("Login timed out after 5 minutes");
+    }
+    // Give the page a moment to fully load after login
+    await page.waitForLoadState("networkidle").catch(() => { });
+    await page.waitForTimeout(2000);
+    // Save the session
+    await saveSession(context, domain);
+    console.log("✅ Login successful! Session saved.\n");
+    return { context, page };
+}
+/**
+ * Gets an authenticated HighLevel session, either from cache or via interactive login.
+ */
+export async function getHighLevelAuthenticatedSession(domain, portalUrl, options = {}) {
+    // Default to headless mode (true) unless explicitly set to false
+    const useHeadless = options.headless !== false;
+    const browser = await chromium.launch({
+        headless: useHeadless,
+    });
+    // Try to use existing session
+    if (!options.forceLogin && hasValidHighLevelSession(domain)) {
+        try {
+            const context = await loadSession(browser, domain);
+            const page = await context.newPage();
+            // Navigate to portal
+            await page.goto(portalUrl);
+            await page.waitForLoadState("domcontentloaded");
+            await page.waitForTimeout(2000);
+            const currentUrl = page.url();
+            // Check if we got redirected to login or SSO
+            if (isHighLevelLoginPage(currentUrl)) {
+                console.log("⚠️  Session expired, need to re-login...");
+                await context.close();
+                await browser.close();
+            }
+            else {
+                // Verify we have a valid Firebase token
+                const hasToken = await hasValidFirebaseToken(page);
+                if (hasToken) {
+                    console.log("✅ Using cached session");
+                    return { browser, session: { context, page } };
+                }
+                else {
+                    console.log("⚠️  No valid auth token, need to re-login...");
+                    await context.close();
+                    await browser.close();
+                }
+            }
+        }
+        catch (error) {
+            console.log("⚠️  Failed to load session, need to re-login...", error);
+            await browser.close();
+        }
+    }
+    else {
+        await browser.close();
+    }
+    // Need fresh login - always visible for interactive login
+    const session = await performHighLevelInteractiveLogin(domain, portalUrl);
+    // Get the browser from the session context
+    const sessionBrowser = session.context.browser();
+    if (!sessionBrowser) {
+        throw new Error("Failed to get browser from session");
+    }
+    // After login, reopen with headless browser (unless explicitly set to false)
+    if (useHeadless) {
+        const newBrowser = await chromium.launch({ headless: true });
+        const context = await loadSession(newBrowser, domain);
+        const page = await context.newPage();
+        // Close the interactive session
+        await sessionBrowser.close();
+        return { browser: newBrowser, session: { context, page } };
+    }
+    return { browser: sessionBrowser, session };
+}
+//# sourceMappingURL=auth.js.map

package/dist/scraper/highlevel/auth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/auth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAOrE;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,MAAc;IACrD,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,OAAgB,EAAE,MAAc;IACzD,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACpE,OAAO,OAAO,CAAC,UAAU,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,OAAuB,EAAE,MAAc;IAChE,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEjC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAClD,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AAC7E,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,GAAW;IACvC,MAAM,aAAa,GAAG;QACpB,sBAAsB;QACtB,SAAS;QACT,UAAU;QACV,QAAQ;QACR,uBAAuB;QACvB,kBAAkB;KACnB,CAAC;IACF,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,qBAAqB,CAAC,IAAU;IAC7C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;YACxF,IAAI,CAAC,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;YACrE,MAAM,cAAc,GAAG,SAAS,EAAE,eAAe,EAAE,cAAc,CAAC;YAElE,2CAA2C;YAC3C,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;YACrC,CAAC;YAED,OAAO,CAAC,CAAC,SAAS,EAAE,eAAe,EAAE,WAAW,CAAC;QACnD,CAAC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,MAAc,EACd,SAAiB;IAEjB,mCAAmC;IACnC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACpC,QAAQ,EAAE,KAAK,EAAE,uCAAuC;KACzD,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;QACvC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;KACvC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAE3B,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;IAEhF,mBAAmB;IACnB,qCAAqC;IACrC,8CAA8C;IAC9C,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,YAAY;IAEpC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,OAAO,EAAE,CAAC;QACrD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,uCAAuC;QACvC,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAC;YACtC,+CAA+C;YAC/C,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM;YACR,CAAC;YAED,0DAA0D;YAC1D,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACxE,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,mDAAmD;IACnD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAEhC,mBAAmB;IACnB,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IAEpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,MAAc,EACd,SAAiB,EACjB,UAAwD,EAAE;IAE1D,iEAAiE;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC;IAE/C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACpC,QAAQ,EAAE,WAAW;KACtB,CAAC,CAAC;IAEH,8BAA8B;IAC9B,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,wBAAwB,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAErC,qBAAqB;YACrB,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC3B,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;YAChD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAEhC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE9B,6CAA6C;YAC7C,IAAI,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;gBACxD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,wCAAwC;gBACxC,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;gBACnD,IAAI,QAAQ,EAAE,CAAC;oBACb,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;oBACtC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;oBAC5D,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;oBACtB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;YACtE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,0DAA0D;IAC1D,MAAM,OAAO,GAAG,MAAM,gCAAgC,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAE1E,2CAA2C;IAC3C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACjD,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IAED,6EAA6E;IAC7E,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,gCAAgC;QAChC,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;IAC7D,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC"}

package/dist/scraper/highlevel/extractor.d.ts

@@ -0,0 +1,97 @@
+import type { Page } from "playwright";
+export interface HighLevelVideoInfo {
+    type: "hls" | "vimeo" | "loom" | "youtube" | "custom";
+    url: string;
+    masterPlaylistUrl?: string;
+    qualities?: Array<{
+        label: string;
+        url: string;
+        width?: number;
+        height?: number;
+    }>;
+    duration?: number;
+    thumbnailUrl?: string;
+    token?: string;
+}
+export interface HighLevelPostContent {
+    id: string;
+    title: string;
+    description: string | null;
+    htmlContent: string | null;
+    video: HighLevelVideoInfo | null;
+    attachments: Array<{
+        id: string;
+        name: string;
+        url: string;
+        type: string;
+        size?: number;
+    }>;
+    categoryId: string;
+    productId: string;
+}
+/**
+ * Extracts the Firebase auth token from the page.
+ */
+export declare function getAuthToken(page: Page): Promise<string | null>;
+/**
+ * Extracts video info from a HighLevel post page by intercepting network requests.
+ */
+export declare function extractVideoFromPage(page: Page): Promise<HighLevelVideoInfo | null>;
+/**
+ * Extracts video info by intercepting network requests during page load.
+ */
+export declare function interceptVideoRequests(page: Page, postUrl: string): Promise<HighLevelVideoInfo | null>;
+/**
+ * Fetches post details from the API.
+ */
+export declare function fetchPostDetails(page: Page, locationId: string, postId: string): Promise<{
+    title: string;
+    description: string | null;
+    video: {
+        assetId: string;
+        url: string;
+    } | null;
+    materials: Array<{
+        id: string;
+        name: string;
+        url: string;
+        type: string;
+    }>;
+} | null>;
+/**
+ * Fetches the DRM license (HLS token) for a video asset.
+ */
+export declare function fetchVideoLicense(page: Page, assetId: string): Promise<{
+    url: string;
+    token: string;
+} | null>;
+/**
+ * Extracts complete post content including video and attachments.
+ */
+export declare function extractHighLevelPostContent(page: Page, postUrl: string, locationId: string, productId: string, postId: string, categoryId: string): Promise<HighLevelPostContent | null>;
+/**
+ * Parses an HLS master playlist to extract quality variants.
+ * Uses hls-parser for robust parsing.
+ */
+export declare function parseHLSMasterPlaylist(content: string, baseUrl: string): Array<{
+    label: string;
+    url: string;
+    bandwidth: number;
+    width?: number | undefined;
+    height?: number | undefined;
+}>;
+/**
+ * Fetches and parses HLS playlist to get quality options.
+ */
+export declare function getHLSQualities(page: Page, masterPlaylistUrl: string): Promise<Array<{
+    label: string;
+    url: string;
+    bandwidth: number;
+    width?: number | undefined;
+    height?: number | undefined;
+}>>;
+/**
+ * Gets the best quality URL from an HLS master playlist.
+ */
+export declare function getBestHLSQuality(page: Page, masterPlaylistUrl: string): Promise<string | null>;
+//# sourceMappingURL=extractor.d.ts.map

package/dist/scraper/highlevel/extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;IACtD,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACjC,WAAW,EAAE,KAAK,CAAC;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAQrE;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAmFzF;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA0CpC;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IACT,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,GAAG,EAAE,MAAM,CAAC;KACb,GAAG,IAAI,CAAC;IACT,SAAS,EAAE,KAAK,CAAC;QACf,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;CACJ,GAAG,IAAI,CAAC,CAmHR;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAmChD;AAED;;GAEG;AACH,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CA4EtC;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,GACd,KAAK,CAAC;IACP,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B,CAAC,CAsCD;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,IAAI,EACV,iBAAiB,EAAE,MAAM,GACxB,OAAO,CACR,KAAK,CAAC;IACJ,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B,CAAC,CACH,CAcA;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CASxB"}

package/dist/scraper/highlevel/extractor.js

@@ -0,0 +1,386 @@
+import * as HLS from "hls-parser";
+/**
+ * Extracts the Firebase auth token from the page.
+ */
+export async function getAuthToken(page) {
+    return page.evaluate(() => {
+        const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
+        if (!tokenKey)
+            return null;
+        const tokenData = JSON.parse(localStorage.getItem(tokenKey) ?? "{}");
+        return tokenData?.stsTokenManager?.accessToken ?? null;
+    });
+}
+/**
+ * Extracts video info from a HighLevel post page by intercepting network requests.
+ */
+export async function extractVideoFromPage(page) {
+    // First, check if there's an HLS video on the page
+    const hlsUrl = await page.evaluate(() => {
+        // Look for HLS master playlist URLs in the DOM
+        const videoElements = Array.from(document.querySelectorAll("video"));
+        for (const video of videoElements) {
+            const src = video.currentSrc || video.src;
+            if (src && src.includes(".m3u8")) {
+                return src;
+            }
+        }
+        // Check for plyr or other players
+        const sources = Array.from(document.querySelectorAll('source[type*="m3u8"], source[src*=".m3u8"]'));
+        for (const source of sources) {
+            const src = source.src;
+            if (src)
+                return src;
+        }
+        return null;
+    });
+    if (hlsUrl) {
+        return {
+            type: "hls",
+            url: hlsUrl,
+            masterPlaylistUrl: hlsUrl,
+        };
+    }
+    // Check for Vimeo embed
+    const vimeoUrl = await page.evaluate(() => {
+        const iframe = document.querySelector('iframe[src*="vimeo.com"], iframe[src*="player.vimeo"]');
+        if (iframe) {
+            return iframe.src;
+        }
+        return null;
+    });
+    if (vimeoUrl) {
+        return {
+            type: "vimeo",
+            url: vimeoUrl,
+        };
+    }
+    // Check for Loom embed
+    const loomUrl = await page.evaluate(() => {
+        const iframe = document.querySelector('iframe[src*="loom.com"]');
+        if (iframe) {
+            return iframe.src;
+        }
+        return null;
+    });
+    if (loomUrl) {
+        return {
+            type: "loom",
+            url: loomUrl,
+        };
+    }
+    // Check for YouTube embed
+    const youtubeUrl = await page.evaluate(() => {
+        const iframe = document.querySelector('iframe[src*="youtube.com"], iframe[src*="youtube-nocookie.com"], iframe[src*="youtu.be"]');
+        if (iframe) {
+            return iframe.src;
+        }
+        return null;
+    });
+    if (youtubeUrl) {
+        return {
+            type: "youtube",
+            url: youtubeUrl,
+        };
+    }
+    return null;
+}
+/**
+ * Extracts video info by intercepting network requests during page load.
+ */
+export async function interceptVideoRequests(page, postUrl) {
+    const hlsUrls = [];
+    const drmUrls = [];
+    // Set up request interception
+    const requestHandler = (request) => {
+        const url = request.url();
+        // Capture HLS master playlist requests
+        if (url.includes(".m3u8") || url.includes("master.m3u8")) {
+            hlsUrls.push(url);
+        }
+        // Capture DRM license requests
+        if (url.includes("assets-drm/assets-license")) {
+            drmUrls.push(url);
+        }
+    };
+    page.on("request", requestHandler);
+    // Navigate to the post page
+    await page.goto(postUrl, { timeout: 30000 });
+    await page.waitForLoadState("domcontentloaded");
+    await page.waitForTimeout(3000);
+    // Remove the handler
+    page.off("request", requestHandler);
+    // Get the HLS master playlist URL
+    const masterPlaylistUrl = hlsUrls.find((url) => url.includes("master.m3u8"));
+    if (masterPlaylistUrl) {
+        return {
+            type: "hls",
+            url: masterPlaylistUrl,
+            masterPlaylistUrl,
+        };
+    }
+    // Fallback to DOM extraction
+    return extractVideoFromPage(page);
+}
+/**
+ * Fetches post details from the API.
+ */
+export async function fetchPostDetails(page, locationId, postId) {
+    // Fetch raw data from browser context
+    const rawData = await page.evaluate(async ({ locationId, postId }) => {
+        try {
+            const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
+            const tokenData = tokenKey ? JSON.parse(localStorage.getItem(tokenKey) ?? "{}") : null;
+            const token = tokenData?.stsTokenManager?.accessToken;
+            if (!token) {
+                return { error: "No auth token" };
+            }
+            const res = await fetch(`https://services.leadconnectorhq.com/membership/locations/${locationId}/posts/${postId}?source=courses`, {
+                headers: {
+                    Authorization: `Bearer ${token}`,
+                },
+            });
+            if (!res.ok) {
+                return { error: `HTTP ${res.status}`, status: res.status };
+            }
+            const data = await res.json();
+            return { data };
+        }
+        catch (error) {
+            return { error: String(error) };
+        }
+    }, { locationId, postId });
+    // Debug: Log raw response in Node context
+    if (rawData?.error) {
+        console.log(`[DEBUG] API Error: ${rawData.error}`);
+        return null;
+    }
+    const data = rawData?.data;
+    if (!data) {
+        console.log("[DEBUG] No data in response");
+        return null;
+    }
+    // The API returns data directly (not nested under .post)
+    // Check both for backwards compatibility
+    const post = data.post ?? data;
+    let video = null;
+    // Check for video directly on post
+    // Video can have: id, assetId, assetsLicenseId, or direct url
+    if (post.video) {
+        const videoAssetId = post.video.assetsLicenseId ?? post.video.assetId ?? post.video.id;
+        if (videoAssetId || post.video.url) {
+            video = {
+                assetId: videoAssetId ?? "",
+                url: post.video.url ?? "",
+            };
+        }
+    }
+    // Check posterImage for video asset (older format)
+    if (!video && post.posterImage?.assetId) {
+        video = {
+            assetId: post.posterImage.assetId,
+            url: post.posterImage.url ?? "",
+        };
+    }
+    // Check for video in contentBlock
+    if (!video && post.contentBlock) {
+        for (const block of post.contentBlock) {
+            if (block.type === "video") {
+                const blockAssetId = block.assetsLicenseId ?? block.assetId ?? block.id;
+                if (blockAssetId || block.url) {
+                    video = {
+                        assetId: blockAssetId ?? "",
+                        url: block.url ?? "",
+                    };
+                    break;
+                }
+            }
+        }
+    }
+    const materials = [];
+    // Materials can be under 'materials' or 'post_materials'
+    const materialsList = post.materials ?? post.post_materials ?? [];
+    if (Array.isArray(materialsList)) {
+        for (const material of materialsList) {
+            materials.push({
+                id: material.id ?? crypto.randomUUID(),
+                name: material.name ?? "Attachment",
+                url: material.url ?? "",
+                type: material.type ?? "file",
+            });
+        }
+    }
+    return {
+        title: post.title ?? "",
+        description: post.description ?? null,
+        video,
+        materials,
+    };
+}
+/**
+ * Fetches the DRM license (HLS token) for a video asset.
+ */
+export async function fetchVideoLicense(page, assetId) {
+    return page.evaluate(async (assetId) => {
+        try {
+            const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
+            const tokenData = tokenKey ? JSON.parse(localStorage.getItem(tokenKey) ?? "{}") : null;
+            const token = tokenData?.stsTokenManager?.accessToken;
+            if (!token) {
+                return null;
+            }
+            const res = await fetch(`https://backend.leadconnectorhq.com/assets-drm/assets-license/${assetId}`, {
+                headers: {
+                    Authorization: `Bearer ${token}`,
+                },
+            });
+            if (!res.ok) {
+                return null;
+            }
+            const data = await res.json();
+            return {
+                url: data.url ?? "",
+                token: data.token ?? "",
+            };
+        }
+        catch (error) {
+            console.error("Failed to fetch video license:", error);
+            return null;
+        }
+    }, assetId);
+}
+/**
+ * Extracts complete post content including video and attachments.
+ */
+export async function extractHighLevelPostContent(page, postUrl, locationId, productId, postId, categoryId) {
+    // Navigate to post page
+    await page.goto(postUrl, { timeout: 30000 });
+    await page.waitForLoadState("domcontentloaded");
+    await page.waitForTimeout(3000);
+    // Fetch post details from API
+    const postDetails = await fetchPostDetails(page, locationId, postId);
+    if (!postDetails) {
+        console.error("Could not fetch post details");
+        return null;
+    }
+    let video = null;
+    // Check if we have video data
+    if (postDetails.video) {
+        // Option 1: Direct MP4 URL (preferred - no DRM)
+        if (postDetails.video.url && postDetails.video.url.endsWith(".mp4")) {
+            video = {
+                type: "custom", // Direct download, not HLS
+                url: postDetails.video.url,
+            };
+        }
+        // Option 2: Get HLS license URL via assetId
+        else if (postDetails.video.assetId) {
+            const license = await fetchVideoLicense(page, postDetails.video.assetId);
+            if (license?.url) {
+                video = {
+                    type: "hls",
+                    url: license.url,
+                    masterPlaylistUrl: license.url,
+                    token: license.token,
+                };
+            }
+        }
+    }
+    // Fallback: try to extract video from page DOM
+    if (!video) {
+        video = await extractVideoFromPage(page);
+    }
+    // Extract HTML content
+    const htmlContent = await page.evaluate(() => {
+        const contentEl = document.querySelector("[class*='post-content'], [class*='PostContent'], [class*='lesson-content'], article");
+        return contentEl?.innerHTML ?? null;
+    });
+    // Extract text description
+    const description = await page.evaluate(() => {
+        const descEl = document.querySelector("[class*='description'], [class*='Description'], p:first-of-type");
+        return descEl?.textContent?.trim() ?? null;
+    });
+    return {
+        id: postId,
+        title: postDetails.title,
+        description: description ?? postDetails.description,
+        htmlContent,
+        video,
+        attachments: postDetails.materials.map((m) => ({
+            id: m.id,
+            name: m.name,
+            url: m.url,
+            type: m.type,
+        })),
+        categoryId,
+        productId,
+    };
+}
+/**
+ * Parses an HLS master playlist to extract quality variants.
+ * Uses hls-parser for robust parsing.
+ */
+export function parseHLSMasterPlaylist(content, baseUrl) {
+    try {
+        const playlist = HLS.parse(content);
+        // Check if it's a master playlist with variants
+        if (!("variants" in playlist) || !playlist.variants) {
+            return [];
+        }
+        const variants = playlist.variants.map((variant) => {
+            const bandwidth = variant.bandwidth ?? 0;
+            const resolution = variant.resolution;
+            const width = resolution?.width;
+            const height = resolution?.height;
+            // Build absolute URL
+            const variantUrl = variant.uri.startsWith("http")
+                ? variant.uri
+                : new URL(variant.uri, baseUrl).href;
+            const label = height ? `${height}p` : `${Math.round(bandwidth / 1000)}k`;
+            return {
+                label,
+                url: variantUrl,
+                bandwidth,
+                width,
+                height,
+            };
+        });
+        // Sort by bandwidth (highest first)
+        variants.sort((a, b) => b.bandwidth - a.bandwidth);
+        return variants;
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Fetches and parses HLS playlist to get quality options.
+ */
+export async function getHLSQualities(page, masterPlaylistUrl) {
+    try {
+        const content = await page.evaluate(async (url) => {
+            const res = await fetch(url);
+            if (!res.ok)
+                return null;
+            return res.text();
+        }, masterPlaylistUrl);
+        if (!content)
+            return [];
+        return parseHLSMasterPlaylist(content, masterPlaylistUrl);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Gets the best quality URL from an HLS master playlist.
+ */
+export async function getBestHLSQuality(page, masterPlaylistUrl) {
+    const qualities = await getHLSQualities(page, masterPlaylistUrl);
+    if (qualities.length === 0) {
+        return masterPlaylistUrl;
+    }
+    // Return highest quality
+    return qualities[0]?.url ?? null;
+}
+//# sourceMappingURL=extractor.js.map