offcourse 0.0.2 → 1.0.0

package/dist/scraper/auth.d.ts

@@ -0,0 +1,29 @@
+import { Browser, BrowserContext, Page } from "playwright";
+export interface AuthSession {
+    context: BrowserContext;
+    page: Page;
+}
+/**
+ * Checks if a valid session exists for the given domain.
+ */
+export declare function hasValidSession(domain: string): boolean;
+/**
+ * Performs interactive login by opening a browser window.
+ * The user logs in manually, and we capture the session.
+ */
+export declare function performInteractiveLogin(domain: string, loginUrl: string): Promise<AuthSession>;
+/**
+ * Gets an authenticated session, either from cache or via interactive login.
+ */
+export declare function getAuthenticatedSession(domain: string, loginUrl: string, options?: {
+    forceLogin?: boolean;
+    headless?: boolean;
+}): Promise<{
+    browser: Browser;
+    session: AuthSession;
+}>;
+/**
+ * Clears the session for a domain.
+ */
+export declare function clearSession(domain: string): boolean;
+//# sourceMappingURL=auth.d.ts.map

package/dist/scraper/auth.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../../src/scraper/auth.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAY,IAAI,EAAE,MAAM,YAAY,CAAC;AAGrE,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,cAAc,CAAC;IACxB,IAAI,EAAE,IAAI,CAAC;CACZ;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAGvD;AA0BD;;;GAGG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,WAAW,CAAC,CAkCtB;AAED;;GAEG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAO,GACzD,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,WAAW,CAAA;CAAE,CAAC,CA2CrD;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAOpD"}

package/dist/scraper/auth.js

@@ -0,0 +1,115 @@
+import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { chromium } from "playwright";
+import { getSessionPath, SESSIONS_DIR } from "../config/paths.js";
+/**
+ * Checks if a valid session exists for the given domain.
+ */
+export function hasValidSession(domain) {
+    const sessionPath = getSessionPath(domain);
+    return existsSync(sessionPath);
+}
+/**
+ * Loads an existing session from disk.
+ */
+async function loadSession(browser, domain) {
+    const sessionPath = getSessionPath(domain);
+    const storageState = JSON.parse(readFileSync(sessionPath, "utf-8"));
+    return browser.newContext({ storageState });
+}
+/**
+ * Saves the current session to disk.
+ */
+async function saveSession(context, domain) {
+    const sessionPath = getSessionPath(domain);
+    const dir = dirname(sessionPath);
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    const storageState = await context.storageState();
+    writeFileSync(sessionPath, JSON.stringify(storageState, null, 2), "utf-8");
+}
+/**
+ * Performs interactive login by opening a browser window.
+ * The user logs in manually, and we capture the session.
+ */
+export async function performInteractiveLogin(domain, loginUrl) {
+    // Ensure sessions directory exists
+    if (!existsSync(SESSIONS_DIR)) {
+        mkdirSync(SESSIONS_DIR, { recursive: true });
+    }
+    const browser = await chromium.launch({
+        headless: false, // Must be visible for user interaction
+    });
+    const context = await browser.newContext({
+        viewport: { width: 1280, height: 800 },
+    });
+    const page = await context.newPage();
+    await page.goto(loginUrl);
+    console.log("\n🔐 Browser opened. Please log in manually.");
+    console.log("   The window will close automatically after successful login.\n");
+    // Wait for navigation away from login page (indicates successful login)
+    await page.waitForURL((url) => !url.pathname.includes("/login"), {
+        timeout: 300000, // 5 minutes to complete login
+    });
+    // Give the page a moment to fully load after login
+    await page.waitForLoadState("networkidle");
+    // Save the session
+    await saveSession(context, domain);
+    console.log("✅ Login successful! Session saved.\n");
+    return { context, page };
+}
+/**
+ * Gets an authenticated session, either from cache or via interactive login.
+ */
+export async function getAuthenticatedSession(domain, loginUrl, options = {}) {
+    const browser = await chromium.launch({
+        headless: options.headless ?? false,
+    });
+    // Try to use existing session
+    if (!options.forceLogin && hasValidSession(domain)) {
+        try {
+            const context = await loadSession(browser, domain);
+            const page = await context.newPage();
+            // Verify session is still valid by navigating
+            await page.goto(`https://${domain}`);
+            // Check if we got redirected to login
+            const currentUrl = page.url();
+            if (currentUrl.includes("/login")) {
+                console.log("⚠️  Session expired, need to re-login...");
+                await context.close();
+            }
+            else {
+                console.log("✅ Using cached session");
+                return { browser, session: { context, page } };
+            }
+        }
+        catch (error) {
+            console.log("⚠️  Failed to load session, need to re-login...", error);
+        }
+    }
+    // Need fresh login
+    await browser.close();
+    const session = await performInteractiveLogin(domain, loginUrl);
+    // Reopen browser with saved session for headless operation if needed
+    const newBrowser = await chromium.launch({
+        headless: options.headless ?? false,
+    });
+    const context = await loadSession(newBrowser, domain);
+    const page = await context.newPage();
+    // Close the interactive session
+    await session.context.browser()?.close();
+    return { browser: newBrowser, session: { context, page } };
+}
+/**
+ * Clears the session for a domain.
+ */
+export function clearSession(domain) {
+    const sessionPath = getSessionPath(domain);
+    if (existsSync(sessionPath)) {
+        unlinkSync(sessionPath);
+        return true;
+    }
+    return false;
+}
+//# sourceMappingURL=auth.js.map

package/dist/scraper/auth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../src/scraper/auth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACzF,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAA2B,QAAQ,EAAQ,MAAM,YAAY,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAOlE;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,OAAgB,EAAE,MAAc;IACzD,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACpE,OAAO,OAAO,CAAC,UAAU,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,OAAuB,EAAE,MAAc;IAChE,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEjC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAClD,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AAC7E,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,MAAc,EACd,QAAgB;IAEhB,mCAAmC;IACnC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACpC,QAAQ,EAAE,KAAK,EAAE,uCAAuC;KACzD,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;QACvC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;KACvC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE1B,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;IAEhF,wEAAwE;IACxE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAC/D,OAAO,EAAE,MAAM,EAAE,8BAA8B;KAChD,CAAC,CAAC;IAEH,mDAAmD;IACnD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAE3C,mBAAmB;IACnB,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IAEpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,MAAc,EACd,QAAgB,EAChB,UAAwD,EAAE;IAE1D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACpC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,KAAK;KACpC,CAAC,CAAC;IAEH,8BAA8B;IAC9B,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAErC,8CAA8C;YAC9C,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;YAErC,sCAAsC;YACtC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC9B,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;gBACxD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;gBACtC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;YACjD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACtB,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,qEAAqE;IACrE,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACvC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,KAAK;KACpC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAErC,gCAAgC;IAChC,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC;IAEzC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;AAC7D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5B,UAAU,CAAC,WAAW,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/scraper/extractor.d.ts

@@ -0,0 +1,49 @@
+import type { Page } from "playwright";
+export interface DownloadableFile {
+    url: string;
+    filename: string;
+    type: "pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx" | "zip" | "other";
+}
+export interface LessonContent {
+    title: string;
+    videoUrl: string | null;
+    videoType: "loom" | "vimeo" | "youtube" | "wistia" | "native" | "unknown" | null;
+    htmlContent: string;
+    markdownContent: string;
+    downloadableFiles: DownloadableFile[];
+}
+/**
+ * Extracts the video URL from the current lesson page.
+ * Supports Loom, Vimeo, YouTube, Wistia, and native video.
+ *
+ * For Vimeo: Prefers iframe src (has auth params) over __NEXT_DATA__ URL.
+ * For others: Uses __NEXT_DATA__ first, then falls back to DOM inspection.
+ */
+export declare function extractVideoUrl(page: Page): Promise<{
+    url: string | null;
+    type: LessonContent["videoType"];
+}>;
+/**
+ * Extracts the text content from the lesson page.
+ */
+export declare function extractTextContent(page: Page): Promise<{
+    html: string;
+    markdown: string;
+}>;
+/**
+ * Extracts downloadable file links from the page content.
+ */
+export declare function extractDownloadableFiles(page: Page): Promise<DownloadableFile[]>;
+/**
+ * Extracts all content from a lesson page.
+ */
+export declare function extractLessonContent(page: Page, lessonUrl: string): Promise<LessonContent>;
+/**
+ * Extracts the Loom video ID from an embed URL.
+ */
+export declare function extractLoomVideoId(embedUrl: string): string | null;
+/**
+ * Cleans and formats the markdown content.
+ */
+export declare function formatMarkdown(title: string, content: string, videoUrl: string | null, videoType: string | null): string;
+//# sourceMappingURL=extractor.d.ts.map

package/dist/scraper/extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/scraper/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC;CAClF;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,IAAI,CAAC;IACjF,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;CACvC;AA8OD;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,IAAI,GACT,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;CAAE,CAAC,CAgCnE;AAmPD;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAsDhG;AA+BD;;GAEG;AACH,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAkDtF;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAyBhG;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGlE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,SAAS,EAAE,MAAM,GAAG,IAAI,GACvB,MAAM,CAiBR"}