offcourse 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/config.js.map +1 -1
- package/dist/cli/commands/inspect.js +1 -1
- package/dist/cli/commands/inspect.js.map +1 -1
- package/dist/cli/commands/sync.d.ts +1 -2
- package/dist/cli/commands/sync.d.ts.map +1 -1
- package/dist/cli/commands/sync.js +13 -14
- package/dist/cli/commands/sync.js.map +1 -1
- package/dist/cli/commands/syncHighLevel.d.ts +1 -2
- package/dist/cli/commands/syncHighLevel.d.ts.map +1 -1
- package/dist/cli/commands/syncHighLevel.js +4 -8
- package/dist/cli/commands/syncHighLevel.js.map +1 -1
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/config/configManager.d.ts.map +1 -1
- package/dist/config/configManager.js +4 -0
- package/dist/config/configManager.js.map +1 -1
- package/dist/downloader/hlsDownloader.d.ts.map +1 -1
- package/dist/downloader/hlsDownloader.js +23 -14
- package/dist/downloader/hlsDownloader.js.map +1 -1
- package/dist/downloader/hlsValidator.d.ts.map +1 -1
- package/dist/downloader/hlsValidator.js +6 -2
- package/dist/downloader/hlsValidator.js.map +1 -1
- package/dist/downloader/index.d.ts +3 -0
- package/dist/downloader/index.d.ts.map +1 -1
- package/dist/downloader/index.js +3 -0
- package/dist/downloader/index.js.map +1 -1
- package/dist/downloader/loomDownloader.d.ts.map +1 -1
- package/dist/downloader/loomDownloader.js +23 -20
- package/dist/downloader/loomDownloader.js.map +1 -1
- package/dist/downloader/queue.d.ts +4 -4
- package/dist/downloader/queue.d.ts.map +1 -1
- package/dist/downloader/queue.js.map +1 -1
- package/dist/downloader/vimeoDownloader.d.ts.map +1 -1
- package/dist/downloader/vimeoDownloader.js +7 -3
- package/dist/downloader/vimeoDownloader.js.map +1 -1
- package/dist/scraper/extractor.d.ts +4 -0
- package/dist/scraper/extractor.d.ts.map +1 -1
- package/dist/scraper/extractor.js +79 -79
- package/dist/scraper/extractor.js.map +1 -1
- package/dist/scraper/highlevel/extractor.d.ts +11 -19
- package/dist/scraper/highlevel/extractor.d.ts.map +1 -1
- package/dist/scraper/highlevel/extractor.js +72 -85
- package/dist/scraper/highlevel/extractor.js.map +1 -1
- package/dist/scraper/highlevel/navigator.d.ts +3 -10
- package/dist/scraper/highlevel/navigator.d.ts.map +1 -1
- package/dist/scraper/highlevel/navigator.js +140 -127
- package/dist/scraper/highlevel/navigator.js.map +1 -1
- package/dist/scraper/highlevel/schemas.d.ts +188 -0
- package/dist/scraper/highlevel/schemas.d.ts.map +1 -0
- package/dist/scraper/highlevel/schemas.js +139 -0
- package/dist/scraper/highlevel/schemas.js.map +1 -0
- package/dist/scraper/navigator.d.ts +14 -11
- package/dist/scraper/navigator.d.ts.map +1 -1
- package/dist/scraper/navigator.js +61 -104
- package/dist/scraper/navigator.js.map +1 -1
- package/dist/scraper/schemas.d.ts +57 -0
- package/dist/scraper/schemas.d.ts.map +1 -0
- package/dist/scraper/schemas.js +135 -0
- package/dist/scraper/schemas.js.map +1 -0
- package/dist/scraper/videoInterceptor.d.ts +4 -0
- package/dist/scraper/videoInterceptor.d.ts.map +1 -1
- package/dist/scraper/videoInterceptor.js +66 -51
- package/dist/scraper/videoInterceptor.js.map +1 -1
- package/dist/shared/auth.d.ts +9 -9
- package/dist/shared/auth.d.ts.map +1 -1
- package/dist/shared/auth.js +24 -38
- package/dist/shared/auth.js.map +1 -1
- package/dist/shared/firebase.d.ts +60 -0
- package/dist/shared/firebase.d.ts.map +1 -0
- package/dist/shared/firebase.js +102 -0
- package/dist/shared/firebase.js.map +1 -0
- package/dist/shared/fs.d.ts.map +1 -1
- package/dist/shared/fs.js +4 -0
- package/dist/shared/fs.js.map +1 -1
- package/dist/shared/index.d.ts +3 -0
- package/dist/shared/index.d.ts.map +1 -1
- package/dist/shared/index.js +3 -0
- package/dist/shared/index.js.map +1 -1
- package/dist/shared/slug.d.ts +11 -0
- package/dist/shared/slug.d.ts.map +1 -0
- package/{src/shared/slug.ts → dist/shared/slug.js} +10 -11
- package/dist/shared/slug.js.map +1 -0
- package/dist/shared/url.d.ts +43 -0
- package/dist/shared/url.d.ts.map +1 -0
- package/{src/shared/url.ts → dist/shared/url.js} +12 -15
- package/dist/shared/url.js.map +1 -0
- package/dist/state/database.d.ts +1 -0
- package/dist/state/database.d.ts.map +1 -1
- package/dist/state/database.js +3 -0
- package/dist/state/database.js.map +1 -1
- package/dist/storage/fileSystem.d.ts +17 -17
- package/dist/storage/fileSystem.d.ts.map +1 -1
- package/dist/storage/fileSystem.js +39 -31
- package/dist/storage/fileSystem.js.map +1 -1
- package/package.json +5 -2
- package/.github/workflows/ci.yml +0 -50
- package/.husky/commit-msg +0 -2
- package/.husky/pre-commit +0 -1
- package/.husky/pre-push +0 -3
- package/.prettierrc +0 -8
- package/.release-it.json +0 -23
- package/ARCHITECTURE.md +0 -233
- package/CHANGELOG.md +0 -78
- package/commitlint.config.js +0 -4
- package/dist/ai/openRouter.d.ts +0 -47
- package/dist/ai/openRouter.d.ts.map +0 -1
- package/dist/ai/openRouter.js +0 -116
- package/dist/ai/openRouter.js.map +0 -1
- package/dist/ai/transcriptPolisher.d.ts +0 -24
- package/dist/ai/transcriptPolisher.d.ts.map +0 -1
- package/dist/ai/transcriptPolisher.js +0 -89
- package/dist/ai/transcriptPolisher.js.map +0 -1
- package/dist/cli/commands/enrich.d.ts +0 -14
- package/dist/cli/commands/enrich.d.ts.map +0 -1
- package/dist/cli/commands/enrich.js +0 -271
- package/dist/cli/commands/enrich.js.map +0 -1
- package/dist/cli/commands/syncGhl.d.ts +0 -20
- package/dist/cli/commands/syncGhl.d.ts.map +0 -1
- package/dist/cli/commands/syncGhl.js +0 -483
- package/dist/cli/commands/syncGhl.js.map +0 -1
- package/dist/cli/commands/syncHighLevel.test.d.ts +0 -2
- package/dist/cli/commands/syncHighLevel.test.d.ts.map +0 -1
- package/dist/cli/commands/syncHighLevel.test.js +0 -102
- package/dist/cli/commands/syncHighLevel.test.js.map +0 -1
- package/dist/config/paths.test.d.ts +0 -2
- package/dist/config/paths.test.d.ts.map +0 -1
- package/dist/config/paths.test.js +0 -70
- package/dist/config/paths.test.js.map +0 -1
- package/dist/config/schema.test.d.ts +0 -2
- package/dist/config/schema.test.d.ts.map +0 -1
- package/dist/config/schema.test.js +0 -151
- package/dist/config/schema.test.js.map +0 -1
- package/dist/downloader/hlsDownloader.test.d.ts +0 -2
- package/dist/downloader/hlsDownloader.test.d.ts.map +0 -1
- package/dist/downloader/hlsDownloader.test.js +0 -116
- package/dist/downloader/hlsDownloader.test.js.map +0 -1
- package/dist/downloader/loomDownloader.test.d.ts +0 -2
- package/dist/downloader/loomDownloader.test.d.ts.map +0 -1
- package/dist/downloader/loomDownloader.test.js +0 -36
- package/dist/downloader/loomDownloader.test.js.map +0 -1
- package/dist/downloader/queue.test.d.ts +0 -2
- package/dist/downloader/queue.test.d.ts.map +0 -1
- package/dist/downloader/queue.test.js +0 -158
- package/dist/downloader/queue.test.js.map +0 -1
- package/dist/downloader/videoDownloader.d.ts +0 -32
- package/dist/downloader/videoDownloader.d.ts.map +0 -1
- package/dist/downloader/videoDownloader.js +0 -173
- package/dist/downloader/videoDownloader.js.map +0 -1
- package/dist/downloader/vimeoDownloader.test.d.ts +0 -2
- package/dist/downloader/vimeoDownloader.test.d.ts.map +0 -1
- package/dist/downloader/vimeoDownloader.test.js +0 -51
- package/dist/downloader/vimeoDownloader.test.js.map +0 -1
- package/dist/scraper/auth.d.ts +0 -29
- package/dist/scraper/auth.d.ts.map +0 -1
- package/dist/scraper/auth.js +0 -115
- package/dist/scraper/auth.js.map +0 -1
- package/dist/scraper/extractor.test.d.ts +0 -2
- package/dist/scraper/extractor.test.d.ts.map +0 -1
- package/dist/scraper/extractor.test.js +0 -65
- package/dist/scraper/extractor.test.js.map +0 -1
- package/dist/scraper/ghl/auth.d.ts +0 -25
- package/dist/scraper/ghl/auth.d.ts.map +0 -1
- package/dist/scraper/ghl/auth.js +0 -187
- package/dist/scraper/ghl/auth.js.map +0 -1
- package/dist/scraper/ghl/extractor.d.ts +0 -96
- package/dist/scraper/ghl/extractor.d.ts.map +0 -1
- package/dist/scraper/ghl/extractor.js +0 -345
- package/dist/scraper/ghl/extractor.js.map +0 -1
- package/dist/scraper/ghl/index.d.ts +0 -4
- package/dist/scraper/ghl/index.d.ts.map +0 -1
- package/dist/scraper/ghl/index.js +0 -4
- package/dist/scraper/ghl/index.js.map +0 -1
- package/dist/scraper/ghl/navigator.d.ts +0 -93
- package/dist/scraper/ghl/navigator.d.ts.map +0 -1
- package/dist/scraper/ghl/navigator.js +0 -447
- package/dist/scraper/ghl/navigator.js.map +0 -1
- package/dist/scraper/highlevel/auth.d.ts +0 -25
- package/dist/scraper/highlevel/auth.d.ts.map +0 -1
- package/dist/scraper/highlevel/auth.js +0 -189
- package/dist/scraper/highlevel/auth.js.map +0 -1
- package/dist/scraper/highlevel/extractor.test.d.ts +0 -2
- package/dist/scraper/highlevel/extractor.test.d.ts.map +0 -1
- package/dist/scraper/highlevel/extractor.test.js +0 -101
- package/dist/scraper/highlevel/extractor.test.js.map +0 -1
- package/dist/scraper/highlevel/navigator.test.d.ts +0 -2
- package/dist/scraper/highlevel/navigator.test.d.ts.map +0 -1
- package/dist/scraper/highlevel/navigator.test.js +0 -78
- package/dist/scraper/highlevel/navigator.test.js.map +0 -1
- package/dist/scraper/navigator.test.d.ts +0 -2
- package/dist/scraper/navigator.test.d.ts.map +0 -1
- package/dist/scraper/navigator.test.js +0 -63
- package/dist/scraper/navigator.test.js.map +0 -1
- package/dist/scraper/skoolApi.d.ts +0 -17
- package/dist/scraper/skoolApi.d.ts.map +0 -1
- package/dist/scraper/skoolApi.js +0 -72
- package/dist/scraper/skoolApi.js.map +0 -1
- package/dist/state/database.test.d.ts +0 -2
- package/dist/state/database.test.d.ts.map +0 -1
- package/dist/state/database.test.js +0 -34
- package/dist/state/database.test.js.map +0 -1
- package/dist/transcription/whisperService.d.ts +0 -27
- package/dist/transcription/whisperService.d.ts.map +0 -1
- package/dist/transcription/whisperService.js +0 -102
- package/dist/transcription/whisperService.js.map +0 -1
- package/eslint.config.js +0 -55
- package/src/__fixtures__/highlevel-post-response.json +0 -68
- package/src/__fixtures__/hls-master-playlist.m3u8 +0 -24
- package/src/cli/commands/__snapshots__/syncHighLevel.test.ts.snap +0 -38
- package/src/cli/commands/config.ts +0 -74
- package/src/cli/commands/inspect.ts +0 -441
- package/src/cli/commands/login.ts +0 -68
- package/src/cli/commands/status.ts +0 -147
- package/src/cli/commands/sync.ts +0 -1235
- package/src/cli/commands/syncHighLevel.test.ts +0 -144
- package/src/cli/commands/syncHighLevel.ts +0 -639
- package/src/cli/index.ts +0 -121
- package/src/config/configManager.ts +0 -75
- package/src/config/paths.test.ts +0 -83
- package/src/config/paths.ts +0 -36
- package/src/config/schema.test.ts +0 -173
- package/src/config/schema.ts +0 -65
- package/src/downloader/hlsDownloader.test.ts +0 -148
- package/src/downloader/hlsDownloader.ts +0 -327
- package/src/downloader/hlsValidator.ts +0 -196
- package/src/downloader/index.ts +0 -122
- package/src/downloader/loomDownloader.test.ts +0 -43
- package/src/downloader/loomDownloader.ts +0 -742
- package/src/downloader/queue.test.ts +0 -199
- package/src/downloader/queue.ts +0 -118
- package/src/downloader/vimeoDownloader.test.ts +0 -62
- package/src/downloader/vimeoDownloader.ts +0 -722
- package/src/scraper/extractor.test.ts +0 -124
- package/src/scraper/extractor.ts +0 -757
- package/src/scraper/highlevel/__snapshots__/extractor.test.ts.snap +0 -41
- package/src/scraper/highlevel/extractor.test.ts +0 -134
- package/src/scraper/highlevel/extractor.ts +0 -537
- package/src/scraper/highlevel/index.ts +0 -2
- package/src/scraper/highlevel/navigator.test.ts +0 -110
- package/src/scraper/highlevel/navigator.ts +0 -668
- package/src/scraper/highlevel/schemas.ts +0 -183
- package/src/scraper/navigator.test.ts +0 -122
- package/src/scraper/navigator.ts +0 -355
- package/src/scraper/schemas.ts +0 -177
- package/src/scraper/videoInterceptor.ts +0 -435
- package/src/shared/auth.test.ts +0 -58
- package/src/shared/auth.ts +0 -251
- package/src/shared/firebase.ts +0 -151
- package/src/shared/fs.ts +0 -80
- package/src/shared/http.ts +0 -34
- package/src/shared/index.ts +0 -6
- package/src/shared/url.test.ts +0 -122
- package/src/state/database.test.ts +0 -49
- package/src/state/database.ts +0 -919
- package/src/state/index.ts +0 -14
- package/src/storage/fileSystem.test.ts +0 -64
- package/src/storage/fileSystem.ts +0 -175
- package/tsconfig.json +0 -28
- package/vitest.config.ts +0 -29
|
@@ -1,189 +0,0 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { dirname } from "node:path";
|
|
3
|
-
import { chromium } from "playwright";
|
|
4
|
-
import { getSessionPath, SESSIONS_DIR } from "../../config/paths.js";
|
|
5
|
-
/**
|
|
6
|
-
* Checks if a valid HighLevel session exists for the given domain.
|
|
7
|
-
*/
|
|
8
|
-
export function hasValidHighLevelSession(domain) {
|
|
9
|
-
const sessionPath = getSessionPath(domain);
|
|
10
|
-
return existsSync(sessionPath);
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Loads an existing session from disk.
|
|
14
|
-
*/
|
|
15
|
-
async function loadSession(browser, domain) {
|
|
16
|
-
const sessionPath = getSessionPath(domain);
|
|
17
|
-
const storageState = JSON.parse(readFileSync(sessionPath, "utf-8"));
|
|
18
|
-
return browser.newContext({ storageState });
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* Saves the current session to disk.
|
|
22
|
-
*/
|
|
23
|
-
async function saveSession(context, domain) {
|
|
24
|
-
const sessionPath = getSessionPath(domain);
|
|
25
|
-
const dir = dirname(sessionPath);
|
|
26
|
-
if (!existsSync(dir)) {
|
|
27
|
-
mkdirSync(dir, { recursive: true });
|
|
28
|
-
}
|
|
29
|
-
const storageState = await context.storageState();
|
|
30
|
-
writeFileSync(sessionPath, JSON.stringify(storageState, null, 2), "utf-8");
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Checks if the current page is on a HighLevel login page.
|
|
34
|
-
*/
|
|
35
|
-
function isHighLevelLoginPage(url) {
|
|
36
|
-
const loginPatterns = [
|
|
37
|
-
/sso\.clientclub\.net/,
|
|
38
|
-
/\/login/,
|
|
39
|
-
/\/signin/,
|
|
40
|
-
/\/auth/,
|
|
41
|
-
/accounts\.google\.com/,
|
|
42
|
-
/firebaseapp\.com/,
|
|
43
|
-
];
|
|
44
|
-
return loginPatterns.some((p) => p.test(url));
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Checks if the page has a valid Firebase auth token.
|
|
48
|
-
*/
|
|
49
|
-
async function hasValidFirebaseToken(page) {
|
|
50
|
-
try {
|
|
51
|
-
const hasToken = await page.evaluate(() => {
|
|
52
|
-
const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
|
|
53
|
-
if (!tokenKey)
|
|
54
|
-
return false;
|
|
55
|
-
const tokenData = JSON.parse(localStorage.getItem(tokenKey) ?? "{}");
|
|
56
|
-
const expirationTime = tokenData?.stsTokenManager?.expirationTime;
|
|
57
|
-
// Check if token exists and is not expired
|
|
58
|
-
if (expirationTime) {
|
|
59
|
-
return Date.now() < expirationTime;
|
|
60
|
-
}
|
|
61
|
-
return !!tokenData?.stsTokenManager?.accessToken;
|
|
62
|
-
});
|
|
63
|
-
return hasToken;
|
|
64
|
-
}
|
|
65
|
-
catch {
|
|
66
|
-
return false;
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
/**
|
|
70
|
-
* Performs interactive login for HighLevel by opening a browser window.
|
|
71
|
-
* The user logs in manually, and we capture the session.
|
|
72
|
-
*/
|
|
73
|
-
export async function performHighLevelInteractiveLogin(domain, portalUrl) {
|
|
74
|
-
// Ensure sessions directory exists
|
|
75
|
-
if (!existsSync(SESSIONS_DIR)) {
|
|
76
|
-
mkdirSync(SESSIONS_DIR, { recursive: true });
|
|
77
|
-
}
|
|
78
|
-
const browser = await chromium.launch({
|
|
79
|
-
headless: false, // Must be visible for user interaction
|
|
80
|
-
});
|
|
81
|
-
const context = await browser.newContext({
|
|
82
|
-
viewport: { width: 1280, height: 800 },
|
|
83
|
-
});
|
|
84
|
-
const page = await context.newPage();
|
|
85
|
-
await page.goto(portalUrl);
|
|
86
|
-
console.log("\n🔐 Browser opened. Please log in manually.");
|
|
87
|
-
console.log(" The window will close automatically after successful login.\n");
|
|
88
|
-
// Wait for either:
|
|
89
|
-
// 1. Navigation away from login page
|
|
90
|
-
// 2. Firebase token to appear in localStorage
|
|
91
|
-
let loggedIn = false;
|
|
92
|
-
const startTime = Date.now();
|
|
93
|
-
const timeout = 300000; // 5 minutes
|
|
94
|
-
while (!loggedIn && Date.now() - startTime < timeout) {
|
|
95
|
-
await page.waitForTimeout(1000);
|
|
96
|
-
const currentUrl = page.url();
|
|
97
|
-
// Check if we're still on a login page
|
|
98
|
-
if (!isHighLevelLoginPage(currentUrl)) {
|
|
99
|
-
// Might be logged in, check for Firebase token
|
|
100
|
-
const hasToken = await hasValidFirebaseToken(page);
|
|
101
|
-
if (hasToken) {
|
|
102
|
-
loggedIn = true;
|
|
103
|
-
break;
|
|
104
|
-
}
|
|
105
|
-
// Also check if we're on a course page (successful login)
|
|
106
|
-
if (currentUrl.includes("/courses/") || currentUrl.includes("/library")) {
|
|
107
|
-
loggedIn = true;
|
|
108
|
-
break;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
if (!loggedIn) {
|
|
113
|
-
await browser.close();
|
|
114
|
-
throw new Error("Login timed out after 5 minutes");
|
|
115
|
-
}
|
|
116
|
-
// Give the page a moment to fully load after login
|
|
117
|
-
await page.waitForLoadState("networkidle").catch(() => { });
|
|
118
|
-
await page.waitForTimeout(2000);
|
|
119
|
-
// Save the session
|
|
120
|
-
await saveSession(context, domain);
|
|
121
|
-
console.log("✅ Login successful! Session saved.\n");
|
|
122
|
-
return { context, page };
|
|
123
|
-
}
|
|
124
|
-
/**
|
|
125
|
-
* Gets an authenticated HighLevel session, either from cache or via interactive login.
|
|
126
|
-
*/
|
|
127
|
-
export async function getHighLevelAuthenticatedSession(domain, portalUrl, options = {}) {
|
|
128
|
-
// Default to headless mode (true) unless explicitly set to false
|
|
129
|
-
const useHeadless = options.headless !== false;
|
|
130
|
-
const browser = await chromium.launch({
|
|
131
|
-
headless: useHeadless,
|
|
132
|
-
});
|
|
133
|
-
// Try to use existing session
|
|
134
|
-
if (!options.forceLogin && hasValidHighLevelSession(domain)) {
|
|
135
|
-
try {
|
|
136
|
-
const context = await loadSession(browser, domain);
|
|
137
|
-
const page = await context.newPage();
|
|
138
|
-
// Navigate to portal
|
|
139
|
-
await page.goto(portalUrl);
|
|
140
|
-
await page.waitForLoadState("domcontentloaded");
|
|
141
|
-
await page.waitForTimeout(2000);
|
|
142
|
-
const currentUrl = page.url();
|
|
143
|
-
// Check if we got redirected to login or SSO
|
|
144
|
-
if (isHighLevelLoginPage(currentUrl)) {
|
|
145
|
-
console.log("⚠️ Session expired, need to re-login...");
|
|
146
|
-
await context.close();
|
|
147
|
-
await browser.close();
|
|
148
|
-
}
|
|
149
|
-
else {
|
|
150
|
-
// Verify we have a valid Firebase token
|
|
151
|
-
const hasToken = await hasValidFirebaseToken(page);
|
|
152
|
-
if (hasToken) {
|
|
153
|
-
console.log("✅ Using cached session");
|
|
154
|
-
return { browser, session: { context, page } };
|
|
155
|
-
}
|
|
156
|
-
else {
|
|
157
|
-
console.log("⚠️ No valid auth token, need to re-login...");
|
|
158
|
-
await context.close();
|
|
159
|
-
await browser.close();
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
catch (error) {
|
|
164
|
-
console.log("⚠️ Failed to load session, need to re-login...", error);
|
|
165
|
-
await browser.close();
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
else {
|
|
169
|
-
await browser.close();
|
|
170
|
-
}
|
|
171
|
-
// Need fresh login - always visible for interactive login
|
|
172
|
-
const session = await performHighLevelInteractiveLogin(domain, portalUrl);
|
|
173
|
-
// Get the browser from the session context
|
|
174
|
-
const sessionBrowser = session.context.browser();
|
|
175
|
-
if (!sessionBrowser) {
|
|
176
|
-
throw new Error("Failed to get browser from session");
|
|
177
|
-
}
|
|
178
|
-
// After login, reopen with headless browser (unless explicitly set to false)
|
|
179
|
-
if (useHeadless) {
|
|
180
|
-
const newBrowser = await chromium.launch({ headless: true });
|
|
181
|
-
const context = await loadSession(newBrowser, domain);
|
|
182
|
-
const page = await context.newPage();
|
|
183
|
-
// Close the interactive session
|
|
184
|
-
await sessionBrowser.close();
|
|
185
|
-
return { browser: newBrowser, session: { context, page } };
|
|
186
|
-
}
|
|
187
|
-
return { browser: sessionBrowser, session };
|
|
188
|
-
}
|
|
189
|
-
//# sourceMappingURL=auth.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/auth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAOrE;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,MAAc;IACrD,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,OAAgB,EAAE,MAAc;IACzD,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACpE,OAAO,OAAO,CAAC,UAAU,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,OAAuB,EAAE,MAAc;IAChE,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEjC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAClD,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AAC7E,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,GAAW;IACvC,MAAM,aAAa,GAAG;QACpB,sBAAsB;QACtB,SAAS;QACT,UAAU;QACV,QAAQ;QACR,uBAAuB;QACvB,kBAAkB;KACnB,CAAC;IACF,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,qBAAqB,CAAC,IAAU;IAC7C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;YACxF,IAAI,CAAC,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;YACrE,MAAM,cAAc,GAAG,SAAS,EAAE,eAAe,EAAE,cAAc,CAAC;YAElE,2CAA2C;YAC3C,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;YACrC,CAAC;YAED,OAAO,CAAC,CAAC,SAAS,EAAE,eAAe,EAAE,WAAW,CAAC;QACnD,CAAC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,MAAc,EACd,SAAiB;IAEjB,mCAAmC;IACnC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACpC,QAAQ,EAAE,KAAK,EAAE,uCAAuC;KACzD,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;QACvC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;KACvC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAE3B,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;IAEhF,mBAAmB;IACnB,qCAAqC;IACrC,8CAA8C;IAC9C,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,YAAY;IAEpC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,OAAO,EAAE,CAAC;QACrD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,uCAAuC;QACvC,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAC;YACtC,+CAA+C;YAC/C,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM;YACR,CAAC;YAED,0DAA0D;YAC1D,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACxE,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,mDAAmD;IACnD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAEhC,mBAAmB;IACnB,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IAEpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,MAAc,EACd,SAAiB,EACjB,UAAwD,EAAE;IAE1D,iEAAiE;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC;IAE/C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACpC,QAAQ,EAAE,WAAW;KACtB,CAAC,CAAC;IAEH,8BAA8B;IAC9B,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,wBAAwB,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAErC,qBAAqB;YACrB,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC3B,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;YAChD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAEhC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE9B,6CAA6C;YAC7C,IAAI,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;gBACxD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,wCAAwC;gBACxC,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;gBACnD,IAAI,QAAQ,EAAE,CAAC;oBACb,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;oBACtC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;oBAC5D,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;oBACtB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;YACtE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,0DAA0D;IAC1D,MAAM,OAAO,GAAG,MAAM,gCAAgC,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAE1E,2CAA2C;IAC3C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACjD,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IAED,6EAA6E;IAC7E,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,gCAAgC;QAChC,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;IAC7D,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.test.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/extractor.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
2
|
-
import { join, dirname } from "node:path";
|
|
3
|
-
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { describe, expect, it } from "vitest";
|
|
5
|
-
import { parseHLSMasterPlaylist } from "./extractor.js";
|
|
6
|
-
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
7
|
-
const fixturesDir = join(__dirname, "../../__fixtures__");
|
|
8
|
-
describe("parseHLSMasterPlaylist", () => {
|
|
9
|
-
const baseUrl = "https://cdn.example.com/video/";
|
|
10
|
-
it("parses master playlist with multiple qualities", () => {
|
|
11
|
-
const content = `#EXTM3U
|
|
12
|
-
#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360
|
|
13
|
-
360p.m3u8
|
|
14
|
-
#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
|
|
15
|
-
720p.m3u8
|
|
16
|
-
#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=1920x1080
|
|
17
|
-
1080p.m3u8`;
|
|
18
|
-
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
19
|
-
expect(result).toHaveLength(3);
|
|
20
|
-
// Should be sorted by bandwidth (highest first)
|
|
21
|
-
expect(result[0].label).toBe("1080p");
|
|
22
|
-
expect(result[0].bandwidth).toBe(5000000);
|
|
23
|
-
expect(result[0].height).toBe(1080);
|
|
24
|
-
expect(result[0].width).toBe(1920);
|
|
25
|
-
});
|
|
26
|
-
it("resolves relative URLs correctly", () => {
|
|
27
|
-
const content = `#EXTM3U
|
|
28
|
-
#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
|
|
29
|
-
720p/index.m3u8`;
|
|
30
|
-
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
31
|
-
expect(result[0].url).toBe("https://cdn.example.com/video/720p/index.m3u8");
|
|
32
|
-
});
|
|
33
|
-
it("preserves absolute URLs", () => {
|
|
34
|
-
const content = `#EXTM3U
|
|
35
|
-
#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
|
|
36
|
-
https://other-cdn.com/720p.m3u8`;
|
|
37
|
-
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
38
|
-
expect(result[0].url).toBe("https://other-cdn.com/720p.m3u8");
|
|
39
|
-
});
|
|
40
|
-
it("handles playlist without resolution (audio only)", () => {
|
|
41
|
-
const content = `#EXTM3U
|
|
42
|
-
#EXT-X-STREAM-INF:BANDWIDTH=128000
|
|
43
|
-
audio.m3u8`;
|
|
44
|
-
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
45
|
-
expect(result).toHaveLength(1);
|
|
46
|
-
expect(result[0].label).toBe("128k");
|
|
47
|
-
expect(result[0].height).toBeUndefined();
|
|
48
|
-
});
|
|
49
|
-
it("returns empty array for empty playlist", () => {
|
|
50
|
-
const content = `#EXTM3U
|
|
51
|
-
#EXT-X-VERSION:3`;
|
|
52
|
-
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
53
|
-
expect(result).toHaveLength(0);
|
|
54
|
-
});
|
|
55
|
-
describe("with fixture file", () => {
|
|
56
|
-
it("parses real-world HLS master playlist", () => {
|
|
57
|
-
const content = readFileSync(join(fixturesDir, "hls-master-playlist.m3u8"), "utf-8");
|
|
58
|
-
const result = parseHLSMasterPlaylist(content, "https://cdn.example.com/video/");
|
|
59
|
-
expect(result).toHaveLength(5);
|
|
60
|
-
// Sorted by bandwidth (highest first)
|
|
61
|
-
expect(result[0].label).toBe("1080p");
|
|
62
|
-
expect(result[0].height).toBe(1080);
|
|
63
|
-
expect(result[0].bandwidth).toBe(6000000);
|
|
64
|
-
expect(result[4].label).toBe("240p");
|
|
65
|
-
expect(result[4].height).toBe(240);
|
|
66
|
-
});
|
|
67
|
-
it("snapshot: parsed playlist structure", () => {
|
|
68
|
-
const content = readFileSync(join(fixturesDir, "hls-master-playlist.m3u8"), "utf-8");
|
|
69
|
-
const result = parseHLSMasterPlaylist(content, "https://cdn.example.com/video/");
|
|
70
|
-
expect(result).toMatchSnapshot();
|
|
71
|
-
});
|
|
72
|
-
});
|
|
73
|
-
});
|
|
74
|
-
describe("HighLevel API Response parsing", () => {
|
|
75
|
-
it("snapshot: fixture structure", () => {
|
|
76
|
-
const response = JSON.parse(readFileSync(join(fixturesDir, "highlevel-post-response.json"), "utf-8"));
|
|
77
|
-
// Verify the structure we depend on
|
|
78
|
-
expect(response).toHaveProperty("id");
|
|
79
|
-
expect(response).toHaveProperty("title");
|
|
80
|
-
expect(response).toHaveProperty("video");
|
|
81
|
-
expect(response).toHaveProperty("post_materials");
|
|
82
|
-
// Video structure
|
|
83
|
-
expect(response.video).toHaveProperty("url");
|
|
84
|
-
expect(response.video).toHaveProperty("assetsLicenseId");
|
|
85
|
-
expect(response.video.url).toMatch(/\.mp4$/);
|
|
86
|
-
// Materials structure
|
|
87
|
-
expect(response.post_materials).toHaveLength(2);
|
|
88
|
-
expect(response.post_materials[0]).toHaveProperty("name");
|
|
89
|
-
expect(response.post_materials[0]).toHaveProperty("url");
|
|
90
|
-
});
|
|
91
|
-
it("extracts video info from response", () => {
|
|
92
|
-
const response = JSON.parse(readFileSync(join(fixturesDir, "highlevel-post-response.json"), "utf-8"));
|
|
93
|
-
// Simulate the extraction logic
|
|
94
|
-
const video = response.video;
|
|
95
|
-
const assetId = video?.assetsLicenseId ?? video?.assetId ?? video?.id;
|
|
96
|
-
const directUrl = video?.url;
|
|
97
|
-
expect(assetId).toBe("689a0d672ea246086539f453");
|
|
98
|
-
expect(directUrl).toContain("1080p.mp4");
|
|
99
|
-
});
|
|
100
|
-
});
|
|
101
|
-
//# sourceMappingURL=extractor.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.test.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/extractor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;AAE1D,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,MAAM,OAAO,GAAG,gCAAgC,CAAC;IAEjD,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,OAAO,GAAG;;;;;;WAMT,CAAC;QAER,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,gDAAgD;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,OAAO,GAAG;;gBAEJ,CAAC;QAEb,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,OAAO,GAAG;;gCAEY,CAAC;QAE7B,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,OAAO,GAAG;;WAET,CAAC;QAER,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,OAAO,GAAG;iBACH,CAAC;QAEd,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,CAAC,EAAE,OAAO,CAAC,CAAC;YAErF,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,gCAAgC,CAAC,CAAC;YAEjF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAE/B,sCAAsC;YACtC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,CAAC,EAAE,OAAO,CAAC,CAAC;YACrF,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,gCAAgC,CAAC,CAAC;YAEjF,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;IAC9C,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,8BAA8B,CAAC,EAAE,OAAO,CAAC,CACzE,CAAC;QAEF,oCAAoC;QACpC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAElD,kBAAkB;QAClB,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE7C,sBAAsB;QACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,8BAA8B,CAAC,EAAE,OAAO,CAAC,CACzE,CAAC;QAEF,gCAAgC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC7B,MAAM,OAAO,GAAG,KAAK,EAAE,eAAe,IAAI,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,KAAK,EAAE,GAAG,CAAC;QAE7B,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACjD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"navigator.test.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/navigator.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { slugify, createFolderName, getHighLevelCourseUrl, getHighLevelPostUrl, } from "./navigator.js";
|
|
3
|
-
describe("slugify", () => {
|
|
4
|
-
it("converts to lowercase", () => {
|
|
5
|
-
expect(slugify("Hello World")).toBe("hello-world");
|
|
6
|
-
});
|
|
7
|
-
it("replaces spaces with hyphens", () => {
|
|
8
|
-
expect(slugify("my test string")).toBe("my-test-string");
|
|
9
|
-
});
|
|
10
|
-
it("removes special characters", () => {
|
|
11
|
-
expect(slugify("Hello! How are you?")).toBe("hello-how-are-you");
|
|
12
|
-
});
|
|
13
|
-
it("replaces German umlauts", () => {
|
|
14
|
-
expect(slugify("Größe")).toBe("groesse");
|
|
15
|
-
expect(slugify("Äpfel")).toBe("aepfel");
|
|
16
|
-
expect(slugify("Übung")).toBe("uebung");
|
|
17
|
-
expect(slugify("Straße")).toBe("strasse");
|
|
18
|
-
});
|
|
19
|
-
it("removes leading and trailing hyphens", () => {
|
|
20
|
-
expect(slugify("--hello--")).toBe("hello");
|
|
21
|
-
expect(slugify(" hello ")).toBe("hello");
|
|
22
|
-
});
|
|
23
|
-
it("collapses multiple hyphens", () => {
|
|
24
|
-
expect(slugify("hello world")).toBe("hello-world");
|
|
25
|
-
expect(slugify("hello---world")).toBe("hello-world");
|
|
26
|
-
});
|
|
27
|
-
it("truncates to 100 characters", () => {
|
|
28
|
-
const longName = "a".repeat(150);
|
|
29
|
-
expect(slugify(longName).length).toBe(100);
|
|
30
|
-
});
|
|
31
|
-
it("handles empty string", () => {
|
|
32
|
-
expect(slugify("")).toBe("");
|
|
33
|
-
});
|
|
34
|
-
it("handles numbers", () => {
|
|
35
|
-
expect(slugify("Lesson 1: Introduction")).toBe("lesson-1-introduction");
|
|
36
|
-
});
|
|
37
|
-
it("handles mixed content", () => {
|
|
38
|
-
expect(slugify("🚀 Getting Started! (Part 1)")).toBe("getting-started-part-1");
|
|
39
|
-
});
|
|
40
|
-
});
|
|
41
|
-
describe("createFolderName", () => {
|
|
42
|
-
it("creates folder name with padded index", () => {
|
|
43
|
-
expect(createFolderName(0, "Introduction")).toBe("01-introduction");
|
|
44
|
-
expect(createFolderName(9, "Conclusion")).toBe("10-conclusion");
|
|
45
|
-
});
|
|
46
|
-
it("handles double-digit indices", () => {
|
|
47
|
-
expect(createFolderName(99, "Final")).toBe("100-final");
|
|
48
|
-
});
|
|
49
|
-
it("slugifies the name", () => {
|
|
50
|
-
expect(createFolderName(0, "Hello World!")).toBe("01-hello-world");
|
|
51
|
-
});
|
|
52
|
-
it("handles German umlauts in name", () => {
|
|
53
|
-
expect(createFolderName(2, "Größe & Übung")).toBe("03-groesse-uebung");
|
|
54
|
-
});
|
|
55
|
-
});
|
|
56
|
-
describe("getHighLevelCourseUrl", () => {
|
|
57
|
-
it("constructs correct course URL", () => {
|
|
58
|
-
const url = getHighLevelCourseUrl("member.example.com", "abc-123");
|
|
59
|
-
expect(url).toBe("https://member.example.com/courses/products/abc-123?source=courses");
|
|
60
|
-
});
|
|
61
|
-
it("handles UUID product IDs", () => {
|
|
62
|
-
const url = getHighLevelCourseUrl("portal.school.com", "e5f64bf3-9d88-4d02-b10e-516f47866094");
|
|
63
|
-
expect(url).toBe("https://portal.school.com/courses/products/e5f64bf3-9d88-4d02-b10e-516f47866094?source=courses");
|
|
64
|
-
});
|
|
65
|
-
});
|
|
66
|
-
describe("getHighLevelPostUrl", () => {
|
|
67
|
-
it("constructs correct post URL", () => {
|
|
68
|
-
const url = getHighLevelPostUrl("member.example.com", "product-1", "category-2", "post-3");
|
|
69
|
-
expect(url).toBe("https://member.example.com/courses/products/product-1/categories/category-2/posts/post-3?source=courses");
|
|
70
|
-
});
|
|
71
|
-
it("handles UUID IDs", () => {
|
|
72
|
-
const url = getHighLevelPostUrl("member.test.com", "e5f64bf3-9d88-4d02-b10e-516f47866094", "cat-abc-123", "post-xyz-789");
|
|
73
|
-
expect(url).toContain("products/e5f64bf3-9d88-4d02-b10e-516f47866094");
|
|
74
|
-
expect(url).toContain("categories/cat-abc-123");
|
|
75
|
-
expect(url).toContain("posts/post-xyz-789");
|
|
76
|
-
});
|
|
77
|
-
});
|
|
78
|
-
//# sourceMappingURL=navigator.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"navigator.test.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/navigator.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,OAAO,EACP,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,gBAAgB,CAAC;AAExB,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACvB,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACrD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iBAAiB,EAAE,GAAG,EAAE;QACzB,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACpE,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CAAC,gBAAgB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAC5B,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,GAAG,GAAG,qBAAqB,CAAC,oBAAoB,EAAE,SAAS,CAAC,CAAC;QACnE,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,GAAG,GAAG,qBAAqB,CAAC,mBAAmB,EAAE,sCAAsC,CAAC,CAAC;QAC/F,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CACd,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,GAAG,GAAG,mBAAmB,CAAC,oBAAoB,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;QAC3F,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CACd,yGAAyG,CAC1G,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,mBAAmB,CAC7B,iBAAiB,EACjB,sCAAsC,EACtC,aAAa,EACb,cAAc,CACf,CAAC;QACF,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,+CAA+C,CAAC,CAAC;QACvE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"navigator.test.d.ts","sourceRoot":"","sources":["../../src/scraper/navigator.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { createFolderName, slugify } from "./navigator.js";
|
|
3
|
-
describe("slugify", () => {
|
|
4
|
-
it("converts to lowercase", () => {
|
|
5
|
-
expect(slugify("Hello World")).toBe("hello-world");
|
|
6
|
-
});
|
|
7
|
-
it("replaces spaces with hyphens", () => {
|
|
8
|
-
expect(slugify("hello world test")).toBe("hello-world-test");
|
|
9
|
-
});
|
|
10
|
-
it("removes special characters", () => {
|
|
11
|
-
expect(slugify("Hello! World? Test.")).toBe("hello-world-test");
|
|
12
|
-
});
|
|
13
|
-
it("handles German umlauts", () => {
|
|
14
|
-
expect(slugify("Größe")).toBe("groesse");
|
|
15
|
-
expect(slugify("Über")).toBe("ueber");
|
|
16
|
-
expect(slugify("Änderung")).toBe("aenderung");
|
|
17
|
-
expect(slugify("Straße")).toBe("strasse");
|
|
18
|
-
});
|
|
19
|
-
it("collapses multiple hyphens", () => {
|
|
20
|
-
expect(slugify("hello world")).toBe("hello-world");
|
|
21
|
-
expect(slugify("hello---world")).toBe("hello-world");
|
|
22
|
-
});
|
|
23
|
-
it("removes leading and trailing hyphens", () => {
|
|
24
|
-
expect(slugify(" hello world ")).toBe("hello-world");
|
|
25
|
-
expect(slugify("---hello---")).toBe("hello");
|
|
26
|
-
});
|
|
27
|
-
it("truncates to 100 characters", () => {
|
|
28
|
-
const longString = "a".repeat(150);
|
|
29
|
-
const result = slugify(longString);
|
|
30
|
-
expect(result.length).toBeLessThanOrEqual(100);
|
|
31
|
-
});
|
|
32
|
-
it("handles numbers", () => {
|
|
33
|
-
expect(slugify("Lesson 1: Introduction")).toBe("lesson-1-introduction");
|
|
34
|
-
});
|
|
35
|
-
it("handles empty string", () => {
|
|
36
|
-
expect(slugify("")).toBe("");
|
|
37
|
-
});
|
|
38
|
-
it("handles string with only special characters", () => {
|
|
39
|
-
// @sindresorhus/slugify converts & to "and"
|
|
40
|
-
expect(slugify("!@#$%^&*()")).toBe("and");
|
|
41
|
-
// Pure symbols without & become empty
|
|
42
|
-
expect(slugify("!@#$%^*()")).toBe("");
|
|
43
|
-
});
|
|
44
|
-
});
|
|
45
|
-
describe("createFolderName", () => {
|
|
46
|
-
it("creates folder name with zero-padded index", () => {
|
|
47
|
-
expect(createFolderName(0, "Introduction")).toBe("01-introduction");
|
|
48
|
-
expect(createFolderName(9, "Advanced Topics")).toBe("10-advanced-topics");
|
|
49
|
-
});
|
|
50
|
-
it("handles double-digit indices", () => {
|
|
51
|
-
expect(createFolderName(99, "Last Module")).toBe("100-last-module");
|
|
52
|
-
});
|
|
53
|
-
it("applies slugify to name", () => {
|
|
54
|
-
expect(createFolderName(0, "Hello World!")).toBe("01-hello-world");
|
|
55
|
-
});
|
|
56
|
-
it("handles German characters in name", () => {
|
|
57
|
-
expect(createFolderName(0, "Einführung")).toBe("01-einfuehrung");
|
|
58
|
-
});
|
|
59
|
-
it("handles empty name", () => {
|
|
60
|
-
expect(createFolderName(0, "")).toBe("01-");
|
|
61
|
-
});
|
|
62
|
-
});
|
|
63
|
-
//# sourceMappingURL=navigator.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"navigator.test.js","sourceRoot":"","sources":["../../src/scraper/navigator.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAE3D,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACvB,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QAChC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACrD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACvD,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iBAAiB,EAAE,GAAG,EAAE;QACzB,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,4CAA4C;QAC5C,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,sCAAsC;QACtC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACpE,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CAAC,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAC5B,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import type { BrowserContext } from "playwright";
|
|
2
|
-
/**
|
|
3
|
-
* Marks a lesson as complete on Skool.
|
|
4
|
-
*/
|
|
5
|
-
export declare function markLessonComplete(context: BrowserContext, lessonId: string): Promise<{
|
|
6
|
-
success: boolean;
|
|
7
|
-
error?: string;
|
|
8
|
-
}>;
|
|
9
|
-
/**
|
|
10
|
-
* Marks multiple lessons as complete.
|
|
11
|
-
*/
|
|
12
|
-
export declare function markLessonsComplete(context: BrowserContext, lessonIds: string[]): Promise<{
|
|
13
|
-
completed: number;
|
|
14
|
-
failed: number;
|
|
15
|
-
errors: string[];
|
|
16
|
-
}>;
|
|
17
|
-
//# sourceMappingURL=skoolApi.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"skoolApi.d.ts","sourceRoot":"","sources":["../../src/scraper/skoolApi.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAwBjD;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAgC/C;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,cAAc,EACvB,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAkBlE"}
|
package/dist/scraper/skoolApi.js
DELETED
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
const SKOOL_API_BASE = "https://api2.skool.com";
|
|
2
|
-
/**
|
|
3
|
-
* Extracts auth cookies from a Playwright browser context.
|
|
4
|
-
*/
|
|
5
|
-
async function getAuthCookies(context) {
|
|
6
|
-
const cookies = await context.cookies("https://www.skool.com");
|
|
7
|
-
const authToken = cookies.find((c) => c.name === "auth_token")?.value;
|
|
8
|
-
const clientId = cookies.find((c) => c.name === "client_id")?.value;
|
|
9
|
-
const parts = [];
|
|
10
|
-
if (clientId) {
|
|
11
|
-
parts.push(`client_id=${clientId}`);
|
|
12
|
-
}
|
|
13
|
-
if (authToken) {
|
|
14
|
-
parts.push(`auth_token=${authToken}`);
|
|
15
|
-
}
|
|
16
|
-
return parts.join("; ");
|
|
17
|
-
}
|
|
18
|
-
/**
|
|
19
|
-
* Marks a lesson as complete on Skool.
|
|
20
|
-
*/
|
|
21
|
-
export async function markLessonComplete(context, lessonId) {
|
|
22
|
-
try {
|
|
23
|
-
const cookieHeader = await getAuthCookies(context);
|
|
24
|
-
if (!cookieHeader.includes("auth_token")) {
|
|
25
|
-
return { success: false, error: "No auth token found" };
|
|
26
|
-
}
|
|
27
|
-
const response = await fetch(`${SKOOL_API_BASE}/courses/${lessonId}/mark-done`, {
|
|
28
|
-
method: "POST",
|
|
29
|
-
headers: {
|
|
30
|
-
"Accept": "*/*",
|
|
31
|
-
"Content-Type": "application/json",
|
|
32
|
-
"Cookie": cookieHeader,
|
|
33
|
-
"Origin": "https://www.skool.com",
|
|
34
|
-
"Referer": "https://www.skool.com/",
|
|
35
|
-
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
|
36
|
-
},
|
|
37
|
-
});
|
|
38
|
-
if (!response.ok) {
|
|
39
|
-
const text = await response.text();
|
|
40
|
-
return { success: false, error: `HTTP ${response.status}: ${text}` };
|
|
41
|
-
}
|
|
42
|
-
return { success: true };
|
|
43
|
-
}
|
|
44
|
-
catch (error) {
|
|
45
|
-
return {
|
|
46
|
-
success: false,
|
|
47
|
-
error: error instanceof Error ? error.message : String(error)
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Marks multiple lessons as complete.
|
|
53
|
-
*/
|
|
54
|
-
export async function markLessonsComplete(context, lessonIds) {
|
|
55
|
-
let completed = 0;
|
|
56
|
-
let failed = 0;
|
|
57
|
-
const errors = [];
|
|
58
|
-
for (const lessonId of lessonIds) {
|
|
59
|
-
const result = await markLessonComplete(context, lessonId);
|
|
60
|
-
if (result.success) {
|
|
61
|
-
completed++;
|
|
62
|
-
}
|
|
63
|
-
else {
|
|
64
|
-
failed++;
|
|
65
|
-
errors.push(`${lessonId}: ${result.error}`);
|
|
66
|
-
}
|
|
67
|
-
// Small delay to avoid rate limiting
|
|
68
|
-
await new Promise((r) => setTimeout(r, 100));
|
|
69
|
-
}
|
|
70
|
-
return { completed, failed, errors };
|
|
71
|
-
}
|
|
72
|
-
//# sourceMappingURL=skoolApi.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"skoolApi.js","sourceRoot":"","sources":["../../src/scraper/skoolApi.ts"],"names":[],"mappings":"AAEA,MAAM,cAAc,GAAG,wBAAwB,CAAC;AAEhD;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,OAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAE/D,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,KAAK,CAAC;IACtE,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,EAAE,KAAK,CAAC;IAEpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAAuB,EACvB,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YACzC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;QAC1D,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,cAAc,YAAY,QAAQ,YAAY,EAAE;YAC9E,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,kBAAkB;gBAClC,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,uBAAuB;gBACjC,SAAS,EAAE,wBAAwB;gBACnC,YAAY,EAAE,oEAAoE;aACnF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC9D,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAuB,EACvB,SAAmB;IAEnB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC3D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,SAAS,EAAE,CAAC;QACd,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,CAAC;YACT,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC9C,CAAC;QACD,qCAAqC;QACrC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"database.test.d.ts","sourceRoot":"","sources":["../../src/state/database.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { extractCommunitySlug } from "./database.js";
|
|
3
|
-
describe("extractCommunitySlug", () => {
|
|
4
|
-
it("extracts slug from standard Skool URL", () => {
|
|
5
|
-
expect(extractCommunitySlug("https://www.skool.com/my-community")).toBe("my-community");
|
|
6
|
-
});
|
|
7
|
-
it("extracts slug from Skool URL without www", () => {
|
|
8
|
-
expect(extractCommunitySlug("https://skool.com/test-group")).toBe("test-group");
|
|
9
|
-
});
|
|
10
|
-
it("extracts slug from URL with path", () => {
|
|
11
|
-
expect(extractCommunitySlug("https://www.skool.com/my-community/classroom")).toBe("my-community");
|
|
12
|
-
expect(extractCommunitySlug("https://www.skool.com/my-community/classroom/lessons/123")).toBe("my-community");
|
|
13
|
-
});
|
|
14
|
-
it("extracts slug from URL with query params (includes params in slug)", () => {
|
|
15
|
-
// Note: current implementation doesn't strip query params
|
|
16
|
-
expect(extractCommunitySlug("https://www.skool.com/my-community?ref=abc")).toBe("my-community?ref=abc");
|
|
17
|
-
});
|
|
18
|
-
it("handles complex community names", () => {
|
|
19
|
-
expect(extractCommunitySlug("https://skool.com/the-best-community-ever-2024")).toBe("the-best-community-ever-2024");
|
|
20
|
-
});
|
|
21
|
-
it("returns 'unknown' for non-Skool URLs", () => {
|
|
22
|
-
expect(extractCommunitySlug("https://example.com/path")).toBe("unknown");
|
|
23
|
-
expect(extractCommunitySlug("https://youtube.com/channel/abc")).toBe("unknown");
|
|
24
|
-
});
|
|
25
|
-
it("returns 'unknown' for invalid URLs", () => {
|
|
26
|
-
expect(extractCommunitySlug("not-a-url")).toBe("unknown");
|
|
27
|
-
expect(extractCommunitySlug("")).toBe("unknown");
|
|
28
|
-
});
|
|
29
|
-
it("returns 'unknown' for Skool root URL", () => {
|
|
30
|
-
expect(extractCommunitySlug("https://www.skool.com/")).toBe("unknown");
|
|
31
|
-
expect(extractCommunitySlug("https://www.skool.com")).toBe("unknown");
|
|
32
|
-
});
|
|
33
|
-
});
|
|
34
|
-
//# sourceMappingURL=database.test.js.map
|