offcourse 0.0.2 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -20
- package/dist/cli/commands/config.d.ts +13 -0
- package/dist/cli/commands/config.d.ts.map +1 -0
- package/dist/cli/commands/config.js +66 -0
- package/dist/cli/commands/config.js.map +1 -0
- package/dist/cli/commands/inspect.d.ts +11 -0
- package/dist/cli/commands/inspect.d.ts.map +1 -0
- package/dist/cli/commands/inspect.js +365 -0
- package/dist/cli/commands/inspect.js.map +1 -0
- package/dist/cli/commands/login.d.ts +12 -0
- package/dist/cli/commands/login.d.ts.map +1 -0
- package/dist/cli/commands/login.js +55 -0
- package/dist/cli/commands/login.js.map +1 -0
- package/dist/cli/commands/status.d.ts +15 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +118 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/sync.d.ts +15 -0
- package/dist/cli/commands/sync.d.ts.map +1 -0
- package/dist/cli/commands/sync.js +921 -0
- package/dist/cli/commands/sync.js.map +1 -0
- package/dist/cli/commands/syncHighLevel.d.ts +23 -0
- package/dist/cli/commands/syncHighLevel.d.ts.map +1 -0
- package/dist/cli/commands/syncHighLevel.js +479 -0
- package/dist/cli/commands/syncHighLevel.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +106 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/config/configManager.d.ts +31 -0
- package/dist/config/configManager.d.ts.map +1 -0
- package/dist/config/configManager.js +68 -0
- package/dist/config/configManager.js.map +1 -0
- package/dist/config/paths.d.ts +21 -0
- package/dist/config/paths.d.ts.map +1 -0
- package/dist/config/paths.js +33 -0
- package/dist/config/paths.js.map +1 -0
- package/dist/config/schema.d.ts +60 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +50 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/downloader/hlsDownloader.d.ts +58 -0
- package/dist/downloader/hlsDownloader.d.ts.map +1 -0
- package/dist/downloader/hlsDownloader.js +263 -0
- package/dist/downloader/hlsDownloader.js.map +1 -0
- package/dist/downloader/hlsValidator.d.ts +35 -0
- package/dist/downloader/hlsValidator.d.ts.map +1 -0
- package/dist/downloader/hlsValidator.js +152 -0
- package/dist/downloader/hlsValidator.js.map +1 -0
- package/dist/downloader/index.d.ts +29 -0
- package/dist/downloader/index.d.ts.map +1 -0
- package/dist/downloader/index.js +55 -0
- package/dist/downloader/index.js.map +1 -0
- package/dist/downloader/loomDownloader.d.ts +56 -0
- package/dist/downloader/loomDownloader.d.ts.map +1 -0
- package/dist/downloader/loomDownloader.js +562 -0
- package/dist/downloader/loomDownloader.js.map +1 -0
- package/dist/downloader/queue.d.ts +56 -0
- package/dist/downloader/queue.d.ts.map +1 -0
- package/dist/downloader/queue.js +88 -0
- package/dist/downloader/queue.js.map +1 -0
- package/dist/downloader/vimeoDownloader.d.ts +52 -0
- package/dist/downloader/vimeoDownloader.d.ts.map +1 -0
- package/dist/downloader/vimeoDownloader.js +569 -0
- package/dist/downloader/vimeoDownloader.js.map +1 -0
- package/dist/scraper/extractor.d.ts +53 -0
- package/dist/scraper/extractor.d.ts.map +1 -0
- package/dist/scraper/extractor.js +627 -0
- package/dist/scraper/extractor.js.map +1 -0
- package/dist/scraper/highlevel/extractor.d.ts +89 -0
- package/dist/scraper/highlevel/extractor.d.ts.map +1 -0
- package/dist/scraper/highlevel/extractor.js +373 -0
- package/dist/scraper/highlevel/extractor.js.map +1 -0
- package/dist/scraper/highlevel/index.d.ts +3 -0
- package/dist/scraper/highlevel/index.d.ts.map +1 -0
- package/dist/scraper/highlevel/index.js +3 -0
- package/dist/scraper/highlevel/index.js.map +1 -0
- package/dist/scraper/highlevel/navigator.d.ts +86 -0
- package/dist/scraper/highlevel/navigator.d.ts.map +1 -0
- package/dist/scraper/highlevel/navigator.js +505 -0
- package/dist/scraper/highlevel/navigator.js.map +1 -0
- package/dist/scraper/highlevel/schemas.d.ts +188 -0
- package/dist/scraper/highlevel/schemas.d.ts.map +1 -0
- package/dist/scraper/highlevel/schemas.js +139 -0
- package/dist/scraper/highlevel/schemas.js.map +1 -0
- package/dist/scraper/navigator.d.ts +68 -0
- package/dist/scraper/navigator.d.ts.map +1 -0
- package/dist/scraper/navigator.js +257 -0
- package/dist/scraper/navigator.js.map +1 -0
- package/dist/scraper/schemas.d.ts +57 -0
- package/dist/scraper/schemas.d.ts.map +1 -0
- package/dist/scraper/schemas.js +135 -0
- package/dist/scraper/schemas.js.map +1 -0
- package/dist/scraper/videoInterceptor.d.ts +23 -0
- package/dist/scraper/videoInterceptor.d.ts.map +1 -0
- package/dist/scraper/videoInterceptor.js +330 -0
- package/dist/scraper/videoInterceptor.js.map +1 -0
- package/dist/shared/auth.d.ts +58 -0
- package/dist/shared/auth.d.ts.map +1 -0
- package/dist/shared/auth.js +197 -0
- package/dist/shared/auth.js.map +1 -0
- package/dist/shared/firebase.d.ts +60 -0
- package/dist/shared/firebase.d.ts.map +1 -0
- package/dist/shared/firebase.js +102 -0
- package/dist/shared/firebase.js.map +1 -0
- package/dist/shared/fs.d.ts +31 -0
- package/dist/shared/fs.d.ts.map +1 -0
- package/dist/shared/fs.js +77 -0
- package/dist/shared/fs.js.map +1 -0
- package/dist/shared/http.d.ts +15 -0
- package/dist/shared/http.d.ts.map +1 -0
- package/dist/shared/http.js +31 -0
- package/dist/shared/http.js.map +1 -0
- package/dist/shared/index.d.ts +7 -0
- package/dist/shared/index.d.ts.map +1 -0
- package/dist/shared/index.js +7 -0
- package/dist/shared/index.js.map +1 -0
- package/dist/shared/slug.d.ts +11 -0
- package/dist/shared/slug.d.ts.map +1 -0
- package/dist/shared/slug.js +25 -0
- package/dist/shared/slug.js.map +1 -0
- package/dist/shared/url.d.ts +43 -0
- package/dist/shared/url.d.ts.map +1 -0
- package/dist/shared/url.js +54 -0
- package/dist/shared/url.js.map +1 -0
- package/dist/state/database.d.ts +246 -0
- package/dist/state/database.d.ts.map +1 -0
- package/dist/state/database.js +679 -0
- package/dist/state/database.js.map +1 -0
- package/dist/state/index.d.ts +2 -0
- package/dist/state/index.d.ts.map +1 -0
- package/dist/state/index.js +2 -0
- package/dist/state/index.js.map +1 -0
- package/dist/storage/fileSystem.d.ts +56 -0
- package/dist/storage/fileSystem.d.ts.map +1 -0
- package/dist/storage/fileSystem.js +129 -0
- package/dist/storage/fileSystem.js.map +1 -0
- package/package.json +71 -11
- package/cli.js +0 -45
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import type { Page } from "playwright";
|
|
2
|
+
import { parseHLSPlaylist } from "../../downloader/hlsDownloader.js";
|
|
3
|
+
declare const parseHLSMasterPlaylist: typeof parseHLSPlaylist;
|
|
4
|
+
export interface HighLevelVideoInfo {
|
|
5
|
+
type: "hls" | "vimeo" | "loom" | "youtube" | "custom";
|
|
6
|
+
url: string;
|
|
7
|
+
masterPlaylistUrl?: string;
|
|
8
|
+
qualities?: {
|
|
9
|
+
label: string;
|
|
10
|
+
url: string;
|
|
11
|
+
width?: number;
|
|
12
|
+
height?: number;
|
|
13
|
+
}[];
|
|
14
|
+
duration?: number;
|
|
15
|
+
thumbnailUrl?: string;
|
|
16
|
+
token?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface HighLevelPostContent {
|
|
19
|
+
id: string;
|
|
20
|
+
title: string;
|
|
21
|
+
description: string | null;
|
|
22
|
+
htmlContent: string | null;
|
|
23
|
+
video: HighLevelVideoInfo | null;
|
|
24
|
+
attachments: {
|
|
25
|
+
id: string;
|
|
26
|
+
name: string;
|
|
27
|
+
url: string;
|
|
28
|
+
type: string;
|
|
29
|
+
size?: number;
|
|
30
|
+
}[];
|
|
31
|
+
categoryId: string;
|
|
32
|
+
productId: string;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Extracts the Firebase auth token from the page.
|
|
36
|
+
*/
|
|
37
|
+
export declare function getAuthToken(page: Page): Promise<string | null>;
|
|
38
|
+
/**
|
|
39
|
+
* Extracts video info from a HighLevel post page by intercepting network requests.
|
|
40
|
+
*/
|
|
41
|
+
export declare function extractVideoFromPage(page: Page): Promise<HighLevelVideoInfo | null>;
|
|
42
|
+
/**
|
|
43
|
+
* Extracts video info by intercepting network requests during page load.
|
|
44
|
+
*/
|
|
45
|
+
export declare function interceptVideoRequests(page: Page, postUrl: string): Promise<HighLevelVideoInfo | null>;
|
|
46
|
+
/**
|
|
47
|
+
* Fetches post details from the API.
|
|
48
|
+
*/
|
|
49
|
+
export declare function fetchPostDetails(page: Page, locationId: string, postId: string): Promise<{
|
|
50
|
+
title: string;
|
|
51
|
+
description: string | null;
|
|
52
|
+
video: {
|
|
53
|
+
assetId: string;
|
|
54
|
+
url: string;
|
|
55
|
+
} | null;
|
|
56
|
+
materials: {
|
|
57
|
+
id: string;
|
|
58
|
+
name: string;
|
|
59
|
+
url: string;
|
|
60
|
+
type: string;
|
|
61
|
+
}[];
|
|
62
|
+
} | null>;
|
|
63
|
+
/**
|
|
64
|
+
* Fetches the DRM license (HLS token) for a video asset.
|
|
65
|
+
*/
|
|
66
|
+
export declare function fetchVideoLicense(page: Page, assetId: string): Promise<{
|
|
67
|
+
url: string;
|
|
68
|
+
token: string;
|
|
69
|
+
} | null>;
|
|
70
|
+
/**
|
|
71
|
+
* Extracts complete post content including video and attachments.
|
|
72
|
+
*/
|
|
73
|
+
export declare function extractHighLevelPostContent(page: Page, postUrl: string, locationId: string, productId: string, postId: string, categoryId: string): Promise<HighLevelPostContent | null>;
|
|
74
|
+
export { parseHLSMasterPlaylist };
|
|
75
|
+
/**
|
|
76
|
+
* Fetches and parses HLS playlist to get quality options.
|
|
77
|
+
*/
|
|
78
|
+
export declare function getHLSQualities(page: Page, masterPlaylistUrl: string): Promise<{
|
|
79
|
+
label: string;
|
|
80
|
+
url: string;
|
|
81
|
+
bandwidth: number;
|
|
82
|
+
width?: number | undefined;
|
|
83
|
+
height?: number | undefined;
|
|
84
|
+
}[]>;
|
|
85
|
+
/**
|
|
86
|
+
* Gets the best quality URL from an HLS master playlist.
|
|
87
|
+
*/
|
|
88
|
+
export declare function getBestHLSQuality(page: Page, masterPlaylistUrl: string): Promise<string | null>;
|
|
89
|
+
//# sourceMappingURL=extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAUrE,QAAA,MAAM,sBAAsB,yBAAmB,CAAC;AAEhD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;IACtD,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QACV,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,EAAE,CAAC;IACJ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACjC,WAAW,EAAE;QACX,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;IACJ,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAKD;;GAEG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgBrE;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAmFzF;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA0CpC;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IACT,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,GAAG,EAAE,MAAM,CAAC;KACb,GAAG,IAAI,CAAC;IACT,SAAS,EAAE;QACT,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;KACd,EAAE,CAAC;CACL,GAAG,IAAI,CAAC,CA4HR;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAsChD;AAED;;GAEG;AACH,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CA0EtC;AAID,OAAO,EAAE,sBAAsB,EAAE,CAAC;AAGlC;;GAEG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,IAAI,EACV,iBAAiB,EAAE,MAAM,GACxB,OAAO,CACR;IACE,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B,EAAE,CACJ,CAcA;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CASxB"}
|
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
import { parseHLSPlaylist } from "../../downloader/hlsDownloader.js";
|
|
2
|
+
import { FirebaseAuthTokenSchema, PostDetailsResponseSchema, VideoLicenseResponseSchema, safeParse, } from "./schemas.js";
|
|
3
|
+
// Alias for backwards compatibility and internal use
|
|
4
|
+
const parseHLSMasterPlaylist = parseHLSPlaylist;
|
|
5
|
+
// Browser/API automation - requires Playwright
|
|
6
|
+
/* v8 ignore start */
|
|
7
|
+
/**
|
|
8
|
+
* Extracts the Firebase auth token from the page.
|
|
9
|
+
*/
|
|
10
|
+
export async function getAuthToken(page) {
|
|
11
|
+
const rawData = await page.evaluate(() => {
|
|
12
|
+
const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
|
|
13
|
+
if (!tokenKey)
|
|
14
|
+
return null;
|
|
15
|
+
try {
|
|
16
|
+
return JSON.parse(localStorage.getItem(tokenKey) ?? "{}");
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
if (!rawData)
|
|
23
|
+
return null;
|
|
24
|
+
const parsed = safeParse(FirebaseAuthTokenSchema, rawData, "getAuthToken");
|
|
25
|
+
return parsed?.stsTokenManager.accessToken ?? null;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Extracts video info from a HighLevel post page by intercepting network requests.
|
|
29
|
+
*/
|
|
30
|
+
export async function extractVideoFromPage(page) {
|
|
31
|
+
// First, check if there's an HLS video on the page
|
|
32
|
+
const hlsUrl = await page.evaluate(() => {
|
|
33
|
+
// Look for HLS master playlist URLs in the DOM
|
|
34
|
+
const videoElements = Array.from(document.querySelectorAll("video"));
|
|
35
|
+
for (const video of videoElements) {
|
|
36
|
+
const src = video.currentSrc ?? video.src;
|
|
37
|
+
if (src?.includes(".m3u8")) {
|
|
38
|
+
return src;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
// Check for plyr or other players
|
|
42
|
+
const sources = Array.from(document.querySelectorAll('source[type*="m3u8"], source[src*=".m3u8"]'));
|
|
43
|
+
for (const source of sources) {
|
|
44
|
+
const src = source.src;
|
|
45
|
+
if (src)
|
|
46
|
+
return src;
|
|
47
|
+
}
|
|
48
|
+
return null;
|
|
49
|
+
});
|
|
50
|
+
if (hlsUrl) {
|
|
51
|
+
return {
|
|
52
|
+
type: "hls",
|
|
53
|
+
url: hlsUrl,
|
|
54
|
+
masterPlaylistUrl: hlsUrl,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
// Check for Vimeo embed
|
|
58
|
+
const vimeoUrl = await page.evaluate(() => {
|
|
59
|
+
const iframe = document.querySelector('iframe[src*="vimeo.com"], iframe[src*="player.vimeo"]');
|
|
60
|
+
if (iframe) {
|
|
61
|
+
return iframe.src;
|
|
62
|
+
}
|
|
63
|
+
return null;
|
|
64
|
+
});
|
|
65
|
+
if (vimeoUrl) {
|
|
66
|
+
return {
|
|
67
|
+
type: "vimeo",
|
|
68
|
+
url: vimeoUrl,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
// Check for Loom embed
|
|
72
|
+
const loomUrl = await page.evaluate(() => {
|
|
73
|
+
const iframe = document.querySelector('iframe[src*="loom.com"]');
|
|
74
|
+
if (iframe) {
|
|
75
|
+
return iframe.src;
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
});
|
|
79
|
+
if (loomUrl) {
|
|
80
|
+
return {
|
|
81
|
+
type: "loom",
|
|
82
|
+
url: loomUrl,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
// Check for YouTube embed
|
|
86
|
+
const youtubeUrl = await page.evaluate(() => {
|
|
87
|
+
const iframe = document.querySelector('iframe[src*="youtube.com"], iframe[src*="youtube-nocookie.com"], iframe[src*="youtu.be"]');
|
|
88
|
+
if (iframe) {
|
|
89
|
+
return iframe.src;
|
|
90
|
+
}
|
|
91
|
+
return null;
|
|
92
|
+
});
|
|
93
|
+
if (youtubeUrl) {
|
|
94
|
+
return {
|
|
95
|
+
type: "youtube",
|
|
96
|
+
url: youtubeUrl,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Extracts video info by intercepting network requests during page load.
|
|
103
|
+
*/
|
|
104
|
+
export async function interceptVideoRequests(page, postUrl) {
|
|
105
|
+
const hlsUrls = [];
|
|
106
|
+
const drmUrls = [];
|
|
107
|
+
// Set up request interception
|
|
108
|
+
const requestHandler = (request) => {
|
|
109
|
+
const url = request.url();
|
|
110
|
+
// Capture HLS master playlist requests
|
|
111
|
+
if (url.includes(".m3u8") || url.includes("master.m3u8")) {
|
|
112
|
+
hlsUrls.push(url);
|
|
113
|
+
}
|
|
114
|
+
// Capture DRM license requests
|
|
115
|
+
if (url.includes("assets-drm/assets-license")) {
|
|
116
|
+
drmUrls.push(url);
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
page.on("request", requestHandler);
|
|
120
|
+
// Navigate to the post page
|
|
121
|
+
await page.goto(postUrl, { timeout: 30000 });
|
|
122
|
+
await page.waitForLoadState("domcontentloaded");
|
|
123
|
+
await page.waitForTimeout(3000);
|
|
124
|
+
// Remove the handler
|
|
125
|
+
page.off("request", requestHandler);
|
|
126
|
+
// Get the HLS master playlist URL
|
|
127
|
+
const masterPlaylistUrl = hlsUrls.find((url) => url.includes("master.m3u8"));
|
|
128
|
+
if (masterPlaylistUrl) {
|
|
129
|
+
return {
|
|
130
|
+
type: "hls",
|
|
131
|
+
url: masterPlaylistUrl,
|
|
132
|
+
masterPlaylistUrl,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
// Fallback to DOM extraction
|
|
136
|
+
return extractVideoFromPage(page);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Fetches post details from the API.
|
|
140
|
+
*/
|
|
141
|
+
export async function fetchPostDetails(page, locationId, postId) {
|
|
142
|
+
const rawData = await page.evaluate(async ({ locationId, postId }) => {
|
|
143
|
+
try {
|
|
144
|
+
const tokenKey = Object.keys(localStorage).find((k) => k.includes("firebase:authUser"));
|
|
145
|
+
const tokenData = tokenKey
|
|
146
|
+
? JSON.parse(localStorage.getItem(tokenKey) ?? "{}")
|
|
147
|
+
: null;
|
|
148
|
+
const token = tokenData?.stsTokenManager?.accessToken;
|
|
149
|
+
if (!token) {
|
|
150
|
+
return { error: "No auth token" };
|
|
151
|
+
}
|
|
152
|
+
const res = await fetch(`https://services.leadconnectorhq.com/membership/locations/${locationId}/posts/${postId}?source=courses`, {
|
|
153
|
+
headers: {
|
|
154
|
+
Authorization: `Bearer ${token}`,
|
|
155
|
+
},
|
|
156
|
+
});
|
|
157
|
+
if (!res.ok) {
|
|
158
|
+
return { error: `HTTP ${res.status}`, status: res.status };
|
|
159
|
+
}
|
|
160
|
+
const data = await res.json();
|
|
161
|
+
return { data };
|
|
162
|
+
}
|
|
163
|
+
catch (error) {
|
|
164
|
+
return { error: String(error) };
|
|
165
|
+
}
|
|
166
|
+
}, { locationId, postId });
|
|
167
|
+
// Debug: Log raw response in Node context
|
|
168
|
+
if ("error" in rawData) {
|
|
169
|
+
console.log(`[DEBUG] API Error: ${rawData.error}`);
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
const data = rawData.data;
|
|
173
|
+
if (!data) {
|
|
174
|
+
console.log("[DEBUG] No data in response");
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
// Validate response with Zod schema
|
|
178
|
+
const parsed = safeParse(PostDetailsResponseSchema, data, "fetchPostDetails");
|
|
179
|
+
if (!parsed) {
|
|
180
|
+
console.log("[DEBUG] Response validation failed");
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
// The API returns data directly (not nested under .post)
|
|
184
|
+
// Check both for backwards compatibility
|
|
185
|
+
const post = parsed.post ?? parsed;
|
|
186
|
+
let video = null;
|
|
187
|
+
// Check for video directly on post
|
|
188
|
+
// Video can have: id, assetId, assetsLicenseId, or direct url
|
|
189
|
+
if (post.video) {
|
|
190
|
+
const videoAssetId = post.video.assetsLicenseId ?? post.video.assetId ?? post.video.id;
|
|
191
|
+
if (videoAssetId || post.video.url) {
|
|
192
|
+
video = {
|
|
193
|
+
assetId: videoAssetId ?? "",
|
|
194
|
+
url: post.video.url ?? "",
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
// Check posterImage for video asset (older format)
|
|
199
|
+
if (!video && post.posterImage?.assetId) {
|
|
200
|
+
video = {
|
|
201
|
+
assetId: post.posterImage.assetId,
|
|
202
|
+
url: post.posterImage.url ?? "",
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
// Check for video in contentBlock
|
|
206
|
+
if (!video && post.contentBlock) {
|
|
207
|
+
for (const block of post.contentBlock) {
|
|
208
|
+
if (block.type === "video") {
|
|
209
|
+
const blockAssetId = block.assetsLicenseId ?? block.assetId ?? block.id;
|
|
210
|
+
if (blockAssetId || block.url) {
|
|
211
|
+
video = {
|
|
212
|
+
assetId: blockAssetId ?? "",
|
|
213
|
+
url: block.url ?? "",
|
|
214
|
+
};
|
|
215
|
+
break;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
const materials = [];
|
|
221
|
+
// Materials can be under 'materials' or 'post_materials'
|
|
222
|
+
const materialsList = post.materials ?? post.post_materials ?? [];
|
|
223
|
+
for (const material of materialsList) {
|
|
224
|
+
materials.push({
|
|
225
|
+
id: material.id ?? crypto.randomUUID(),
|
|
226
|
+
name: material.name ?? "Attachment",
|
|
227
|
+
url: material.url ?? "",
|
|
228
|
+
type: material.type ?? "file",
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
title: post.title ?? "",
|
|
233
|
+
description: post.description ?? null,
|
|
234
|
+
video,
|
|
235
|
+
materials,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Fetches the DRM license (HLS token) for a video asset.
|
|
240
|
+
*/
|
|
241
|
+
export async function fetchVideoLicense(page, assetId) {
|
|
242
|
+
// Get auth token first
|
|
243
|
+
const authToken = await getAuthToken(page);
|
|
244
|
+
if (!authToken) {
|
|
245
|
+
return null;
|
|
246
|
+
}
|
|
247
|
+
try {
|
|
248
|
+
// Use page.request to make the API call (bypasses CORS)
|
|
249
|
+
const response = await page.request.get(`https://backend.leadconnectorhq.com/assets-drm/assets-license/${assetId}`, {
|
|
250
|
+
headers: {
|
|
251
|
+
Authorization: `Bearer ${authToken}`,
|
|
252
|
+
},
|
|
253
|
+
});
|
|
254
|
+
if (!response.ok()) {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
const data = await response.json();
|
|
258
|
+
// Validate response with Zod schema
|
|
259
|
+
const parsed = safeParse(VideoLicenseResponseSchema, data, "fetchVideoLicense");
|
|
260
|
+
if (!parsed) {
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
return {
|
|
264
|
+
url: parsed.url,
|
|
265
|
+
token: parsed.token,
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
catch (error) {
|
|
269
|
+
console.error("Failed to fetch video license:", error);
|
|
270
|
+
return null;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Extracts complete post content including video and attachments.
|
|
275
|
+
*/
|
|
276
|
+
export async function extractHighLevelPostContent(page, postUrl, locationId, productId, postId, categoryId) {
|
|
277
|
+
// Navigate to post page
|
|
278
|
+
await page.goto(postUrl, { timeout: 30000 });
|
|
279
|
+
await page.waitForLoadState("domcontentloaded");
|
|
280
|
+
await page.waitForTimeout(3000);
|
|
281
|
+
// Fetch post details from API
|
|
282
|
+
const postDetails = await fetchPostDetails(page, locationId, postId);
|
|
283
|
+
if (!postDetails) {
|
|
284
|
+
console.error("Could not fetch post details");
|
|
285
|
+
return null;
|
|
286
|
+
}
|
|
287
|
+
let video = null;
|
|
288
|
+
// Check if we have video data
|
|
289
|
+
if (postDetails.video) {
|
|
290
|
+
// Option 1: Direct MP4 URL (preferred - no DRM)
|
|
291
|
+
if (postDetails.video.url?.endsWith(".mp4")) {
|
|
292
|
+
video = {
|
|
293
|
+
type: "custom", // Direct download, not HLS
|
|
294
|
+
url: postDetails.video.url,
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
// Option 2: Get HLS license URL via assetId
|
|
298
|
+
else if (postDetails.video.assetId) {
|
|
299
|
+
const license = await fetchVideoLicense(page, postDetails.video.assetId);
|
|
300
|
+
if (license?.url) {
|
|
301
|
+
video = {
|
|
302
|
+
type: "hls",
|
|
303
|
+
url: license.url,
|
|
304
|
+
masterPlaylistUrl: license.url,
|
|
305
|
+
token: license.token,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
// Fallback: try to extract video from page DOM
|
|
311
|
+
video ??= await extractVideoFromPage(page);
|
|
312
|
+
// Extract HTML content
|
|
313
|
+
const htmlContent = await page.evaluate(() => {
|
|
314
|
+
const contentEl = document.querySelector("[class*='post-content'], [class*='PostContent'], [class*='lesson-content'], article");
|
|
315
|
+
return contentEl?.innerHTML ?? null;
|
|
316
|
+
});
|
|
317
|
+
// Extract text description
|
|
318
|
+
const description = await page.evaluate(() => {
|
|
319
|
+
const descEl = document.querySelector("[class*='description'], [class*='Description'], p:first-of-type");
|
|
320
|
+
return descEl?.textContent?.trim() ?? null;
|
|
321
|
+
});
|
|
322
|
+
return {
|
|
323
|
+
id: postId,
|
|
324
|
+
title: postDetails.title,
|
|
325
|
+
description: description ?? postDetails.description,
|
|
326
|
+
htmlContent,
|
|
327
|
+
video,
|
|
328
|
+
attachments: postDetails.materials.map((m) => ({
|
|
329
|
+
id: m.id,
|
|
330
|
+
name: m.name,
|
|
331
|
+
url: m.url,
|
|
332
|
+
type: m.type,
|
|
333
|
+
})),
|
|
334
|
+
categoryId,
|
|
335
|
+
productId,
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
/* v8 ignore stop */
|
|
339
|
+
// Re-export for backwards compatibility
|
|
340
|
+
export { parseHLSMasterPlaylist };
|
|
341
|
+
/* v8 ignore start */
|
|
342
|
+
/**
|
|
343
|
+
* Fetches and parses HLS playlist to get quality options.
|
|
344
|
+
*/
|
|
345
|
+
export async function getHLSQualities(page, masterPlaylistUrl) {
|
|
346
|
+
try {
|
|
347
|
+
const content = await page.evaluate(async (url) => {
|
|
348
|
+
const res = await fetch(url);
|
|
349
|
+
if (!res.ok)
|
|
350
|
+
return null;
|
|
351
|
+
return res.text();
|
|
352
|
+
}, masterPlaylistUrl);
|
|
353
|
+
if (!content)
|
|
354
|
+
return [];
|
|
355
|
+
return parseHLSMasterPlaylist(content, masterPlaylistUrl);
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
return [];
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Gets the best quality URL from an HLS master playlist.
|
|
363
|
+
*/
|
|
364
|
+
export async function getBestHLSQuality(page, masterPlaylistUrl) {
|
|
365
|
+
const qualities = await getHLSQualities(page, masterPlaylistUrl);
|
|
366
|
+
if (qualities.length === 0) {
|
|
367
|
+
return masterPlaylistUrl;
|
|
368
|
+
}
|
|
369
|
+
// Return highest quality
|
|
370
|
+
return qualities[0]?.url ?? null;
|
|
371
|
+
}
|
|
372
|
+
/* v8 ignore stop */
|
|
373
|
+
//# sourceMappingURL=extractor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/extractor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EACL,uBAAuB,EACvB,yBAAyB,EACzB,0BAA0B,EAC1B,SAAS,GAEV,MAAM,cAAc,CAAC;AAEtB,qDAAqD;AACrD,MAAM,sBAAsB,GAAG,gBAAgB,CAAC;AAkChD,+CAA+C;AAC/C,qBAAqB;AAErB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU;IAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAA2B,EAAE;QAC/D,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAoB,CAAC;QAC/E,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,MAAM,GAAG,SAAS,CAAC,uBAAuB,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;IAC3E,OAAO,MAAM,EAAE,eAAe,CAAC,WAAW,IAAI,IAAI,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAU;IACnD,mDAAmD;IACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACtC,+CAA+C;QAC/C,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC;YAC1C,IAAI,GAAG,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,QAAQ,CAAC,gBAAgB,CAAC,4CAA4C,CAAC,CACxE,CAAC;QACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAI,MAA4B,CAAC,GAAG,CAAC;YAC9C,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;QACtB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,IAAI,EAAE,KAAK;YACX,GAAG,EAAE,MAAM;YACX,iBAAiB,EAAE,MAAM;SAC1B,CAAC;IACJ,CAAC;IAED,wBAAwB;IACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,uDAAuD,CAAC,CAAC;QAC/F,IAAI,MAAM,EAAE,CAAC;YACX,OAAQ,MAA4B,CAAC,GAAG,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,QAAQ;SACd,CAAC;IACJ,CAAC;IAED,uBAAuB;IACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;QACjE,IAAI,MAAM,EAAE,CAAC;YACX,OAAQ,MAA4B,CAAC,GAAG,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,OAAO;SACb,CAAC;IACJ,CAAC;IAED,0BAA0B;IAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CACnC,0FAA0F,CAC3F,CAAC;QACF,IAAI,MAAM,EAAE,CAAC;YACX,OAAQ,MAA4B,CAAC,GAAG,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,EAAE,CAAC;QACf,OAAO;YACL,IAAI,EAAE,SAAS;YACf,GAAG,EAAE,UAAU;SAChB,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAU,EACV,OAAe;IAEf,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,cAAc,GAAG,CAAC,OAA8B,EAAE,EAAE;QACxD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAE1B,uCAAuC;QACvC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;QAED,+BAA+B;QAC/B,IAAI,GAAG,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAEnC,4BAA4B;IAC5B,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;IAChD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAEhC,qBAAqB;IACrB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAEpC,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;IAE7E,IAAI,iBAAiB,EAAE,CAAC;QACtB,OAAO;YACL,IAAI,EAAE,KAAK;YACX,GAAG,EAAE,iBAAiB;YACtB,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAED,6BAA6B;IAC7B,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAU,EACV,UAAkB,EAClB,MAAc;IAkBd,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CACjC,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,EAAwB,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;YACxF,MAAM,SAAS,GAAG,QAAQ;gBACxB,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAqB;gBACzE,CAAC,CAAC,IAAI,CAAC;YACT,MAAM,KAAK,GAAG,SAAS,EAAE,eAAe,EAAE,WAAW,CAAC;YAEtD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;YACpC,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,KAAK,CACrB,6DAA6D,UAAU,UAAU,MAAM,iBAAiB,EACxG;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,KAAK,EAAE;iBACjC;aACF,CACF,CAAC;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;YAC7D,CAAC;YAED,MAAM,IAAI,GAAY,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACvC,OAAO,EAAE,IAAI,EAAE,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,CAAC;IACH,CAAC,EACD,EAAE,UAAU,EAAE,MAAM,EAAE,CACvB,CAAC;IAEF,0CAA0C;IAC1C,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,sBAAsB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oCAAoC;IACpC,MAAM,MAAM,GAAG,SAAS,CAAC,yBAAyB,EAAE,IAAI,EAAE,kBAAkB,CAAC,CAAC;IAC9E,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,yDAAyD;IACzD,yCAAyC;IACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC;IAEnC,IAAI,KAAK,GAA4C,IAAI,CAAC;IAE1D,mCAAmC;IACnC,8DAA8D;IAC9D,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACvF,IAAI,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACnC,KAAK,GAAG;gBACN,OAAO,EAAE,YAAY,IAAI,EAAE;gBAC3B,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE;aAC1B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;QACxC,KAAK,GAAG;YACN,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO;YACjC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,EAAE;SAChC,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE,CAAC;gBACxE,IAAI,YAAY,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;oBAC9B,KAAK,GAAG;wBACN,OAAO,EAAE,YAAY,IAAI,EAAE;wBAC3B,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,EAAE;qBACrB,CAAC;oBACF,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAKT,EAAE,CAAC;IAET,yDAAyD;IACzD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC;IAClE,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;QACrC,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,QAAQ,CAAC,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE;YACtC,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,YAAY;YACnC,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,EAAE;YACvB,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,MAAM;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;QACvB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;QACrC,KAAK;QACL,SAAS;KACV,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAU,EACV,OAAe;IAEf,uBAAuB;IACvB,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,wDAAwD;QACxD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CACrC,iEAAiE,OAAO,EAAE,EAC1E;YACE,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,SAAS,EAAE;aACrC;SACF,CACF,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAY,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE5C,oCAAoC;QACpC,MAAM,MAAM,GAAG,SAAS,CAAC,0BAA0B,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACvD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,IAAU,EACV,OAAe,EACf,UAAkB,EAClB,SAAiB,EACjB,MAAc,EACd,UAAkB;IAElB,wBAAwB;IACxB,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;IAChD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAEhC,8BAA8B;IAC9B,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAErE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,KAAK,GAA8B,IAAI,CAAC;IAE5C,8BAA8B;IAC9B,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;QACtB,gDAAgD;QAChD,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,KAAK,GAAG;gBACN,IAAI,EAAE,QAAQ,EAAE,2BAA2B;gBAC3C,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,GAAG;aAC3B,CAAC;QACJ,CAAC;QACD,4CAA4C;aACvC,IAAI,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAEzE,IAAI,OAAO,EAAE,GAAG,EAAE,CAAC;gBACjB,KAAK,GAAG;oBACN,IAAI,EAAE,KAAK;oBACX,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,iBAAiB,EAAE,OAAO,CAAC,GAAG;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACrB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,KAAK,KAAK,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAE3C,uBAAuB;IACvB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,CACtC,qFAAqF,CACtF,CAAC;QACF,OAAO,SAAS,EAAE,SAAS,IAAI,IAAI,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,2BAA2B;IAC3B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CACnC,iEAAiE,CAClE,CAAC;QACF,OAAO,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,WAAW,EAAE,WAAW,IAAI,WAAW,CAAC,WAAW;QACnD,WAAW;QACX,KAAK;QACL,WAAW,EAAE,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,IAAI,EAAE,CAAC,CAAC,IAAI;SACb,CAAC,CAAC;QACH,UAAU;QACV,SAAS;KACV,CAAC;AACJ,CAAC;AACD,oBAAoB;AAEpB,wCAAwC;AACxC,OAAO,EAAE,sBAAsB,EAAE,CAAC;AAElC,qBAAqB;AACrB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAU,EACV,iBAAyB;IAUzB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAChD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,OAAO,IAAI,CAAC;YACzB,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC,EAAE,iBAAiB,CAAC,CAAC;QAEtB,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,OAAO,sBAAsB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAU,EACV,iBAAyB;IAEzB,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IAEjE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,yBAAyB;IACzB,OAAO,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC;AACnC,CAAC;AACD,oBAAoB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { Page } from "playwright";
|
|
2
|
+
export interface HighLevelCourse {
|
|
3
|
+
id: string;
|
|
4
|
+
title: string;
|
|
5
|
+
description: string;
|
|
6
|
+
slug: string;
|
|
7
|
+
thumbnailUrl: string | null;
|
|
8
|
+
instructor: string | null;
|
|
9
|
+
totalLessons: number;
|
|
10
|
+
progress: number;
|
|
11
|
+
}
|
|
12
|
+
export interface HighLevelCategory {
|
|
13
|
+
id: string;
|
|
14
|
+
title: string;
|
|
15
|
+
description: string | null;
|
|
16
|
+
position: number;
|
|
17
|
+
postCount: number;
|
|
18
|
+
isLocked: boolean;
|
|
19
|
+
}
|
|
20
|
+
export interface HighLevelPost {
|
|
21
|
+
id: string;
|
|
22
|
+
title: string;
|
|
23
|
+
position: number;
|
|
24
|
+
categoryId: string;
|
|
25
|
+
isLocked: boolean;
|
|
26
|
+
isCompleted: boolean;
|
|
27
|
+
}
|
|
28
|
+
export interface HighLevelCourseStructure {
|
|
29
|
+
course: HighLevelCourse;
|
|
30
|
+
categories: (HighLevelCategory & {
|
|
31
|
+
posts: HighLevelPost[];
|
|
32
|
+
})[];
|
|
33
|
+
locationId: string;
|
|
34
|
+
domain: string;
|
|
35
|
+
}
|
|
36
|
+
export interface HighLevelScanProgress {
|
|
37
|
+
phase: "init" | "course" | "categories" | "posts" | "done";
|
|
38
|
+
courseName?: string;
|
|
39
|
+
totalCategories?: number;
|
|
40
|
+
currentCategory?: string;
|
|
41
|
+
currentCategoryIndex?: number;
|
|
42
|
+
postsFound?: number;
|
|
43
|
+
skippedLocked?: boolean;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Extracts the location ID from the HighLevel portal.
|
|
47
|
+
* The location ID is used in all API calls.
|
|
48
|
+
*/
|
|
49
|
+
export declare function extractLocationId(page: Page): Promise<string | null>;
|
|
50
|
+
/**
|
|
51
|
+
* Extracts portal settings including location ID from the API.
|
|
52
|
+
*/
|
|
53
|
+
export declare function extractPortalSettings(page: Page, domain: string): Promise<{
|
|
54
|
+
locationId: string;
|
|
55
|
+
portalName: string;
|
|
56
|
+
} | null>;
|
|
57
|
+
/**
|
|
58
|
+
* Extracts course list from the courses library page.
|
|
59
|
+
*/
|
|
60
|
+
export declare function extractCourses(page: Page): Promise<HighLevelCourse[]>;
|
|
61
|
+
/**
|
|
62
|
+
* Extracts course details from the course overview page via API.
|
|
63
|
+
*/
|
|
64
|
+
export declare function extractCourseDetails(page: Page, courseUrl: string, locationId?: string): Promise<HighLevelCourse | null>;
|
|
65
|
+
/**
|
|
66
|
+
* Extracts categories (modules) from a course page.
|
|
67
|
+
*/
|
|
68
|
+
export declare function extractCategories(page: Page, productId: string, locationId: string): Promise<HighLevelCategory[]>;
|
|
69
|
+
/**
|
|
70
|
+
* Extracts posts (lessons) from a category.
|
|
71
|
+
*/
|
|
72
|
+
export declare function extractPosts(page: Page, productId: string, categoryId: string, locationId: string): Promise<HighLevelPost[]>;
|
|
73
|
+
/**
|
|
74
|
+
* Builds the complete course structure.
|
|
75
|
+
*/
|
|
76
|
+
export declare function buildHighLevelCourseStructure(page: Page, courseUrl: string, onProgress?: (progress: HighLevelScanProgress) => void): Promise<HighLevelCourseStructure | null>;
|
|
77
|
+
export { slugify, createFolderName } from "../../shared/slug.js";
|
|
78
|
+
/**
|
|
79
|
+
* Constructs the URL for a HighLevel course page.
|
|
80
|
+
*/
|
|
81
|
+
export declare function getHighLevelCourseUrl(domain: string, productId: string): string;
|
|
82
|
+
/**
|
|
83
|
+
* Constructs the URL for a HighLevel lesson (post) page.
|
|
84
|
+
*/
|
|
85
|
+
export declare function getHighLevelPostUrl(domain: string, productId: string, categoryId: string, postId: string): string;
|
|
86
|
+
//# sourceMappingURL=navigator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"navigator.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/navigator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAWvC,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,CAAC,iBAAiB,GAAG;QAAE,KAAK,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC,EAAE,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAKD;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgC1E;AAED;;GAEG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAwB5D;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA6C3E;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CA4IjC;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAwD9B;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,EAAE,CAAC,CAsD1B;AAED;;GAEG;AACH,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,GACrD,OAAO,CAAC,wBAAwB,GAAG,IAAI,CAAC,CAmL1C;AAID,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAEjE;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAE/E;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,MAAM,CAER"}
|