offcourse 0.0.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +50 -0
- package/.husky/commit-msg +2 -0
- package/.husky/pre-commit +1 -0
- package/.husky/pre-push +3 -0
- package/.prettierrc +8 -0
- package/.release-it.json +23 -0
- package/ARCHITECTURE.md +233 -0
- package/CHANGELOG.md +78 -0
- package/README.md +256 -16
- package/commitlint.config.js +4 -0
- package/dist/ai/openRouter.d.ts +47 -0
- package/dist/ai/openRouter.d.ts.map +1 -0
- package/dist/ai/openRouter.js +116 -0
- package/dist/ai/openRouter.js.map +1 -0
- package/dist/ai/transcriptPolisher.d.ts +24 -0
- package/dist/ai/transcriptPolisher.d.ts.map +1 -0
- package/dist/ai/transcriptPolisher.js +89 -0
- package/dist/ai/transcriptPolisher.js.map +1 -0
- package/dist/cli/commands/config.d.ts +13 -0
- package/dist/cli/commands/config.d.ts.map +1 -0
- package/dist/cli/commands/config.js +66 -0
- package/dist/cli/commands/config.js.map +1 -0
- package/dist/cli/commands/enrich.d.ts +14 -0
- package/dist/cli/commands/enrich.d.ts.map +1 -0
- package/dist/cli/commands/enrich.js +271 -0
- package/dist/cli/commands/enrich.js.map +1 -0
- package/dist/cli/commands/inspect.d.ts +11 -0
- package/dist/cli/commands/inspect.d.ts.map +1 -0
- package/dist/cli/commands/inspect.js +365 -0
- package/dist/cli/commands/inspect.js.map +1 -0
- package/dist/cli/commands/login.d.ts +12 -0
- package/dist/cli/commands/login.d.ts.map +1 -0
- package/dist/cli/commands/login.js +55 -0
- package/dist/cli/commands/login.js.map +1 -0
- package/dist/cli/commands/status.d.ts +15 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +118 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/sync.d.ts +16 -0
- package/dist/cli/commands/sync.d.ts.map +1 -0
- package/dist/cli/commands/sync.js +922 -0
- package/dist/cli/commands/sync.js.map +1 -0
- package/dist/cli/commands/syncGhl.d.ts +20 -0
- package/dist/cli/commands/syncGhl.d.ts.map +1 -0
- package/dist/cli/commands/syncGhl.js +483 -0
- package/dist/cli/commands/syncGhl.js.map +1 -0
- package/dist/cli/commands/syncHighLevel.d.ts +24 -0
- package/dist/cli/commands/syncHighLevel.d.ts.map +1 -0
- package/dist/cli/commands/syncHighLevel.js +483 -0
- package/dist/cli/commands/syncHighLevel.js.map +1 -0
- package/dist/cli/commands/syncHighLevel.test.d.ts +2 -0
- package/dist/cli/commands/syncHighLevel.test.d.ts.map +1 -0
- package/dist/cli/commands/syncHighLevel.test.js +102 -0
- package/dist/cli/commands/syncHighLevel.test.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +106 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/config/configManager.d.ts +31 -0
- package/dist/config/configManager.d.ts.map +1 -0
- package/dist/config/configManager.js +64 -0
- package/dist/config/configManager.js.map +1 -0
- package/dist/config/paths.d.ts +21 -0
- package/dist/config/paths.d.ts.map +1 -0
- package/dist/config/paths.js +33 -0
- package/dist/config/paths.js.map +1 -0
- package/dist/config/paths.test.d.ts +2 -0
- package/dist/config/paths.test.d.ts.map +1 -0
- package/dist/config/paths.test.js +70 -0
- package/dist/config/paths.test.js.map +1 -0
- package/dist/config/schema.d.ts +60 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +50 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/schema.test.d.ts +2 -0
- package/dist/config/schema.test.d.ts.map +1 -0
- package/dist/config/schema.test.js +151 -0
- package/dist/config/schema.test.js.map +1 -0
- package/dist/downloader/hlsDownloader.d.ts +58 -0
- package/dist/downloader/hlsDownloader.d.ts.map +1 -0
- package/dist/downloader/hlsDownloader.js +254 -0
- package/dist/downloader/hlsDownloader.js.map +1 -0
- package/dist/downloader/hlsDownloader.test.d.ts +2 -0
- package/dist/downloader/hlsDownloader.test.d.ts.map +1 -0
- package/dist/downloader/hlsDownloader.test.js +116 -0
- package/dist/downloader/hlsDownloader.test.js.map +1 -0
- package/dist/downloader/hlsValidator.d.ts +35 -0
- package/dist/downloader/hlsValidator.d.ts.map +1 -0
- package/dist/downloader/hlsValidator.js +148 -0
- package/dist/downloader/hlsValidator.js.map +1 -0
- package/dist/downloader/index.d.ts +26 -0
- package/dist/downloader/index.d.ts.map +1 -0
- package/dist/downloader/index.js +52 -0
- package/dist/downloader/index.js.map +1 -0
- package/dist/downloader/loomDownloader.d.ts +56 -0
- package/dist/downloader/loomDownloader.d.ts.map +1 -0
- package/dist/downloader/loomDownloader.js +559 -0
- package/dist/downloader/loomDownloader.js.map +1 -0
- package/dist/downloader/loomDownloader.test.d.ts +2 -0
- package/dist/downloader/loomDownloader.test.d.ts.map +1 -0
- package/dist/downloader/loomDownloader.test.js +36 -0
- package/dist/downloader/loomDownloader.test.js.map +1 -0
- package/dist/downloader/queue.d.ts +56 -0
- package/dist/downloader/queue.d.ts.map +1 -0
- package/dist/downloader/queue.js +88 -0
- package/dist/downloader/queue.js.map +1 -0
- package/dist/downloader/queue.test.d.ts +2 -0
- package/dist/downloader/queue.test.d.ts.map +1 -0
- package/dist/downloader/queue.test.js +158 -0
- package/dist/downloader/queue.test.js.map +1 -0
- package/dist/downloader/videoDownloader.d.ts +32 -0
- package/dist/downloader/videoDownloader.d.ts.map +1 -0
- package/dist/downloader/videoDownloader.js +173 -0
- package/dist/downloader/videoDownloader.js.map +1 -0
- package/dist/downloader/vimeoDownloader.d.ts +52 -0
- package/dist/downloader/vimeoDownloader.d.ts.map +1 -0
- package/dist/downloader/vimeoDownloader.js +565 -0
- package/dist/downloader/vimeoDownloader.js.map +1 -0
- package/dist/downloader/vimeoDownloader.test.d.ts +2 -0
- package/dist/downloader/vimeoDownloader.test.d.ts.map +1 -0
- package/dist/downloader/vimeoDownloader.test.js +51 -0
- package/dist/downloader/vimeoDownloader.test.js.map +1 -0
- package/dist/scraper/auth.d.ts +29 -0
- package/dist/scraper/auth.d.ts.map +1 -0
- package/dist/scraper/auth.js +115 -0
- package/dist/scraper/auth.js.map +1 -0
- package/dist/scraper/extractor.d.ts +49 -0
- package/dist/scraper/extractor.d.ts.map +1 -0
- package/dist/scraper/extractor.js +627 -0
- package/dist/scraper/extractor.js.map +1 -0
- package/dist/scraper/extractor.test.d.ts +2 -0
- package/dist/scraper/extractor.test.d.ts.map +1 -0
- package/dist/scraper/extractor.test.js +65 -0
- package/dist/scraper/extractor.test.js.map +1 -0
- package/dist/scraper/ghl/auth.d.ts +25 -0
- package/dist/scraper/ghl/auth.d.ts.map +1 -0
- package/dist/scraper/ghl/auth.js +187 -0
- package/dist/scraper/ghl/auth.js.map +1 -0
- package/dist/scraper/ghl/extractor.d.ts +96 -0
- package/dist/scraper/ghl/extractor.d.ts.map +1 -0
- package/dist/scraper/ghl/extractor.js +345 -0
- package/dist/scraper/ghl/extractor.js.map +1 -0
- package/dist/scraper/ghl/index.d.ts +4 -0
- package/dist/scraper/ghl/index.d.ts.map +1 -0
- package/dist/scraper/ghl/index.js +4 -0
- package/dist/scraper/ghl/index.js.map +1 -0
- package/dist/scraper/ghl/navigator.d.ts +93 -0
- package/dist/scraper/ghl/navigator.d.ts.map +1 -0
- package/dist/scraper/ghl/navigator.js +447 -0
- package/dist/scraper/ghl/navigator.js.map +1 -0
- package/dist/scraper/highlevel/auth.d.ts +25 -0
- package/dist/scraper/highlevel/auth.d.ts.map +1 -0
- package/dist/scraper/highlevel/auth.js +189 -0
- package/dist/scraper/highlevel/auth.js.map +1 -0
- package/dist/scraper/highlevel/extractor.d.ts +97 -0
- package/dist/scraper/highlevel/extractor.d.ts.map +1 -0
- package/dist/scraper/highlevel/extractor.js +386 -0
- package/dist/scraper/highlevel/extractor.js.map +1 -0
- package/dist/scraper/highlevel/extractor.test.d.ts +2 -0
- package/dist/scraper/highlevel/extractor.test.d.ts.map +1 -0
- package/dist/scraper/highlevel/extractor.test.js +101 -0
- package/dist/scraper/highlevel/extractor.test.js.map +1 -0
- package/dist/scraper/highlevel/index.d.ts +3 -0
- package/dist/scraper/highlevel/index.d.ts.map +1 -0
- package/dist/scraper/highlevel/index.js +3 -0
- package/dist/scraper/highlevel/index.js.map +1 -0
- package/dist/scraper/highlevel/navigator.d.ts +93 -0
- package/dist/scraper/highlevel/navigator.d.ts.map +1 -0
- package/dist/scraper/highlevel/navigator.js +492 -0
- package/dist/scraper/highlevel/navigator.js.map +1 -0
- package/dist/scraper/highlevel/navigator.test.d.ts +2 -0
- package/dist/scraper/highlevel/navigator.test.d.ts.map +1 -0
- package/dist/scraper/highlevel/navigator.test.js +78 -0
- package/dist/scraper/highlevel/navigator.test.js.map +1 -0
- package/dist/scraper/navigator.d.ts +65 -0
- package/dist/scraper/navigator.d.ts.map +1 -0
- package/dist/scraper/navigator.js +300 -0
- package/dist/scraper/navigator.js.map +1 -0
- package/dist/scraper/navigator.test.d.ts +2 -0
- package/dist/scraper/navigator.test.d.ts.map +1 -0
- package/dist/scraper/navigator.test.js +63 -0
- package/dist/scraper/navigator.test.js.map +1 -0
- package/dist/scraper/skoolApi.d.ts +17 -0
- package/dist/scraper/skoolApi.d.ts.map +1 -0
- package/dist/scraper/skoolApi.js +72 -0
- package/dist/scraper/skoolApi.js.map +1 -0
- package/dist/scraper/videoInterceptor.d.ts +19 -0
- package/dist/scraper/videoInterceptor.d.ts.map +1 -0
- package/dist/scraper/videoInterceptor.js +315 -0
- package/dist/scraper/videoInterceptor.js.map +1 -0
- package/dist/shared/auth.d.ts +58 -0
- package/dist/shared/auth.d.ts.map +1 -0
- package/dist/shared/auth.js +211 -0
- package/dist/shared/auth.js.map +1 -0
- package/dist/shared/fs.d.ts +31 -0
- package/dist/shared/fs.d.ts.map +1 -0
- package/dist/shared/fs.js +73 -0
- package/dist/shared/fs.js.map +1 -0
- package/dist/shared/http.d.ts +15 -0
- package/dist/shared/http.d.ts.map +1 -0
- package/dist/shared/http.js +31 -0
- package/dist/shared/http.js.map +1 -0
- package/dist/shared/index.d.ts +4 -0
- package/dist/shared/index.d.ts.map +1 -0
- package/dist/shared/index.js +4 -0
- package/dist/shared/index.js.map +1 -0
- package/dist/state/database.d.ts +245 -0
- package/dist/state/database.d.ts.map +1 -0
- package/dist/state/database.js +676 -0
- package/dist/state/database.js.map +1 -0
- package/dist/state/database.test.d.ts +2 -0
- package/dist/state/database.test.d.ts.map +1 -0
- package/dist/state/database.test.js +34 -0
- package/dist/state/database.test.js.map +1 -0
- package/dist/state/index.d.ts +2 -0
- package/dist/state/index.d.ts.map +1 -0
- package/dist/state/index.js +2 -0
- package/dist/state/index.js.map +1 -0
- package/dist/storage/fileSystem.d.ts +56 -0
- package/dist/storage/fileSystem.d.ts.map +1 -0
- package/dist/storage/fileSystem.js +121 -0
- package/dist/storage/fileSystem.js.map +1 -0
- package/dist/transcription/whisperService.d.ts +27 -0
- package/dist/transcription/whisperService.d.ts.map +1 -0
- package/dist/transcription/whisperService.js +102 -0
- package/dist/transcription/whisperService.js.map +1 -0
- package/eslint.config.js +55 -0
- package/package.json +68 -11
- package/src/__fixtures__/highlevel-post-response.json +68 -0
- package/src/__fixtures__/hls-master-playlist.m3u8 +24 -0
- package/src/cli/commands/__snapshots__/syncHighLevel.test.ts.snap +38 -0
- package/src/cli/commands/config.ts +74 -0
- package/src/cli/commands/inspect.ts +441 -0
- package/src/cli/commands/login.ts +68 -0
- package/src/cli/commands/status.ts +147 -0
- package/src/cli/commands/sync.ts +1235 -0
- package/src/cli/commands/syncHighLevel.test.ts +144 -0
- package/src/cli/commands/syncHighLevel.ts +639 -0
- package/src/cli/index.ts +121 -0
- package/src/config/configManager.ts +75 -0
- package/src/config/paths.test.ts +83 -0
- package/src/config/paths.ts +36 -0
- package/src/config/schema.test.ts +173 -0
- package/src/config/schema.ts +65 -0
- package/src/downloader/hlsDownloader.test.ts +148 -0
- package/src/downloader/hlsDownloader.ts +327 -0
- package/src/downloader/hlsValidator.ts +196 -0
- package/src/downloader/index.ts +122 -0
- package/src/downloader/loomDownloader.test.ts +43 -0
- package/src/downloader/loomDownloader.ts +742 -0
- package/src/downloader/queue.test.ts +199 -0
- package/src/downloader/queue.ts +118 -0
- package/src/downloader/vimeoDownloader.test.ts +62 -0
- package/src/downloader/vimeoDownloader.ts +722 -0
- package/src/scraper/extractor.test.ts +124 -0
- package/src/scraper/extractor.ts +757 -0
- package/src/scraper/highlevel/__snapshots__/extractor.test.ts.snap +41 -0
- package/src/scraper/highlevel/extractor.test.ts +134 -0
- package/src/scraper/highlevel/extractor.ts +537 -0
- package/src/scraper/highlevel/index.ts +2 -0
- package/src/scraper/highlevel/navigator.test.ts +110 -0
- package/src/scraper/highlevel/navigator.ts +668 -0
- package/src/scraper/highlevel/schemas.ts +183 -0
- package/src/scraper/navigator.test.ts +122 -0
- package/src/scraper/navigator.ts +355 -0
- package/src/scraper/schemas.ts +177 -0
- package/src/scraper/videoInterceptor.ts +435 -0
- package/src/shared/auth.test.ts +58 -0
- package/src/shared/auth.ts +251 -0
- package/src/shared/firebase.ts +151 -0
- package/src/shared/fs.ts +80 -0
- package/src/shared/http.ts +34 -0
- package/src/shared/index.ts +6 -0
- package/src/shared/slug.ts +26 -0
- package/src/shared/url.test.ts +122 -0
- package/src/shared/url.ts +57 -0
- package/src/state/database.test.ts +49 -0
- package/src/state/database.ts +919 -0
- package/src/state/index.ts +14 -0
- package/src/storage/fileSystem.test.ts +64 -0
- package/src/storage/fileSystem.ts +175 -0
- package/tsconfig.json +28 -0
- package/vitest.config.ts +29 -0
- package/cli.js +0 -45
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/extractor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,GAAG,MAAM,YAAY,CAAC;AAkClC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACxB,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;QACrE,OAAO,SAAS,EAAE,eAAe,EAAE,WAAW,IAAI,IAAI,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAU;IACnD,mDAAmD;IACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACtC,+CAA+C;QAC/C,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC;YAC1C,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACjC,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,QAAQ,CAAC,gBAAgB,CAAC,4CAA4C,CAAC,CACxE,CAAC;QACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAI,MAA4B,CAAC,GAAG,CAAC;YAC9C,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;QACtB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,IAAI,EAAE,KAAK;YACX,GAAG,EAAE,MAAM;YACX,iBAAiB,EAAE,MAAM;SAC1B,CAAC;IACJ,CAAC;IAED,wBAAwB;IACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,uDAAuD,CAAC,CAAC;QAC/F,IAAI,MAAM,EAAE,CAAC;YACX,OAAQ,MAA4B,CAAC,GAAG,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,QAAQ;SACd,CAAC;IACJ,CAAC;IAED,uBAAuB;IACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;QACjE,IAAI,MAAM,EAAE,CAAC;YACX,OAAQ,MAA4B,CAAC,GAAG,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,OAAO;SACb,CAAC;IACJ,CAAC;IAED,0BAA0B;IAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CACnC,0FAA0F,CAC3F,CAAC;QACF,IAAI,MAAM,EAAE,CAAC;YACX,OAAQ,MAA4B,CAAC,GAAG,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,EAAE,CAAC;QACf,OAAO;YACL,IAAI,EAAE,SAAS;YACf,GAAG,EAAE,UAAU;SAChB,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAU,EACV,OAAe;IAEf,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,cAAc,GAAG,CAAC,OAA8B,EAAE,EAAE;QACxD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAE1B,uCAAuC;QACvC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;QAED,+BAA+B;QAC/B,IAAI,GAAG,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAEnC,4BAA4B;IAC5B,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;IAChD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAEhC,qBAAqB;IACrB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAEpC,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;IAE7E,IAAI,iBAAiB,EAAE,CAAC;QACtB,OAAO;YACL,IAAI,EAAE,KAAK;YACX,GAAG,EAAE,iBAAiB;YACtB,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAED,6BAA6B;IAC7B,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAU,EACV,UAAkB,EAClB,MAAc;IAed,sCAAsC;IACtC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CACjC,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE;QAC/B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;YACxF,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACvF,MAAM,KAAK,GAAG,SAAS,EAAE,eAAe,EAAE,WAAW,CAAC;YAEtD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;YACpC,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,KAAK,CACrB,6DAA6D,UAAU,UAAU,MAAM,iBAAiB,EACxG;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,KAAK,EAAE;iBACjC;aACF,CACF,CAAC;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;YAC7D,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,OAAO,EAAE,IAAI,EAAE,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,CAAC;IACH,CAAC,EACD,EAAE,UAAU,EAAE,MAAM,EAAE,CACvB,CAAC;IAEF,0CAA0C;IAC1C,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,sBAAsB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;IAC3B,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,yDAAyD;IACzD,yCAAyC;IACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;IAE/B,IAAI,KAAK,GAA4C,IAAI,CAAC;IAE1D,mCAAmC;IACnC,8DAA8D;IAC9D,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACvF,IAAI,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACnC,KAAK,GAAG;gBACN,OAAO,EAAE,YAAY,IAAI,EAAE;gBAC3B,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE;aAC1B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;QACxC,KAAK,GAAG;YACN,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO;YACjC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,EAAE;SAChC,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE,CAAC;gBACxE,IAAI,YAAY,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;oBAC9B,KAAK,GAAG;wBACN,OAAO,EAAE,YAAY,IAAI,EAAE;wBAC3B,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,EAAE;qBACrB,CAAC;oBACF,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAKV,EAAE,CAAC;IAER,yDAAyD;IACzD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC;IAClE,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACjC,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,QAAQ,CAAC,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE;gBACtC,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,YAAY;gBACnC,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,EAAE;gBACvB,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,MAAM;aAC9B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;QACvB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;QACrC,KAAK;QACL,SAAS;KACV,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAU,EACV,OAAe;IAEf,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QACrC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;YACxF,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACvF,MAAM,KAAK,GAAG,SAAS,EAAE,eAAe,EAAE,WAAW,CAAC;YAEtD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,KAAK,CACrB,iEAAiE,OAAO,EAAE,EAC1E;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,KAAK,EAAE;iBACjC;aACF,CACF,CAAC;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAE9B,OAAO;gBACL,GAAG,EAAE,IAAI,CAAC,GAAG,IAAI,EAAE;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,EAAE,OAAO,CAAC,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,IAAU,EACV,OAAe,EACf,UAAkB,EAClB,SAAiB,EACjB,MAAc,EACd,UAAkB;IAElB,wBAAwB;IACxB,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;IAChD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAEhC,8BAA8B;IAC9B,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAErE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,KAAK,GAA8B,IAAI,CAAC;IAE5C,8BAA8B;IAC9B,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;QACtB,gDAAgD;QAChD,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACpE,KAAK,GAAG;gBACN,IAAI,EAAE,QAAQ,EAAE,2BAA2B;gBAC3C,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,GAAG;aAC3B,CAAC;QACJ,CAAC;QACD,4CAA4C;aACvC,IAAI,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAEzE,IAAI,OAAO,EAAE,GAAG,EAAE,CAAC;gBACjB,KAAK,GAAG;oBACN,IAAI,EAAE,KAAK;oBACX,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,iBAAiB,EAAE,OAAO,CAAC,GAAG;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACrB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED,uBAAuB;IACvB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,CACtC,qFAAqF,CACtF,CAAC;QACF,OAAO,SAAS,EAAE,SAAS,IAAI,IAAI,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,2BAA2B;IAC3B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CACnC,iEAAiE,CAClE,CAAC;QACF,OAAO,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,WAAW,EAAE,WAAW,IAAI,WAAW,CAAC,WAAW;QACnD,WAAW;QACX,KAAK;QACL,WAAW,EAAE,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,IAAI,EAAE,CAAC,CAAC,IAAI;SACb,CAAC,CAAC;QACH,UAAU;QACV,SAAS;KACV,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,OAAe,EACf,OAAe;IAQf,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEpC,gDAAgD;QAChD,IAAI,CAAC,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACpD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACjD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;YACzC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;YACtC,MAAM,KAAK,GAAG,UAAU,EAAE,KAAK,CAAC;YAChC,MAAM,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC;YAElC,qBAAqB;YACrB,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC/C,CAAC,CAAC,OAAO,CAAC,GAAG;gBACb,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAEvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC;YAEzE,OAAO;gBACL,KAAK;gBACL,GAAG,EAAE,UAAU;gBACf,SAAS;gBACT,KAAK;gBACL,MAAM;aACP,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,oCAAoC;QACpC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QAEnD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAU,EACV,iBAAyB;IAUzB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAChD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,OAAO,IAAI,CAAC;YACzB,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC,EAAE,iBAAiB,CAAC,CAAC;QAEtB,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,OAAO,sBAAsB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAU,EACV,iBAAyB;IAEzB,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IAEjE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,yBAAyB;IACzB,OAAO,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC;AACnC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractor.test.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/extractor.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { join, dirname } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { describe, expect, it } from "vitest";
|
|
5
|
+
import { parseHLSMasterPlaylist } from "./extractor.js";
|
|
6
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
7
|
+
const fixturesDir = join(__dirname, "../../__fixtures__");
|
|
8
|
+
describe("parseHLSMasterPlaylist", () => {
|
|
9
|
+
const baseUrl = "https://cdn.example.com/video/";
|
|
10
|
+
it("parses master playlist with multiple qualities", () => {
|
|
11
|
+
const content = `#EXTM3U
|
|
12
|
+
#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360
|
|
13
|
+
360p.m3u8
|
|
14
|
+
#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
|
|
15
|
+
720p.m3u8
|
|
16
|
+
#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=1920x1080
|
|
17
|
+
1080p.m3u8`;
|
|
18
|
+
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
19
|
+
expect(result).toHaveLength(3);
|
|
20
|
+
// Should be sorted by bandwidth (highest first)
|
|
21
|
+
expect(result[0].label).toBe("1080p");
|
|
22
|
+
expect(result[0].bandwidth).toBe(5000000);
|
|
23
|
+
expect(result[0].height).toBe(1080);
|
|
24
|
+
expect(result[0].width).toBe(1920);
|
|
25
|
+
});
|
|
26
|
+
it("resolves relative URLs correctly", () => {
|
|
27
|
+
const content = `#EXTM3U
|
|
28
|
+
#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
|
|
29
|
+
720p/index.m3u8`;
|
|
30
|
+
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
31
|
+
expect(result[0].url).toBe("https://cdn.example.com/video/720p/index.m3u8");
|
|
32
|
+
});
|
|
33
|
+
it("preserves absolute URLs", () => {
|
|
34
|
+
const content = `#EXTM3U
|
|
35
|
+
#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
|
|
36
|
+
https://other-cdn.com/720p.m3u8`;
|
|
37
|
+
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
38
|
+
expect(result[0].url).toBe("https://other-cdn.com/720p.m3u8");
|
|
39
|
+
});
|
|
40
|
+
it("handles playlist without resolution (audio only)", () => {
|
|
41
|
+
const content = `#EXTM3U
|
|
42
|
+
#EXT-X-STREAM-INF:BANDWIDTH=128000
|
|
43
|
+
audio.m3u8`;
|
|
44
|
+
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
45
|
+
expect(result).toHaveLength(1);
|
|
46
|
+
expect(result[0].label).toBe("128k");
|
|
47
|
+
expect(result[0].height).toBeUndefined();
|
|
48
|
+
});
|
|
49
|
+
it("returns empty array for empty playlist", () => {
|
|
50
|
+
const content = `#EXTM3U
|
|
51
|
+
#EXT-X-VERSION:3`;
|
|
52
|
+
const result = parseHLSMasterPlaylist(content, baseUrl);
|
|
53
|
+
expect(result).toHaveLength(0);
|
|
54
|
+
});
|
|
55
|
+
describe("with fixture file", () => {
|
|
56
|
+
it("parses real-world HLS master playlist", () => {
|
|
57
|
+
const content = readFileSync(join(fixturesDir, "hls-master-playlist.m3u8"), "utf-8");
|
|
58
|
+
const result = parseHLSMasterPlaylist(content, "https://cdn.example.com/video/");
|
|
59
|
+
expect(result).toHaveLength(5);
|
|
60
|
+
// Sorted by bandwidth (highest first)
|
|
61
|
+
expect(result[0].label).toBe("1080p");
|
|
62
|
+
expect(result[0].height).toBe(1080);
|
|
63
|
+
expect(result[0].bandwidth).toBe(6000000);
|
|
64
|
+
expect(result[4].label).toBe("240p");
|
|
65
|
+
expect(result[4].height).toBe(240);
|
|
66
|
+
});
|
|
67
|
+
it("snapshot: parsed playlist structure", () => {
|
|
68
|
+
const content = readFileSync(join(fixturesDir, "hls-master-playlist.m3u8"), "utf-8");
|
|
69
|
+
const result = parseHLSMasterPlaylist(content, "https://cdn.example.com/video/");
|
|
70
|
+
expect(result).toMatchSnapshot();
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
describe("HighLevel API Response parsing", () => {
|
|
75
|
+
it("snapshot: fixture structure", () => {
|
|
76
|
+
const response = JSON.parse(readFileSync(join(fixturesDir, "highlevel-post-response.json"), "utf-8"));
|
|
77
|
+
// Verify the structure we depend on
|
|
78
|
+
expect(response).toHaveProperty("id");
|
|
79
|
+
expect(response).toHaveProperty("title");
|
|
80
|
+
expect(response).toHaveProperty("video");
|
|
81
|
+
expect(response).toHaveProperty("post_materials");
|
|
82
|
+
// Video structure
|
|
83
|
+
expect(response.video).toHaveProperty("url");
|
|
84
|
+
expect(response.video).toHaveProperty("assetsLicenseId");
|
|
85
|
+
expect(response.video.url).toMatch(/\.mp4$/);
|
|
86
|
+
// Materials structure
|
|
87
|
+
expect(response.post_materials).toHaveLength(2);
|
|
88
|
+
expect(response.post_materials[0]).toHaveProperty("name");
|
|
89
|
+
expect(response.post_materials[0]).toHaveProperty("url");
|
|
90
|
+
});
|
|
91
|
+
it("extracts video info from response", () => {
|
|
92
|
+
const response = JSON.parse(readFileSync(join(fixturesDir, "highlevel-post-response.json"), "utf-8"));
|
|
93
|
+
// Simulate the extraction logic
|
|
94
|
+
const video = response.video;
|
|
95
|
+
const assetId = video?.assetsLicenseId ?? video?.assetId ?? video?.id;
|
|
96
|
+
const directUrl = video?.url;
|
|
97
|
+
expect(assetId).toBe("689a0d672ea246086539f453");
|
|
98
|
+
expect(directUrl).toContain("1080p.mp4");
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
//# sourceMappingURL=extractor.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractor.test.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/extractor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;AAE1D,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,MAAM,OAAO,GAAG,gCAAgC,CAAC;IAEjD,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,OAAO,GAAG;;;;;;WAMT,CAAC;QAER,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,gDAAgD;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,OAAO,GAAG;;gBAEJ,CAAC;QAEb,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,OAAO,GAAG;;gCAEY,CAAC;QAE7B,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,OAAO,GAAG;;WAET,CAAC;QAER,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,OAAO,GAAG;iBACH,CAAC;QAEd,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,CAAC,EAAE,OAAO,CAAC,CAAC;YAErF,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,gCAAgC,CAAC,CAAC;YAEjF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAE/B,sCAAsC;YACtC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,CAAC,EAAE,OAAO,CAAC,CAAC;YACrF,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,gCAAgC,CAAC,CAAC;YAEjF,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;IAC9C,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,8BAA8B,CAAC,EAAE,OAAO,CAAC,CACzE,CAAC;QAEF,oCAAoC;QACpC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,CAAC,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAElD,kBAAkB;QAClB,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE7C,sBAAsB;QACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,8BAA8B,CAAC,EAAE,OAAO,CAAC,CACzE,CAAC;QAEF,gCAAgC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC7B,MAAM,OAAO,GAAG,KAAK,EAAE,eAAe,IAAI,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,KAAK,EAAE,GAAG,CAAC;QAE7B,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACjD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scraper/highlevel/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { Page } from "playwright";
|
|
2
|
+
export interface HighLevelCourse {
|
|
3
|
+
id: string;
|
|
4
|
+
title: string;
|
|
5
|
+
description: string;
|
|
6
|
+
slug: string;
|
|
7
|
+
thumbnailUrl: string | null;
|
|
8
|
+
instructor: string | null;
|
|
9
|
+
totalLessons: number;
|
|
10
|
+
progress: number;
|
|
11
|
+
}
|
|
12
|
+
export interface HighLevelCategory {
|
|
13
|
+
id: string;
|
|
14
|
+
title: string;
|
|
15
|
+
description: string | null;
|
|
16
|
+
position: number;
|
|
17
|
+
postCount: number;
|
|
18
|
+
isLocked: boolean;
|
|
19
|
+
}
|
|
20
|
+
export interface HighLevelPost {
|
|
21
|
+
id: string;
|
|
22
|
+
title: string;
|
|
23
|
+
position: number;
|
|
24
|
+
categoryId: string;
|
|
25
|
+
isLocked: boolean;
|
|
26
|
+
isCompleted: boolean;
|
|
27
|
+
}
|
|
28
|
+
export interface HighLevelCourseStructure {
|
|
29
|
+
course: HighLevelCourse;
|
|
30
|
+
categories: Array<HighLevelCategory & {
|
|
31
|
+
posts: HighLevelPost[];
|
|
32
|
+
}>;
|
|
33
|
+
locationId: string;
|
|
34
|
+
domain: string;
|
|
35
|
+
}
|
|
36
|
+
export interface HighLevelScanProgress {
|
|
37
|
+
phase: "init" | "course" | "categories" | "posts" | "done";
|
|
38
|
+
courseName?: string;
|
|
39
|
+
totalCategories?: number;
|
|
40
|
+
currentCategory?: string;
|
|
41
|
+
currentCategoryIndex?: number;
|
|
42
|
+
postsFound?: number;
|
|
43
|
+
skippedLocked?: boolean;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Extracts the location ID from the HighLevel portal.
|
|
47
|
+
* The location ID is used in all API calls.
|
|
48
|
+
*/
|
|
49
|
+
export declare function extractLocationId(page: Page): Promise<string | null>;
|
|
50
|
+
/**
|
|
51
|
+
* Extracts portal settings including location ID from the API.
|
|
52
|
+
*/
|
|
53
|
+
export declare function extractPortalSettings(page: Page, domain: string): Promise<{
|
|
54
|
+
locationId: string;
|
|
55
|
+
portalName: string;
|
|
56
|
+
} | null>;
|
|
57
|
+
/**
|
|
58
|
+
* Extracts course list from the courses library page.
|
|
59
|
+
*/
|
|
60
|
+
export declare function extractCourses(page: Page): Promise<HighLevelCourse[]>;
|
|
61
|
+
/**
|
|
62
|
+
* Extracts course details from the course overview page via API.
|
|
63
|
+
*/
|
|
64
|
+
export declare function extractCourseDetails(page: Page, courseUrl: string, locationId?: string): Promise<HighLevelCourse | null>;
|
|
65
|
+
/**
|
|
66
|
+
* Extracts categories (modules) from a course page.
|
|
67
|
+
*/
|
|
68
|
+
export declare function extractCategories(page: Page, productId: string, locationId: string): Promise<HighLevelCategory[]>;
|
|
69
|
+
/**
|
|
70
|
+
* Extracts posts (lessons) from a category.
|
|
71
|
+
*/
|
|
72
|
+
export declare function extractPosts(page: Page, productId: string, categoryId: string, locationId: string): Promise<HighLevelPost[]>;
|
|
73
|
+
/**
|
|
74
|
+
* Builds the complete course structure.
|
|
75
|
+
*/
|
|
76
|
+
export declare function buildHighLevelCourseStructure(page: Page, courseUrl: string, onProgress?: (progress: HighLevelScanProgress) => void): Promise<HighLevelCourseStructure | null>;
|
|
77
|
+
/**
|
|
78
|
+
* Creates a filesystem-safe name from a string.
|
|
79
|
+
*/
|
|
80
|
+
export declare function slugify(name: string): string;
|
|
81
|
+
/**
|
|
82
|
+
* Creates a folder name with index prefix.
|
|
83
|
+
*/
|
|
84
|
+
export declare function createFolderName(index: number, name: string): string;
|
|
85
|
+
/**
|
|
86
|
+
* Constructs the URL for a HighLevel course page.
|
|
87
|
+
*/
|
|
88
|
+
export declare function getHighLevelCourseUrl(domain: string, productId: string): string;
|
|
89
|
+
/**
|
|
90
|
+
* Constructs the URL for a HighLevel lesson (post) page.
|
|
91
|
+
*/
|
|
92
|
+
export declare function getHighLevelPostUrl(domain: string, productId: string, categoryId: string, postId: string): string;
|
|
93
|
+
//# sourceMappingURL=navigator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"navigator.d.ts","sourceRoot":"","sources":["../../../src/scraper/highlevel/navigator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,KAAK,CAAC,iBAAiB,GAAG;QAAE,KAAK,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC,CAAC;IAClE,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgC1E;AAED;;GAEG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAsB5D;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA6C3E;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CA8HjC;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CA6D9B;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,EAAE,CAAC,CA2D1B;AAED;;GAEG;AACH,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,GACrD,OAAO,CAAC,wBAAwB,GAAG,IAAI,CAAC,CAqL1C;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAe5C;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAIpE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAE/E;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,MAAM,CAER"}
|