@mintlify/scraping 4.0.36 → 4.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +3 -3
- package/bin/cli.js.map +1 -1
- package/bin/constants.d.ts +1 -0
- package/bin/constants.js +1 -3
- package/bin/constants.js.map +1 -1
- package/bin/nav/listItems.js +13 -7
- package/bin/nav/listItems.js.map +1 -1
- package/bin/nav/retrieve.js +4 -5
- package/bin/nav/retrieve.js.map +1 -1
- package/bin/pipeline/color.js.map +1 -0
- package/bin/pipeline/group.js +56 -0
- package/bin/pipeline/group.js.map +1 -0
- package/bin/pipeline/icon.js.map +1 -0
- package/bin/pipeline/images.js.map +1 -0
- package/bin/{scrapingPipeline → pipeline}/logo.js +1 -4
- package/bin/pipeline/logo.js.map +1 -0
- package/bin/pipeline/page.js.map +1 -0
- package/bin/pipeline/root.js.map +1 -0
- package/bin/{scrapingPipeline → pipeline}/site.js +5 -3
- package/bin/pipeline/site.js.map +1 -0
- package/bin/{scrapingPipeline → pipeline}/tabs.js +3 -1
- package/bin/pipeline/tabs.js.map +1 -0
- package/bin/pipeline/title.js.map +1 -0
- package/bin/tabs/{retrieveReadme.js → retrieve.js} +1 -1
- package/bin/tabs/retrieve.js.map +1 -0
- package/bin/tsconfig.build.tsbuildinfo +1 -1
- package/bin/utils/images.js +0 -6
- package/bin/utils/images.js.map +1 -1
- package/bin/utils/intoChunks.d.ts +1 -0
- package/bin/utils/intoChunks.js +7 -0
- package/bin/utils/intoChunks.js.map +1 -0
- package/bin/utils/network.js +12 -19
- package/bin/utils/network.js.map +1 -1
- package/package.json +5 -5
- package/src/cli.ts +3 -3
- package/src/constants.ts +2 -3
- package/src/nav/listItems.ts +14 -10
- package/src/nav/retrieve.ts +6 -5
- package/src/pipeline/group.ts +70 -0
- package/src/{scrapingPipeline → pipeline}/logo.ts +1 -5
- package/src/{scrapingPipeline → pipeline}/site.ts +4 -3
- package/src/{scrapingPipeline → pipeline}/tabs.ts +2 -1
- package/src/utils/images.ts +0 -9
- package/src/utils/intoChunks.ts +6 -0
- package/src/utils/network.ts +23 -24
- package/bin/scrapingPipeline/color.js.map +0 -1
- package/bin/scrapingPipeline/group.js +0 -44
- package/bin/scrapingPipeline/group.js.map +0 -1
- package/bin/scrapingPipeline/icon.js.map +0 -1
- package/bin/scrapingPipeline/images.js.map +0 -1
- package/bin/scrapingPipeline/logo.js.map +0 -1
- package/bin/scrapingPipeline/page.js.map +0 -1
- package/bin/scrapingPipeline/root.js.map +0 -1
- package/bin/scrapingPipeline/site.js.map +0 -1
- package/bin/scrapingPipeline/tabs.js.map +0 -1
- package/bin/scrapingPipeline/title.js.map +0 -1
- package/bin/tabs/retrieveReadme.js.map +0 -1
- package/src/scrapingPipeline/group.ts +0 -59
- /package/bin/{scrapingPipeline → pipeline}/color.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/color.js +0 -0
- /package/bin/{scrapingPipeline → pipeline}/group.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/icon.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/icon.js +0 -0
- /package/bin/{scrapingPipeline → pipeline}/images.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/images.js +0 -0
- /package/bin/{scrapingPipeline → pipeline}/logo.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/page.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/page.js +0 -0
- /package/bin/{scrapingPipeline → pipeline}/root.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/root.js +0 -0
- /package/bin/{scrapingPipeline → pipeline}/site.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/tabs.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/title.d.ts +0 -0
- /package/bin/{scrapingPipeline → pipeline}/title.js +0 -0
- /package/bin/tabs/{retrieveReadme.d.ts → retrieve.d.ts} +0 -0
- /package/src/{scrapingPipeline → pipeline}/color.ts +0 -0
- /package/src/{scrapingPipeline → pipeline}/icon.ts +0 -0
- /package/src/{scrapingPipeline → pipeline}/images.ts +0 -0
- /package/src/{scrapingPipeline → pipeline}/page.ts +0 -0
- /package/src/{scrapingPipeline → pipeline}/root.ts +0 -0
- /package/src/{scrapingPipeline → pipeline}/title.ts +0 -0
- /package/src/tabs/{retrieveReadme.ts → retrieve.ts} +0 -0
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"site.js","sourceRoot":"","sources":["../../src/scrapingPipeline/site.ts"],"names":[],"mappings":"AAEA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AACrF,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAA+C,EAAE;IAEjD,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACrB,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAEnC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAE1B,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,cAAc,EAAE,CAAC;QACvC,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,eAAe,EAAE,EAAE,CAAC;IAE1F,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,wCAAwC,GAAG,EAAE,EAAE,CAAC;IACpF,CAAC;IAED,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,8CAA8C,GAAG,EAAE,EAAE,CAAC;IAC1F,CAAC;IAED,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;IAEpD,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM,CACjF,CAAC;IACF,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAClC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM,CACjF,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7C,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;KAC1C,CAAC;IACF,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;QACnC,MAAM,IAAI,GAAG,2BAA2B,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QACpE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,EAAE;YACzE,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,SAAS,EAAE,YAAY,EAAE;YACjE,aAAa,EAAE,KAAK;YACpB,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CACpC,eAAe;aACZ,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAChC,WAAW;aACR,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBAC1D,CAAC;qBAAM,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjB,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CACvE,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO;YACxC,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC;YAC3D,MAAM,IAAI,GAAG,cAAc;iBACxB,KAAK,CAAC,MAAM,CAAC;iBACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;iBAC3E,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,QAAQ,CAAC,KAAK,CAAC,GAAG;gBAChB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,CAAC,OAAO,CAAC;aACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG;YAChB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACtD,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACtD,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;SACnD,CAAC;QAEF,MAAM,eAAe,GAAG,SAAS;aAC9B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACd,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAC9B,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC7E,OAAO,kBAAkB,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjE,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK;qBACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACf,OAAO,IAAI,KAAK,QAAQ,IAAI,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAC9E;qBACA,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,KAAK,GAAG,CAAC,CAAC;YACV,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;gBACxC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/D,KAAK,EAAE,CAAC;oBACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;wBAChB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACvB,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC7D,CAAC;gBACD,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IACL,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBACpB,KAAK,CAAC,IAAI,CACR,CAAC,GAAG,EAAE,EAAE,CACN,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CACvF,EACD,CAAC;gBACD,IAAI,CAAC,MAAM,CACT,KAAK,CAAC,MAAM,CACV,CAAC,GAAG,EAAE,EAAE,CACN,CAAC,CACC,OAAO,GAAG,KAAK,QAAQ;oBACvB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAC1D,CACJ,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,eAAe,CAAC,2BAA2B,EAAE,eAAe,CAAC,CAAC;QAC9D,eAAe,CAAC,oBAAoB,EAAE,CAAC,GAAG,eAAe,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC;QAE5E,MAAM,oBAAoB,GAAG,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC;QAC3D,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAE1E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAEvC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI;gBACJ,IAAI;gBACJ,MAAM;gBACN,OAAO;gBACP,UAAU,EAAE,QAAsB;gBAClC,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,sEAAsE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"tabs.js","sourceRoot":"","sources":["../../src/scrapingPipeline/tabs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,GAAiB;IAEjB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACvE,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACrC,IACE,CAAC,KAAK;YACN,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,QAAQ,CAAC;YAEjE,OAAO,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACpC,GAAG,EAAE,GAAG,CAAC,QAAQ;aAClB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBACvD,OAAO,MAAM,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,EAAE,OAAO,EAAE,KAAc,EAAE,OAAO,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;YACtE,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,WAAW,GAA2B,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,cAAc,CAAC;QAC7B,IAAI,MAAM,GAAW,aAAa,CAAC;QAEnC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,OAAO;YACzB,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC5C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,cAAc;gBAAE,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1E,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,aAAa;gBAAE,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1B,GAAG,CAAC,sBAAsB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC;QACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAEvC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI;gBACJ,IAAI;gBACJ,MAAM;gBACN,OAAO;gBACP,UAAU,EAAE,WAAyB;gBACrC,IAAI;aACL;SACF,CAAC;IACJ,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"title.js","sourceRoot":"","sources":["../../src/scrapingPipeline/title.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzD,MAAM,YAAY,GAAG,iBAAiB,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAc;IAChD,IAAI,IAAI,GAAuB,SAA+B,CAAC;IAE/D,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YAAE,OAAO,QAAQ,CAAC;QAE9C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,OAAO;YACnC,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI;QAAE,OAAO,YAAY,CAAC;IAE/B,MAAM,KAAK,GAAG,IAAc,CAAC;IAC7B,IAAI,cAAc,GAAG,EAAE,CAAC;IAExB,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,cAAc,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAY,CAAC;IACpE,CAAC;SAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,cAAc,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAY,CAAC;IACpE,CAAC;SAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,cAAc,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAY,CAAC;IACpE,CAAC;SAAM,CAAC;QACN,cAAc,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,OAAO,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC;AACxD,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"retrieveReadme.js","sourceRoot":"","sources":["../../src/tabs/retrieveReadme.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAEhE,MAAM,UAAU,gBAAgB,CAAC,QAAkB;IACjD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,SAAS,CAAC;IAEzF,IAAI,OAAO,GAAwB,SAAgC,CAAC;IACpE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,IACE,IAAI,CAAC,OAAO,KAAK,QAAQ;gBACzB,IAAI,CAAC,UAAU,CAAC,SAAS;gBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,EAC/C,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;gBACtB,IAAI,CAAC,UAAU,CAAC,SAAS;gBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC5C,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAE/B,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,CAAC,OAAkB,EAAE,SAAS,EAAE,UAAU,IAAI;QACjD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;gBACtB,CAAC,CACC,IAAI,CAAC,OAAO,KAAK,KAAK;oBACtB,IAAI,CAAC,UAAU,CAAC,SAAS;oBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;oBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CACtD;gBAED,OAAO,QAAQ,CAAC;YAElB,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;gBACtC,IACE,OAAO,CAAC,OAAO,KAAK,GAAG;oBACvB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI;oBACxB,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;oBAC3C,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBAE1C,OAAO,QAAQ,CAAC;gBAClB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,KAAK,IAAI,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;oBACxD,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI;iBAC7B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;gBAAE,OAAO,QAAQ,CAAC;YAE5C,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,CAAC,EAAE,MAAM;gBACjD,IACE,OAAO,CAAC,OAAO,KAAK,GAAG;oBACvB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI;oBACxB,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;oBAC3C,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBAC1C,CAAC,MAAM;oBACP,MAAM,CAAC,IAAI,KAAK,SAAS;oBACzB,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC3C,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACxC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,eAAe;oBAClD,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBAE5D,OAAO,QAAQ,CAAC;gBAElB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,KAAK,IAAI,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;oBACxD,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI;iBAC7B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
import type { Browser } from 'puppeteer';
|
|
2
|
-
|
|
3
|
-
import type { Result } from '../types/result.js';
|
|
4
|
-
import { getErrorMessage } from '../utils/errors.js';
|
|
5
|
-
import { log } from '../utils/log.js';
|
|
6
|
-
import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
|
|
7
|
-
import { scrapePage } from './page.js';
|
|
8
|
-
|
|
9
|
-
export async function scrapePageGroup(
|
|
10
|
-
navGroup: Array<URL>,
|
|
11
|
-
needsBrowser: boolean,
|
|
12
|
-
opts: {
|
|
13
|
-
externalLinks: boolean;
|
|
14
|
-
rootPaths?: Array<string>;
|
|
15
|
-
} = { externalLinks: false }
|
|
16
|
-
): Promise<Array<Result<[string, string]>>> {
|
|
17
|
-
const browser: Browser | undefined = needsBrowser ? await startPuppeteer() : undefined;
|
|
18
|
-
|
|
19
|
-
try {
|
|
20
|
-
const res = await Promise.all(
|
|
21
|
-
navGroup.map(async (url, index) => {
|
|
22
|
-
try {
|
|
23
|
-
if (opts.externalLinks) {
|
|
24
|
-
const res = scrapePage(`external-link-${index}`, url, { externalLink: true });
|
|
25
|
-
return res;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
let isOverviewPage = false;
|
|
29
|
-
if (url.toString().endsWith('/mintie_overview')) {
|
|
30
|
-
isOverviewPage = true;
|
|
31
|
-
url = new URL(url.toString().replace('/mintie_overview', ''));
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
const html = await fetchPageHtml(url, browser);
|
|
35
|
-
const res = scrapePage(html, url, {
|
|
36
|
-
externalLink: false,
|
|
37
|
-
isOverviewPage,
|
|
38
|
-
rootPath: opts.rootPaths ? opts.rootPaths[index] : undefined,
|
|
39
|
-
});
|
|
40
|
-
return res;
|
|
41
|
-
} catch (error) {
|
|
42
|
-
const errorMessage = getErrorMessage(error);
|
|
43
|
-
throw new Error(`We encountered an error when scraping ${url}${errorMessage}`);
|
|
44
|
-
}
|
|
45
|
-
})
|
|
46
|
-
);
|
|
47
|
-
return res;
|
|
48
|
-
} catch (error) {
|
|
49
|
-
const errorMessage = getErrorMessage(error);
|
|
50
|
-
log(
|
|
51
|
-
`We encountered an error when scraping the page group from ${
|
|
52
|
-
navGroup[0]?.origin ?? 'the URL provided'
|
|
53
|
-
}${errorMessage}`
|
|
54
|
-
);
|
|
55
|
-
throw error;
|
|
56
|
-
} finally {
|
|
57
|
-
if (browser) await browser.close();
|
|
58
|
-
}
|
|
59
|
-
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|