@mintlify/scraping 4.0.36 → 4.0.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/bin/cli.js +3 -3
  2. package/bin/cli.js.map +1 -1
  3. package/bin/constants.d.ts +1 -0
  4. package/bin/constants.js +1 -3
  5. package/bin/constants.js.map +1 -1
  6. package/bin/nav/listItems.js +13 -7
  7. package/bin/nav/listItems.js.map +1 -1
  8. package/bin/nav/retrieve.js +4 -5
  9. package/bin/nav/retrieve.js.map +1 -1
  10. package/bin/pipeline/color.js.map +1 -0
  11. package/bin/pipeline/group.js +56 -0
  12. package/bin/pipeline/group.js.map +1 -0
  13. package/bin/pipeline/icon.js.map +1 -0
  14. package/bin/pipeline/images.js.map +1 -0
  15. package/bin/{scrapingPipeline → pipeline}/logo.js +1 -4
  16. package/bin/pipeline/logo.js.map +1 -0
  17. package/bin/pipeline/page.js.map +1 -0
  18. package/bin/pipeline/root.js.map +1 -0
  19. package/bin/{scrapingPipeline → pipeline}/site.js +5 -3
  20. package/bin/pipeline/site.js.map +1 -0
  21. package/bin/{scrapingPipeline → pipeline}/tabs.js +3 -1
  22. package/bin/pipeline/tabs.js.map +1 -0
  23. package/bin/pipeline/title.js.map +1 -0
  24. package/bin/tabs/{retrieveReadme.js → retrieve.js} +1 -1
  25. package/bin/tabs/retrieve.js.map +1 -0
  26. package/bin/tsconfig.build.tsbuildinfo +1 -1
  27. package/bin/utils/images.js +0 -6
  28. package/bin/utils/images.js.map +1 -1
  29. package/bin/utils/intoChunks.d.ts +1 -0
  30. package/bin/utils/intoChunks.js +7 -0
  31. package/bin/utils/intoChunks.js.map +1 -0
  32. package/bin/utils/network.js +12 -19
  33. package/bin/utils/network.js.map +1 -1
  34. package/package.json +2 -2
  35. package/src/cli.ts +3 -3
  36. package/src/constants.ts +2 -3
  37. package/src/nav/listItems.ts +14 -10
  38. package/src/nav/retrieve.ts +6 -5
  39. package/src/pipeline/group.ts +70 -0
  40. package/src/{scrapingPipeline → pipeline}/logo.ts +1 -5
  41. package/src/{scrapingPipeline → pipeline}/site.ts +4 -3
  42. package/src/{scrapingPipeline → pipeline}/tabs.ts +2 -1
  43. package/src/utils/images.ts +0 -9
  44. package/src/utils/intoChunks.ts +6 -0
  45. package/src/utils/network.ts +23 -24
  46. package/bin/scrapingPipeline/color.js.map +0 -1
  47. package/bin/scrapingPipeline/group.js +0 -44
  48. package/bin/scrapingPipeline/group.js.map +0 -1
  49. package/bin/scrapingPipeline/icon.js.map +0 -1
  50. package/bin/scrapingPipeline/images.js.map +0 -1
  51. package/bin/scrapingPipeline/logo.js.map +0 -1
  52. package/bin/scrapingPipeline/page.js.map +0 -1
  53. package/bin/scrapingPipeline/root.js.map +0 -1
  54. package/bin/scrapingPipeline/site.js.map +0 -1
  55. package/bin/scrapingPipeline/tabs.js.map +0 -1
  56. package/bin/scrapingPipeline/title.js.map +0 -1
  57. package/bin/tabs/retrieveReadme.js.map +0 -1
  58. package/src/scrapingPipeline/group.ts +0 -59
  59. /package/bin/{scrapingPipeline → pipeline}/color.d.ts +0 -0
  60. /package/bin/{scrapingPipeline → pipeline}/color.js +0 -0
  61. /package/bin/{scrapingPipeline → pipeline}/group.d.ts +0 -0
  62. /package/bin/{scrapingPipeline → pipeline}/icon.d.ts +0 -0
  63. /package/bin/{scrapingPipeline → pipeline}/icon.js +0 -0
  64. /package/bin/{scrapingPipeline → pipeline}/images.d.ts +0 -0
  65. /package/bin/{scrapingPipeline → pipeline}/images.js +0 -0
  66. /package/bin/{scrapingPipeline → pipeline}/logo.d.ts +0 -0
  67. /package/bin/{scrapingPipeline → pipeline}/page.d.ts +0 -0
  68. /package/bin/{scrapingPipeline → pipeline}/page.js +0 -0
  69. /package/bin/{scrapingPipeline → pipeline}/root.d.ts +0 -0
  70. /package/bin/{scrapingPipeline → pipeline}/root.js +0 -0
  71. /package/bin/{scrapingPipeline → pipeline}/site.d.ts +0 -0
  72. /package/bin/{scrapingPipeline → pipeline}/tabs.d.ts +0 -0
  73. /package/bin/{scrapingPipeline → pipeline}/title.d.ts +0 -0
  74. /package/bin/{scrapingPipeline → pipeline}/title.js +0 -0
  75. /package/bin/tabs/{retrieveReadme.d.ts → retrieve.d.ts} +0 -0
  76. /package/src/{scrapingPipeline → pipeline}/color.ts +0 -0
  77. /package/src/{scrapingPipeline → pipeline}/icon.ts +0 -0
  78. /package/src/{scrapingPipeline → pipeline}/images.ts +0 -0
  79. /package/src/{scrapingPipeline → pipeline}/page.ts +0 -0
  80. /package/src/{scrapingPipeline → pipeline}/root.ts +0 -0
  81. /package/src/{scrapingPipeline → pipeline}/title.ts +0 -0
  82. /package/src/tabs/{retrieveReadme.ts → retrieve.ts} +0 -0
package/bin/cli.js CHANGED
@@ -3,9 +3,9 @@ import yargs from 'yargs';
3
3
  import { hideBin } from 'yargs/helpers';
4
4
  import { FINAL_SUCCESS_MESSAGE } from './constants.js';
5
5
  import { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
6
- import { scrapePageGroup } from './scrapingPipeline/group.js';
7
- import { htmlToHast } from './scrapingPipeline/root.js';
8
- import { scrapeAllSiteTabs } from './scrapingPipeline/tabs.js';
6
+ import { scrapePageGroup } from './pipeline/group.js';
7
+ import { htmlToHast } from './pipeline/root.js';
8
+ import { scrapeAllSiteTabs } from './pipeline/tabs.js';
9
9
  import { detectFramework, framework } from './utils/detectFramework.js';
10
10
  import { getErrorMessage } from './utils/errors.js';
11
11
  import { write } from './utils/file.js';
package/bin/cli.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KAC/B,OAAO,CACN,YAAY,EACZ,4CAA4C,EAC5C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,eAAe,EACf,wDAAwD,EACxD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,gCAAgC,EAChC,wCAAwC,EACxC,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,iBAAiB,EAAE;IAC7B,QAAQ,EAAE,kCAAkC;IAC5C,IAAI,EAAE,QAAQ;IACd,YAAY,EAAE,IAAI;CACnB,CAAC;KACD,MAAM,CAAC,YAAY,EAAE;IACpB,QAAQ,EAAE,+CAA+C;IACzD,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,QAAQ,EAAE;IAChB,QAAQ,EAAE,4DAA4D;IACtE,IAAI,EAAE,QAAQ;IACd,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,WAAW,EAAE;IACnB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,EAAE,KAAK;IACd,IAAI,EAAE,SAAS;CAChB,CAAC,EACN,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,eAAe,EAAE;YAC/D,eAAe,EAAE,SAAS;YAC1B,OAAO,EAAE,SAAS;YAClB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,CACF;KAEA,cAAc,EAAE;KAChB,aAAa,CAAC,CAAC,EAAE,gEAAgE,CAAC;KAClF,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,KAAK,EAAE,CAAC;AAEX,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,GAAG,CAAC,gDAAgD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC9B,eAAe,CAAC,IAAI,CAAC,CAAC;QAEtB,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI;YAC3B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,2CAA2C,GAAG,EAAE;SAC1D,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,GAAG,CAAC,wBAAwB,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,GAAG,CAAC,gDAAgD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9D,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KAC/B,OAAO,CACN,YAAY,EACZ,4CAA4C,EAC5C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,eAAe,EACf,wDAAwD,EACxD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,gCAAgC,EAChC,wCAAwC,EACxC,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,iBAAiB,EAAE;IAC7B,QAAQ,EAAE,kCAAkC;IAC5C,IAAI,EAAE,QAAQ;IACd,YAAY,EAAE,IAAI;CACnB,CAAC;KACD,MAAM,CAAC,YAAY,EAAE;IACpB,QAAQ,EAAE,+CAA+C;IACzD,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,QAAQ,EAAE;IAChB,QAAQ,EAAE,4DAA4D;IACtE,IAAI,EAAE,QAAQ;IACd,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,WAAW,EAAE;IACnB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,EAAE,KAAK;IACd,IAAI,EAAE,SAAS;CAChB,CAAC,EACN,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,eAAe,EAAE;YAC/D,eAAe,EAAE,SAAS;YAC1B,OAAO,EAAE,SAAS;YAClB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,CACF;KAEA,cAAc,EAAE;KAChB,aAAa,CAAC,CAAC,EAAE,gEAAgE,CAAC;KAClF,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,KAAK,EAAE,CAAC;AAEX,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,GAAG,CAAC,gDAAgD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC9B,eAAe,CAAC,IAAI,CAAC,CAAC;QAEtB,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI;YAC3B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,2CAA2C,GAAG,EAAE;SAC1D,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,GAAG,CAAC,wBAAwB,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,GAAG,CAAC,gDAAgD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9D,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
package/bin/constants.d.ts CHANGED
@@ -1,3 +1,4 @@
1
+ export declare const OVERVIEW_PAGE_SLUG = "/mintie_overview";
1
2
  export declare const SUPPORTED_MEDIA_EXTENSIONS: string[];
2
3
  export declare const ESCAPED_COMPONENTS: readonly ["Accordion", "AccordionGroup", "Note", "Warning", "Info", "Tip", "Check", "Card", "CardGroup", "CodeGroup", "Frame", "Icon", "Steps", "Step", "Tabs", "Tab", "Tooltip"];
3
4
  export declare const SPACES: string;
package/bin/constants.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { activeColors } from './utils/log.js';
2
+ export const OVERVIEW_PAGE_SLUG = '/mintie_overview';
2
3
  export const SUPPORTED_MEDIA_EXTENSIONS = [
3
4
  'png',
4
5
  'jpeg',
@@ -7,13 +8,10 @@ export const SUPPORTED_MEDIA_EXTENSIONS = [
7
8
  'avif',
8
9
  'svg',
9
10
  'ico',
10
- 'gif',
11
- 'mp4',
12
11
  'jfif',
13
12
  'pjpeg',
14
13
  'pjp',
15
14
  'svgz',
16
- 'apng',
17
15
  'bmp',
18
16
  ];
19
17
  export const ESCAPED_COMPONENTS = [
package/bin/constants.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;CACN,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,WAAW;IACX,gBAAgB;IAChB,MAAM;IACN,SAAS;IACT,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,WAAW;IACX,WAAW;IACX,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,KAAK;IACL,SAAS;CACD,CAAC;AAEX,MAAM,CAAC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAErC,MAAM,CAAC,MAAM,mBAAmB,GAAG;EACjC,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,eAAe,GAAG;EAC7B,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,iBAAiB,GAAG,4CAA4C,CAAC;AAE9E,MAAM,CAAC,MAAM,qBAAqB,GAAG;EACnC,MAAM,wBAAwB,YAAY,CAAC,IAAI,iBAAiB,YAAY,CAAC,OAAO,iCAAiC,YAAY,CAAC,IAAI,WAAW,YAAY,CAAC,OAAO;EACrK,MAAM,QAAQ,YAAY,CAAC,IAAI,gBAAgB,YAAY,CAAC,OAAO,GAAG,CAAC"}
1
+ {"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,MAAM,CAAC,MAAM,kBAAkB,GAAG,kBAAkB,CAAC;AAErD,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,MAAM;IACN,KAAK;CACN,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,WAAW;IACX,gBAAgB;IAChB,MAAM;IACN,SAAS;IACT,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,WAAW;IACX,WAAW;IACX,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,KAAK;IACL,SAAS;CACD,CAAC;AAEX,MAAM,CAAC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAErC,MAAM,CAAC,MAAM,mBAAmB,GAAG;EACjC,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,eAAe,GAAG;EAC7B,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,iBAAiB,GAAG,4CAA4C,CAAC;AAE9E,MAAM,CAAC,MAAM,qBAAqB,GAAG;EACnC,MAAM,wBAAwB,YAAY,CAAC,IAAI,iBAAiB,YAAY,CAAC,OAAO,iCAAiC,YAAY,CAAC,IAAI,WAAW,YAAY,CAAC,OAAO;EACrK,MAAM,QAAQ,YAAY,CAAC,IAAI,gBAAgB,YAAY,CAAC,OAAO,GAAG,CAAC"}
package/bin/nav/listItems.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { EXIT, visit } from 'unist-util-visit';
2
+ import { OVERVIEW_PAGE_SLUG } from '../constants.js';
2
3
  import { dedupedAppend } from '../utils/append.js';
3
4
  import { framework } from '../utils/detectFramework.js';
4
5
  import { findFirstChild } from '../utils/firstChild.js';
@@ -15,7 +16,7 @@ export function processListItem(node, opts = {
15
16
  return undefined;
16
17
  let linkHref = undefined;
17
18
  linkHref = link.properties.href;
18
- if (linkHref === undefined || linkHref === '#') {
19
+ if (linkHref === undefined || (framework.vendor !== 'docusaurus' && linkHref === '#')) {
19
20
  return undefined;
20
21
  }
21
22
  let isApiReferenceLink = false;
@@ -33,7 +34,10 @@ export function processListItem(node, opts = {
33
34
  linkHref = linkHref.substring(1);
34
35
  const sectionHeader = findFirstChild(node, opts.sectionTagName);
35
36
  const childList = findFirstChild(node, opts.childListTagName);
36
- if (!childList) {
37
+ if (!childList && framework.vendor === 'docusaurus' && linkHref === '#') {
38
+ return undefined;
39
+ }
40
+ else if (!childList) {
37
41
  return linkHref;
38
42
  }
39
43
  let title = opts.title;
@@ -46,18 +50,20 @@ export function processListItem(node, opts = {
46
50
  }
47
51
  }
48
52
  let childEntries = retrieveNavItems(childList);
49
- const newLink = childEntries.find((child) => typeof child === 'string' && child.startsWith(linkHref))
50
- ? removeTrailingSlash(linkHref) + '/mintie_overview'
53
+ const newLink = linkHref !== '#' &&
54
+ childEntries.find((child) => typeof child === 'string' && child.startsWith(linkHref))
55
+ ? removeTrailingSlash(linkHref) + OVERVIEW_PAGE_SLUG
51
56
  : linkHref;
52
- if (childEntries.includes(linkHref)) {
57
+ if (linkHref !== '#' && childEntries.includes(linkHref)) {
53
58
  childEntries.forEach((child, index) => {
54
59
  if (child === linkHref)
55
60
  childEntries[index] = newLink;
56
61
  });
62
+ childEntries = dedupedAppend(newLink, childEntries, true);
57
63
  }
58
- else {
64
+ else if (linkHref !== '#') {
59
65
  childEntries = dedupedAppend(newLink, childEntries, true);
60
66
  }
61
- return { group: title, pages: dedupedAppend(newLink, childEntries, true) };
67
+ return { group: title, pages: childEntries };
62
68
  }
63
69
  //# sourceMappingURL=listItems.js.map
package/bin/nav/listItems.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"listItems.js","sourceRoot":"","sources":["../../src/nav/listItems.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAQjD,MAAM,UAAU,eAAe,CAC7B,IAAa,EACb,OAAwB;IACtB,cAAc,EAAE,KAAK;IACrB,gBAAgB,EAAE,IAAI;IACtB,KAAK,EAAE,SAAS;CACjB;IAED,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvC,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAE5B,IAAI,QAAQ,GAAuB,SAAS,CAAC;IAC7C,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAA0B,CAAC;IAEtD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QAC/C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,kBAAkB,GAAG,KAAgB,CAAC;IAC1C,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;QACtC,IACE,OAAO,CAAC,OAAO,KAAK,MAAM;YAC1B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,EACrD,CAAC;YACD,kBAAkB,GAAG,IAAI,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,kBAAkB;QAAE,OAAO,SAAS,CAAC;IAEzC,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAE/D,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9D,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACxD,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QACxD,CAAC;IACH,CAAC;IAED,IAAI,YAAY,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAC/B,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CACnE;QACC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,GAAG,kBAAkB;QACpD,CAAC,CAAC,QAAQ,CAAC;IAEb,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACpC,IAAI,KAAK,KAAK,QAAQ;gBAAE,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,YAAY,GAAG,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,CAAC;AAC7E,CAAC"}
1
+ {"version":3,"file":"listItems.js","sourceRoot":"","sources":["../../src/nav/listItems.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAQjD,MAAM,UAAU,eAAe,CAC7B,IAAa,EACb,OAAwB;IACtB,cAAc,EAAE,KAAK;IACrB,gBAAgB,EAAE,IAAI;IACtB,KAAK,EAAE,SAAS;CACjB;IAED,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvC,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAE5B,IAAI,QAAQ,GAAuB,SAAS,CAAC;IAC7C,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAA0B,CAAC;IAEtD,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,YAAY,IAAI,QAAQ,KAAK,GAAG,CAAC,EAAE,CAAC;QACtF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,kBAAkB,GAAG,KAAgB,CAAC;IAC1C,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;QACtC,IACE,OAAO,CAAC,OAAO,KAAK,MAAM;YAC1B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,EACrD,CAAC;YACD,kBAAkB,GAAG,IAAI,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,kBAAkB;QAAE,OAAO,SAAS,CAAC;IAEzC,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAE/D,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9D,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QACxE,OAAO,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACxD,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QACxD,CAAC;IACH,CAAC;IAED,IAAI,YAAY,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,OAAO,GACX,QAAQ,KAAK,GAAG;QAChB,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACnF,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,GAAG,kBAAkB;QACpD,CAAC,CAAC,QAAQ,CAAC;IAEf,IAAI,QAAQ,KAAK,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxD,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACpC,IAAI,KAAK,KAAK,QAAQ;gBAAE,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxD,CAAC,CAAC,CAAC;QACH,YAAY,GAAG,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;SAAM,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QAC5B,YAAY,GAAG,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;AAC/C,CAAC"}
package/bin/nav/retrieve.js CHANGED
@@ -19,7 +19,7 @@ export function retrieveNavItems(rootNode) {
19
19
  let innerSectionTagName = 'div';
20
20
  switch (framework.vendor) {
21
21
  case 'docusaurus':
22
- innerSectionTagName = 'li';
22
+ innerSectionTagName = 'div';
23
23
  break;
24
24
  case 'gitbook':
25
25
  innerSectionTagName = 'div';
@@ -36,12 +36,11 @@ export function retrieveNavItems(rootNode) {
36
36
  let title = undefined;
37
37
  if (node.children[0] &&
38
38
  node.children[1] &&
39
- node.children[1].type === 'element' &&
40
- node.children[1].tagName === 'ul' &&
41
39
  node.children[0].type === 'element' &&
42
40
  node.children[0].tagName === 'div' &&
43
- node.children[0].children.filter((child) => child.type === 'text').length ===
44
- node.children[0].children.length) {
41
+ node.children[1].type === 'element' &&
42
+ node.children[1].tagName === 'ul' &&
43
+ node.children[0].children.filter((child) => framework.vendor === 'docusaurus' || child.type === 'text').length === node.children[0].children.length) {
45
44
  title = findTitle(node.children[0], { delete: false });
46
45
  }
47
46
  if (framework.vendor === 'readme' &&
package/bin/nav/retrieve.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../src/nav/retrieve.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEjD,MAAM,UAAU,gBAAgB,CAAC,QAAiB;IAChD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,IAAI,kBAAkB,GAAG,IAAI,CAAC;IAC9B,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,kBAAkB,GAAG,IAAI,CAAC;YAC1B,MAAM;QACR,KAAK,SAAS;YACZ,kBAAkB,GAAG,IAAI,CAAC;YAC1B,MAAM;QACR,KAAK,QAAQ;YACX,kBAAkB,GAAG,SAAS,CAAC;YAC/B,MAAM;IACV,CAAC;IAED,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,mBAAmB,GAAG,IAAI,CAAC;YAC3B,MAAM;QACR,KAAK,SAAS;YACZ,mBAAmB,GAAG,KAAK,CAAC;YAC5B,MAAM;QACR,KAAK,QAAQ;YACX,mBAAmB,GAAG,IAAI,CAAC;YAC3B,MAAM;IACV,CAAC;IAED,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI,EAAE,KAAK,EAAE,MAAM;QACtD,IAAI,IAAI,CAAC,OAAO,KAAK,kBAAkB;YAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAC7D,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;QAE3C,IAAI,KAAK,GAAuB,SAAS,CAAC;QAC1C,IACE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YACjC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK;YAClC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,MAAM;gBACvE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAClC,CAAC;YACD,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,IACE,SAAS,CAAC,MAAM,KAAK,QAAQ;YAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YACjC,OAAO,KAAK,KAAK,QAAQ;YACzB,MAAM,EACN,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG;gBACd;oBACE,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,EAAE;oBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACxB;aACF,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,EAAE;YAClC,cAAc,EAAE,mBAAmB;YACnC,gBAAgB,EAAE,IAAI;YACtB,KAAK;SACN,CAAC,CAAC;QAEH,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../src/nav/retrieve.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEjD,MAAM,UAAU,gBAAgB,CAAC,QAAiB;IAChD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,IAAI,kBAAkB,GAAG,IAAI,CAAC;IAC9B,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,kBAAkB,GAAG,IAAI,CAAC;YAC1B,MAAM;QACR,KAAK,SAAS;YACZ,kBAAkB,GAAG,IAAI,CAAC;YAC1B,MAAM;QACR,KAAK,QAAQ;YACX,kBAAkB,GAAG,SAAS,CAAC;YAC/B,MAAM;IACV,CAAC;IAED,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,mBAAmB,GAAG,KAAK,CAAC;YAC5B,MAAM;QACR,KAAK,SAAS;YACZ,mBAAmB,GAAG,KAAK,CAAC;YAC5B,MAAM;QACR,KAAK,QAAQ;YACX,mBAAmB,GAAG,IAAI,CAAC;YAC3B,MAAM;IACV,CAAC;IAED,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI,EAAE,KAAK,EAAE,MAAM;QACtD,IAAI,IAAI,CAAC,OAAO,KAAK,kBAAkB;YAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAC7D,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;QAE3C,IAAI,KAAK,GAAuB,SAAS,CAAC;QAC1C,IACE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK;YAClC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YACjC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,CACtE,CAAC,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAC7C,CAAC;YACD,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,IACE,SAAS,CAAC,MAAM,KAAK,QAAQ;YAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YACjC,OAAO,KAAK,KAAK,QAAQ;YACzB,MAAM,EACN,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG;gBACd;oBACE,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,EAAE;oBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACxB;aACF,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,EAAE;YAClC,cAAc,EAAE,mBAAmB;YACnC,gBAAgB,EAAE,IAAI;YACtB,KAAK;SACN,CAAC,CAAC;QAEH,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
package/bin/pipeline/color.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"color.js","sourceRoot":"","sources":["../../src/pipeline/color.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAExD,SAAS,KAAK,CAAC,KAAa;IAC1B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,aAAa,CAAC,GAAuB;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,OAAO,oCAAoC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,cAAc,CAAC,GAAG,OAAsB;IAC/C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,GAAG;YAAE,OAAO,KAAK,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,aAAa,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEnC,IAAI,CAAqB,EAAE,CAAqB,EAAE,CAAqB,CAAC;IAExE,IAAI,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAE1E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAErC,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAE/C,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;AAC5D,CAAC;AAED,SAAS,WAAW,CAAC,SAAiB,EAAE,GAAW;IACjD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,uBAAuB,EAAE,GAAG,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrC,OAAO,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,SAAS;IAChB,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAc;IACjD,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,aAAa,CAAC;IAE5D,IAAI,cAAc,GAAuB,SAAS,CAAC;IACnD,IAAI,YAAY,GAAuB,SAAS,CAAC;IACjD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YAAE,OAAO,QAAQ,CAAC;QAC9C,IACE,CAAC,SAAS,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;YACzE,CAAC,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,KAAK,eAAe,CAAC;YAE5E,OAAO,QAAQ,CAAC;QAElB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM;YACrF,OAAO,QAAQ,CAAC;QAElB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACtC,MAAM,eAAe,GACnB,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACjF,MAAM,aAAa,GACjB,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,qBAAqB,CAAC;QAEjF,MAAM,oBAAoB,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAClE,MAAM,kBAAkB,GAAG,WAAW,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC9D,IAAI,CAAC,oBAAoB,IAAI,CAAC,kBAAkB;YAAE,OAAO,QAAQ,CAAC;QAElE,cAAc,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAChD,YAAY,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;QACnC,OAAO;YACL,OAAO,EAAE,cAAe;YACxB,KAAK,EAAE,YAAY;YACnB,IAAI,EAAE,cAAc;SACrB,CAAC;IACJ,CAAC;SAAM,IAAI,cAAc,EAAE,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,cAAe;YACxB,IAAI,EAAE,cAAc;SACrB,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,aAAa,CAAC;IACvB,CAAC;AACH,CAAC"}
package/bin/pipeline/group.js ADDED
@@ -0,0 +1,56 @@
1
+ import { OVERVIEW_PAGE_SLUG } from '../constants.js';
2
+ import { getErrorMessage } from '../utils/errors.js';
3
+ import { intoChunks } from '../utils/intoChunks.js';
4
+ import { log } from '../utils/log.js';
5
+ import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
6
+ import { scrapePage } from './page.js';
7
+ export async function scrapePageGroup(navGroup, needsBrowser, opts = { externalLinks: false }) {
8
+ const browser = needsBrowser ? await startPuppeteer() : undefined;
9
+ const allResults = [];
10
+ try {
11
+ for (const chunk of intoChunks(navGroup)) {
12
+ const res = await Promise.all(chunk.map(async (url, index) => {
13
+ try {
14
+ if (opts.externalLinks) {
15
+ const res = scrapePage(`external-link-${index}`, url, { externalLink: true });
16
+ return res;
17
+ }
18
+ let isOverviewPage = false;
19
+ if (url.toString().endsWith(OVERVIEW_PAGE_SLUG)) {
20
+ isOverviewPage = true;
21
+ url = new URL(url.toString().replace(OVERVIEW_PAGE_SLUG, ''));
22
+ }
23
+ const html = await fetchPageHtml(url, browser);
24
+ const res = scrapePage(html, url, {
25
+ externalLink: false,
26
+ isOverviewPage,
27
+ rootPath: opts.rootPaths ? opts.rootPaths[index] : undefined,
28
+ });
29
+ return res;
30
+ }
31
+ catch (error) {
32
+ const errorMessage = getErrorMessage(error);
33
+ return {
34
+ success: false,
35
+ message: `We encountered an error when scraping ${url}${errorMessage}`,
36
+ data: [url.toString(), ''],
37
+ };
38
+ }
39
+ }));
40
+ allResults.push(...res);
41
+ }
42
+ }
43
+ catch (error) {
44
+ const errorMessage = getErrorMessage(error);
45
+ log(`We encountered an error when scraping the page group from ${navGroup[0]?.origin ?? 'the URL provided'}${errorMessage}`);
46
+ if (browser)
47
+ await browser.close();
48
+ throw error;
49
+ }
50
+ finally {
51
+ if (browser)
52
+ await browser.close();
53
+ return allResults;
54
+ }
55
+ }
56
+ //# sourceMappingURL=group.js.map
package/bin/pipeline/group.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"group.js","sourceRoot":"","sources":["../../src/pipeline/group.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAErD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAAoB,EACpB,YAAqB,EACrB,OAGI,EAAE,aAAa,EAAE,KAAK,EAAE;IAE5B,MAAM,OAAO,GAAwB,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,MAAM,UAAU,GAAoC,EAAE,CAAC;IACvD,IAAI,CAAC;QACH,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAC3B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;gBAC7B,IAAI,CAAC;oBACH,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBACvB,MAAM,GAAG,GAAG,UAAU,CAAC,iBAAiB,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC9E,OAAO,GAAG,CAAC;oBACb,CAAC;oBAED,IAAI,cAAc,GAAG,KAAK,CAAC;oBAC3B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;wBAChD,cAAc,GAAG,IAAI,CAAC;wBACtB,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;oBAChE,CAAC;oBAED,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBAC/C,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE;wBAChC,YAAY,EAAE,KAAK;wBACnB,cAAc;wBACd,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;qBAC7D,CAAC,CAAC;oBACH,OAAO,GAAG,CAAC;gBACb,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;oBAC5C,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,OAAO,EAAE,yCAAyC,GAAG,GAAG,YAAY,EAAE;wBACtE,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAqB;qBAC/C,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YACF,UAAU,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CACD,6DACE,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,kBACzB,GAAG,YAAY,EAAE,CAClB,CAAC;QACF,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACnC,OAAO,UAAU,CAAC;IACpB,CAAC;AACH,CAAC"}
package/bin/pipeline/icon.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"icon.js","sourceRoot":"","sources":["../../src/pipeline/icon.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAc;IAClD,IAAI,GAAG,GAAW,EAAE,CAAC;IACrB,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,MAAM;YACvB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAClC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EACpC,CAAC;YACD,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,IAAc,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO,cAAc,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,OAAO,cAAc,CAAC;IAErC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC"}
package/bin/pipeline/images.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/pipeline/images.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGnD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAe,EACf,GAAiB;IAEjB,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,KAAK,CAAC,IAAI,EAAE,UAAU,IAAI;QACxB,IAAI,QAAQ,GAAuB,SAAS,CAAC;QAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1B,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;QACtB,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;YAC7C,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAiB,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,CAC1F,EAAE,KAA2B,CAAC;QACjC,CAAC;QAED,IAAI,CAAC,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE/B,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QACtD,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACnD,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE9D,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC/B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACjG,CAAC;IAEF,KAAK,CAAC,IAAI,EAAE,UAAU,IAAI,EAAE,KAAK,EAAE,MAAM;QACvC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC;YACnD,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;QACzE,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;YAC7C,MAAM,OAAO,GAAI,IAAI,CAAC,UAAqC,CAAC,IAAI;YAC9D,uEAAuE;YACvE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAiB,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,CAC1F,CAAC;YACF,IAAI,CAAC,OAAO;gBAAE,OAAO,QAAQ,CAAC;YAE9B,OAAO,CAAC,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,KAAe,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC;YAC5E,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;QACzE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,YAAY,CAAC;AACtB,CAAC"}
package/bin/{scrapingPipeline → pipeline}/logo.js RENAMED
@@ -73,10 +73,7 @@ export async function downloadLogos(url, browser) {
73
73
  await page.click('.rm-ThemeToggle');
74
74
  htmls.push(await page.content());
75
75
  }
76
- catch {
77
- // do nothing, it just means there's no theme toggle
78
- }
79
- await browser.close();
76
+ catch { }
80
77
  await Promise.all(htmls.map(async (html) => {
81
78
  return await findLogosFromHtml(html, findReadmeLogoNodes, filepaths);
82
79
  }));
package/bin/pipeline/logo.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logo.js","sourceRoot":"","sources":["../../src/pipeline/logo.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,SAAS,mBAAmB,CAAC,IAAc;IACzC,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC;YAEjD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAc;IAC1C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,MAAM,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAc;IAC7C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,EACnD,CAAC;YACD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;gBACtC,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK;oBAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,IAAY,EACZ,UAA0D,EAC1D,SAAwB;IAExB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,QAAQ,EAAE,CAAC;QACb,SAAS,CAAC,IAAI,CACZ,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CACnB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,MAAM,aAAa,CAC7B,IAAI,CAAC,UAAU,CAAC,GAAa,EAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAC9B,CAAC;YAEF,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC5B,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CACH,CAAC,CACH,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QACpC,IAAI,CAAC,QAAQ;YAAE,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,OAA4B;IAE5B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAkB,EAAE,CAAC;QAEhC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;gBAC9B,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;YAEH,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACjC,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,MAAM,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,CAAC,CAAC;QACvE,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,iBAAiB,CACrB,IAAI,EACJ,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,uBAAuB,EAC/E,SAAS,CACV,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YAC5C,GAAG,CAAC,sCAAsC,YAAY,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzD,OAAO,eAAe,CAAC,MAAM,KAAK,CAAC;QACjC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACpB,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;YAC1B,CAAC,CAAC;gBACE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAW;gBACnC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAW;aACnC;YACH,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC"}
package/bin/pipeline/page.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"page.js","sourceRoot":"","sources":["../../src/pipeline/page.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,eAAe,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EACL,aAAa,EACb,UAAU,EACV,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,UAAU,EACV,eAAe,GAChB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAII,EAAE,YAAY,EAAE,KAAK,EAAE;IAE3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC;QACtB,MAAM,eAAe,GAAG,GAAG,QAAQ,MAAM,CAAC;QAC1C,SAAS,CAAC,eAAe,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAEzB,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,OAAO;QACV,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,mBAAmB,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;IAE9F,MAAM,aAAa,GAAa;QAC9B,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB,CAAC;IAEF,MAAM,SAAS,GAAc,OAAO,EAAE;SACnC,GAAG,CAAC,mBAAmB,CAAC;SACxB,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,WAAW,CAAC;SAChB,GAAG,CAAC,aAAa,CAAC;SAClB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,oBAAoB,CAAC;SACzB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,uBAAuB,CAAC;SAC5B,GAAG,CAAC,4BAA4B,CAAC;SACjC,GAAG,CAAC,sBAAsB,CAAC;SAC3B,GAAG,CAAC,qBAAqB,CAAC;QAE3B,iDAAiD;QACjD,mDAAmD;QACnD,4BAA4B;SAC3B,GAAG,CAAC,8BAA8B,CAAC;SACnC,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,mBAAmB,CAAC;SACxB,GAAG,CAAC,0BAA0B,CAAC;SAC/B,GAAG,CAAC,qBAAqB,CAAC;SAC1B,GAAG,CAAC,yBAAyB,CAAC;SAC9B,GAAG,CAAC,4BAA4B,CAAC;SACjC,GAAG,CAAC,6BAA6B,CAAC;QACnC,6IAA6I;SAC5I,OAAO,CAAC,aAAa,CAAc,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAClE,eAAe,CAAC,wBAAwB,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,YAAY,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,yDAAyD,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;QAC9F,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEtD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,EAAE;aACrB,GAAG,CAAC,SAAS,CAAC;aACd,GAAG,CAAC,SAAS,CAAC;YACf,kGAAkG;aACjG,GAAG,CAAC,eAAe,CAAC;aACpB,SAAS,CAAC,SAAS,CAAC,CAAC;QAExB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC9D,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACjF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,IAAI,CAAC,QAAQ;gBACjB,CAAC,CAAC,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC;gBACpF,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,GAAG,MAAM,KAAK,iBAAiB,GAAG,YAAY,EAAE;YACzD,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;SACnB,CAAC;IACJ,CAAC;AACH,CAAC"}
package/bin/pipeline/root.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"root.js","sourceRoot":"","sources":["../../src/pipeline/root.ts"],"names":[],"mappings":"AACA,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,OAAO,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OACE,OAAO,EAAE;QACP,kGAAkG;SACjG,GAAG,CAAC,WAAW,CAAC;SAChB,GAAG,CAAC,sBAAsB,CAAC;SAC3B,GAAG,CAAC,wBAAwB,CAAC;SAC7B,KAAK,CAAC,IAAI,CACd,CAAC;AACJ,CAAC"}
package/bin/{scrapingPipeline → pipeline}/site.js RENAMED
@@ -1,5 +1,6 @@
1
1
  import traverse from 'traverse';
2
2
  import { NAV_FAILURE_MSG } from '../constants.js';
3
+ import { OVERVIEW_PAGE_SLUG } from '../constants.js';
3
4
  import { iterateOverNavItems } from '../nav/iterate.js';
4
5
  import { retrieveNavItems } from '../nav/retrieve.js';
5
6
  import { retrieveRootNavElement } from '../nav/root.js';
@@ -20,7 +21,8 @@ export async function scrapeSite(html, url, opts = {}) {
20
21
  hast = htmlToHast(html);
21
22
  url = new URL(url);
22
23
  const origin = url.origin;
23
- detectFramework(hast);
24
+ if (!framework.vendor)
25
+ detectFramework(hast);
24
26
  if (framework.vendor === 'docusaurus') {
25
27
  const browser = await startPuppeteer();
26
28
  html = await fetchPageHtml(url, browser);
@@ -87,10 +89,10 @@ export async function scrapeSite(html, url, opts = {}) {
87
89
  });
88
90
  traverse(navItems).forEach(function (value) {
89
91
  if (typeof value === 'string') {
90
- this.update(value.replace('/mintie_overview', ''));
92
+ this.update(value.replace(OVERVIEW_PAGE_SLUG, ''));
91
93
  }
92
94
  else if (Array.isArray(value)) {
93
- this.update(value.map((item) => typeof item === 'string' ? item.replace('/mintie_overview', '') : item));
95
+ this.update(value.map((item) => typeof item === 'string' ? item.replace(OVERVIEW_PAGE_SLUG, '') : item));
94
96
  }
95
97
  });
96
98
  navItems.forEach((navItem, index) => {
package/bin/pipeline/site.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"site.js","sourceRoot":"","sources":["../../src/pipeline/site.ts"],"names":[],"mappings":"AAEA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AACrF,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAA+C,EAAE;IAEjD,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACrB,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAEnC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAE1B,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAE7C,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,cAAc,EAAE,CAAC;QACvC,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,eAAe,EAAE,EAAE,CAAC;IAE1F,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,wCAAwC,GAAG,EAAE,EAAE,CAAC;IACpF,CAAC;IAED,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,8CAA8C,GAAG,EAAE,EAAE,CAAC;IAC1F,CAAC;IAED,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;IAEpD,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM,CACjF,CAAC;IACF,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAClC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM,CACjF,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7C,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;KAC1C,CAAC;IACF,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;QACnC,MAAM,IAAI,GAAG,2BAA2B,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QACpE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,EAAE;YACzE,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,SAAS,EAAE,YAAY,EAAE;YACjE,aAAa,EAAE,KAAK;YACpB,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CACpC,eAAe;aACZ,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAChC,WAAW;aACR,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBAC1D,CAAC;qBAAM,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjB,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CACvE,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO;YACxC,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC;YAC3D,MAAM,IAAI,GAAG,cAAc;iBACxB,KAAK,CAAC,MAAM,CAAC;iBACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;iBAC3E,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,QAAQ,CAAC,KAAK,CAAC,GAAG;gBAChB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,CAAC,OAAO,CAAC;aACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG;YAChB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACtD,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACtD,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;SACnD,CAAC;QAEF,MAAM,eAAe,GAAG,SAAS;aAC9B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACd,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAC9B,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC7E,OAAO,kBAAkB,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjE,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK;qBACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACf,OAAO,IAAI,KAAK,QAAQ,IAAI,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAC9E;qBACA,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,KAAK,GAAG,CAAC,CAAC;YACV,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;gBACxC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/D,KAAK,EAAE,CAAC;oBACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;wBAChB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACvB,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC7D,CAAC;gBACD,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IACL,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBACpB,KAAK,CAAC,IAAI,CACR,CAAC,GAAG,EAAE,EAAE,CACN,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CACvF,EACD,CAAC;gBACD,IAAI,CAAC,MAAM,CACT,KAAK,CAAC,MAAM,CACV,CAAC,GAAG,EAAE,EAAE,CACN,CAAC,CACC,OAAO,GAAG,KAAK,QAAQ;oBACvB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAC1D,CACJ,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,eAAe,CAAC,2BAA2B,EAAE,eAAe,CAAC,CAAC;QAC9D,eAAe,CAAC,oBAAoB,EAAE,CAAC,GAAG,eAAe,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC;QAE5E,MAAM,oBAAoB,GAAG,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC;QAC3D,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAE1E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAEvC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI;gBACJ,IAAI;gBACJ,MAAM;gBACN,OAAO;gBACP,UAAU,EAAE,QAAsB;gBAClC,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,sEAAsE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC"}
package/bin/{scrapingPipeline → pipeline}/tabs.js RENAMED
@@ -1,4 +1,4 @@
1
- import { retrieveTabLinks } from '../tabs/retrieveReadme.js';
1
+ import { retrieveTabLinks } from '../tabs/retrieve.js';
2
2
  import { detectFramework, framework } from '../utils/detectFramework.js';
3
3
  import { getErrorMessage } from '../utils/errors.js';
4
4
  import { log } from '../utils/log.js';
@@ -60,6 +60,8 @@ export async function scrapeAllSiteTabs(html, url) {
60
60
  const browser = needsBrowser ? await startPuppeteer() : undefined;
61
61
  const logo = await downloadLogos(url, browser);
62
62
  const name = await downloadTitle(hast);
63
+ if (browser)
64
+ await browser.close();
63
65
  return {
64
66
  success: true,
65
67
  data: {
package/bin/pipeline/tabs.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tabs.js","sourceRoot":"","sources":["../../src/pipeline/tabs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,GAAiB;IAEjB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACvE,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACrC,IACE,CAAC,KAAK;YACN,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,QAAQ,CAAC;YAEjE,OAAO,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACpC,GAAG,EAAE,GAAG,CAAC,QAAQ;aAClB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBACvD,OAAO,MAAM,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,EAAE,OAAO,EAAE,KAAc,EAAE,OAAO,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;YACtE,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,WAAW,GAA2B,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,cAAc,CAAC;QAC7B,IAAI,MAAM,GAAW,aAAa,CAAC;QAEnC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,OAAO;YACzB,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC5C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,cAAc;gBAAE,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1E,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,aAAa;gBAAE,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1B,GAAG,CAAC,sBAAsB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC;QACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEnC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI;gBACJ,IAAI;gBACJ,MAAM;gBACN,OAAO;gBACP,UAAU,EAAE,WAAyB;gBACrC,IAAI;aACL;SACF,CAAC;IACJ,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC"}
package/bin/pipeline/title.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"title.js","sourceRoot":"","sources":["../../src/pipeline/title.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzD,MAAM,YAAY,GAAG,iBAAiB,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAc;IAChD,IAAI,IAAI,GAAuB,SAA+B,CAAC;IAE/D,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YAAE,OAAO,QAAQ,CAAC;QAE9C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,OAAO;YACnC,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI;QAAE,OAAO,YAAY,CAAC;IAE/B,MAAM,KAAK,GAAG,IAAc,CAAC;IAC7B,IAAI,cAAc,GAAG,EAAE,CAAC;IAExB,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,cAAc,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAY,CAAC;IACpE,CAAC;SAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,cAAc,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAY,CAAC;IACpE,CAAC;SAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,cAAc,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAY,CAAC;IACpE,CAAC;SAAM,CAAC;QACN,cAAc,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,OAAO,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC;AACxD,CAAC"}
package/bin/tabs/{retrieveReadme.js → retrieve.js} RENAMED
@@ -74,4 +74,4 @@ export function retrieveTabLinks(rootNode) {
74
74
  });
75
75
  return links;
76
76
  }
77
- //# sourceMappingURL=retrieveReadme.js.map
77
+ //# sourceMappingURL=retrieve.js.map
package/bin/tabs/retrieve.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../src/tabs/retrieve.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAEhE,MAAM,UAAU,gBAAgB,CAAC,QAAkB;IACjD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,SAAS,CAAC;IAEzF,IAAI,OAAO,GAAwB,SAAgC,CAAC;IACpE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,IACE,IAAI,CAAC,OAAO,KAAK,QAAQ;gBACzB,IAAI,CAAC,UAAU,CAAC,SAAS;gBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,EAC/C,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;gBACtB,IAAI,CAAC,UAAU,CAAC,SAAS;gBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC5C,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAE/B,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,CAAC,OAAkB,EAAE,SAAS,EAAE,UAAU,IAAI;QACjD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;gBACtB,CAAC,CACC,IAAI,CAAC,OAAO,KAAK,KAAK;oBACtB,IAAI,CAAC,UAAU,CAAC,SAAS;oBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;oBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CACtD;gBAED,OAAO,QAAQ,CAAC;YAElB,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;gBACtC,IACE,OAAO,CAAC,OAAO,KAAK,GAAG;oBACvB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI;oBACxB,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;oBAC3C,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBAE1C,OAAO,QAAQ,CAAC;gBAClB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,KAAK,IAAI,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;oBACxD,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI;iBAC7B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;gBAAE,OAAO,QAAQ,CAAC;YAE5C,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,CAAC,EAAE,MAAM;gBACjD,IACE,OAAO,CAAC,OAAO,KAAK,GAAG;oBACvB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI;oBACxB,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;oBAC3C,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBAC1C,CAAC,MAAM;oBACP,MAAM,CAAC,IAAI,KAAK,SAAS;oBACzB,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC3C,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACxC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,eAAe;oBAClD,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBAE5D,OAAO,QAAQ,CAAC;gBAElB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,KAAK,IAAI,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;oBACxD,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI;iBAC7B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC"}