@tyroneross/blog-scraper 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +254 -279
  3. package/dist/lib/circuit-breaker.d.ts +29 -0
  4. package/dist/lib/circuit-breaker.d.ts.map +1 -0
  5. package/dist/lib/circuit-breaker.js +89 -0
  6. package/dist/lib/circuit-breaker.js.map +1 -0
  7. package/dist/lib/content-extractor.d.ts +13 -0
  8. package/dist/lib/content-extractor.d.ts.map +1 -0
  9. package/dist/lib/content-extractor.js +75 -0
  10. package/dist/lib/content-extractor.js.map +1 -0
  11. package/dist/lib/formatters/html-to-markdown.d.ts +21 -0
  12. package/dist/lib/formatters/html-to-markdown.d.ts.map +1 -0
  13. package/dist/lib/formatters/html-to-markdown.js +146 -0
  14. package/dist/lib/formatters/html-to-markdown.js.map +1 -0
  15. package/dist/lib/formatters/text-cleaner.d.ts +44 -0
  16. package/dist/lib/formatters/text-cleaner.d.ts.map +1 -0
  17. package/dist/lib/formatters/text-cleaner.js +143 -0
  18. package/dist/lib/formatters/text-cleaner.js.map +1 -0
  19. package/dist/lib/index.d.ts +96 -0
  20. package/dist/lib/index.d.ts.map +1 -0
  21. package/dist/lib/index.js +184 -0
  22. package/dist/lib/index.js.map +1 -0
  23. package/dist/lib/quality-scorer.d.ts +83 -0
  24. package/dist/lib/quality-scorer.d.ts.map +1 -0
  25. package/dist/lib/quality-scorer.js +376 -0
  26. package/dist/lib/quality-scorer.js.map +1 -0
  27. package/dist/lib/rss-utils.d.ts +31 -0
  28. package/dist/lib/rss-utils.d.ts.map +1 -0
  29. package/dist/lib/rss-utils.js +175 -0
  30. package/dist/lib/rss-utils.js.map +1 -0
  31. package/dist/lib/scraping-rate-limiter.d.ts +52 -0
  32. package/dist/lib/scraping-rate-limiter.d.ts.map +1 -0
  33. package/dist/lib/scraping-rate-limiter.js +238 -0
  34. package/dist/lib/scraping-rate-limiter.js.map +1 -0
  35. package/dist/lib/source-orchestrator.d.ts +306 -0
  36. package/dist/lib/source-orchestrator.d.ts.map +1 -0
  37. package/dist/lib/source-orchestrator.js +840 -0
  38. package/dist/lib/source-orchestrator.js.map +1 -0
  39. package/dist/lib/types.d.ts +143 -0
  40. package/dist/lib/types.d.ts.map +1 -0
  41. package/dist/lib/types.js +7 -0
  42. package/dist/lib/types.js.map +1 -0
  43. package/dist/lib/web-scrapers/content-extractor.d.ts +62 -0
  44. package/dist/lib/web-scrapers/content-extractor.d.ts.map +1 -0
  45. package/dist/lib/web-scrapers/content-extractor.js +531 -0
  46. package/dist/lib/web-scrapers/content-extractor.js.map +1 -0
  47. package/dist/lib/web-scrapers/html-scraper.d.ts +74 -0
  48. package/dist/lib/web-scrapers/html-scraper.d.ts.map +1 -0
  49. package/dist/lib/web-scrapers/html-scraper.js +598 -0
  50. package/dist/lib/web-scrapers/html-scraper.js.map +1 -0
  51. package/dist/lib/web-scrapers/playwright-scraper.d.ts +57 -0
  52. package/dist/lib/web-scrapers/playwright-scraper.d.ts.map +1 -0
  53. package/dist/lib/web-scrapers/playwright-scraper.js +355 -0
  54. package/dist/lib/web-scrapers/playwright-scraper.js.map +1 -0
  55. package/dist/lib/web-scrapers/robots-checker.d.ts +42 -0
  56. package/dist/lib/web-scrapers/robots-checker.d.ts.map +1 -0
  57. package/dist/lib/web-scrapers/robots-checker.js +285 -0
  58. package/dist/lib/web-scrapers/robots-checker.js.map +1 -0
  59. package/dist/lib/web-scrapers/rss-discovery.d.ts +62 -0
  60. package/dist/lib/web-scrapers/rss-discovery.d.ts.map +1 -0
  61. package/dist/lib/web-scrapers/rss-discovery.js +384 -0
  62. package/dist/lib/web-scrapers/rss-discovery.js.map +1 -0
  63. package/dist/lib/web-scrapers/sitemap-parser.d.ts +65 -0
  64. package/dist/lib/web-scrapers/sitemap-parser.d.ts.map +1 -0
  65. package/dist/lib/web-scrapers/sitemap-parser.js +430 -0
  66. package/dist/lib/web-scrapers/sitemap-parser.js.map +1 -0
  67. package/package.json +54 -33
  68. package/dist/index.d.mts +0 -949
  69. package/dist/index.d.ts +0 -949
  70. package/dist/index.js +0 -3236
  71. package/dist/index.mjs +0 -3165
@@ -0,0 +1 @@
1
+ {"version":3,"file":"source-orchestrator.js","sourceRoot":"","sources":["../../lib/source-orchestrator.ts"],"names":[],"mappings":";;;;;;AAAA,6BAAwB;AACxB,oDAA4B;AAC5B,sDAA6B;AAC7B,2CAAoD;AACpD,gEAAkF;AAClF,kEAAkF;AAClF,8DAA4F;AAC5F,wEAAsF;AACtF,kEAA8D;AAC9D,0EAAkG;AAClG,qDAA0D;AAE1D,mBAAmB;AACnB,MAAM,iBAAiB,GAAG,IAAI,0BAAW,EAAE,CAAC;AAC5C,MAAM,sBAAsB,GAAG,IAAI,oCAAgB,EAAE,CAAC;AACtD,MAAM,mBAAmB,GAAG,IAAI,8BAAa,EAAE,CAAC;AAChD,uDAAoD;AAEpD,8BAA8B;AACjB,QAAA,sBAAsB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC7C,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACrB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,WAAW,EAAE,OAAC,CAAC,IAAI,EAAE;IACrB,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE;IAChB,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IACvD,gBAAgB,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,oBAAoB,CAAC,CAAC;IAChF,QAAQ,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC;AAIU,QAAA,kBAAkB,GAAG,OAAC,CAAC,MAAM,CAAC;IACzC,UAAU,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,UAAU,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,SAAS,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnD,UAAU,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAClC,YAAY,EAAE,OAAC,CAAC,MAAM,CAAC;QACrB,SAAS,EAAE,OAAC,CAAC,MAAM,CAAC;YAClB,YAAY,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC5C,cAAc,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC9C,aAAa,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC7C,gBAAgB,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;SACjD,CAAC,CAAC,QAAQ,EAAE;QACb,OAAO,EAAE,OAAC,CAAC,MAAM,CAAC;YAChB,cAAc,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACrC,cAAc,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACrC,eAAe,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC/C,eAAe,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;SAChD,CAAC,CAAC,QAAQ,EAAE;QACb,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,eAAe,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACtC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SAChC,CAAC,CAAC,QAAQ,EAAE;KACd,CAAC,CAAC,QAAQ,EAAE;CACd,CAAC,CAAC;AAuBH,MAAa,kBAAkB;IAA/B;QACmB,yBAAoB,GAAG,IAAI,CAAC;QAC5B,oBAAe,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;QAEnE;;WAEG;QACc,wBAAmB,GAAG;YACrC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU;YACnD,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,WAAW;YACnD,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc;SAC3D,CAAC;QAEF;;;WAGG;QACc,mBAAc,GAAG;YAChC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO;YAC5C,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU;SAC9C,CAAC;IAy2BJ,CAAC;IAv2BC;;;OAGG;IACK,uBAAuB,CAAC,GAAW,EAAE,MAAoB;QAC/D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAE3C,8DAA8D;YAC9D,IAAI,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBACnD,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;wBACjC,oDAAoD;wBACpD,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;4BAC/B,OAAO,CAAC,GAAG,CAAC,qDAAqD,WAAW,IAAI,CAAC,CAAC;4BAClF,OAAO;gCACL,GAAG,MAAM;gCACT,UAAU,EAAE,CAAC,GAAG,WAAW,IAAI,EAAE,GAAG,WAAW,EAAE,CAAC;6BACnD,CAAC;wBACJ,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBACjE,OAAO,CAAC,GAAG,CAAC,qDAAqD,UAAU,IAAI,CAAC,CAAC;oBACjF,OAAO;wBACL,GAAG,MAAM;wBACT,UAAU,EAAE,CAAC,GAAG,UAAU,IAAI,EAAE,UAAU,CAAC;qBAC5C,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,4BAA4B;QAC9B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACK,kCAAkC,CAAC,OAA0B;QACnE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE7C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAClC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAE7D,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;oBAEnD,iDAAiD;oBACjD,IAAI,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC5C,SAAS,CAAC,KAAK,CAAC,gDAAgD,CAAC,EAAE,CAAC;wBACtE,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,oBAAoB;YACtB,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAEzB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,kDAAkD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;OAGG;IACK,uBAAuB,CAC7B,QAA4B,EAC5B,eAAyB;QAEzB,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,8CAA8C;QAC9C,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAElE,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAE3C,+DAA+D;gBAC/D,IAAI,IAAA,uCAAsB,EAAC,IAAI,CAAC,EAAE,CAAC;oBACjC,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YAC3E,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,sBAAsB,CAAC,MAAc;QACjD,+CAA+C;QAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChD,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAE1C,OAAO,CAAC,GAAG,CAAC,qDAAqD,UAAU,KAAK,CAAC,CAAC;QAElF,sCAAsC;QACtC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,YAAY,GAAG,WAAW,SAAS,IAAI,UAAU,EAAE,CAAC;YAE1D,IAAI,CAAC;gBACH,kDAAkD;gBAClD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;gBAE7D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE;oBACzC,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,YAAY,EAAE,6EAA6E;qBAC5F;iBACF,CAAC,CAAC;gBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;gBAExB,IAAI,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACtE,OAAO,CAAC,GAAG,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;oBACtE,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;YAChD,CAAC;QACH,CAAC;QAED,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,GAAW,EACX,SAAuB,EAAE,UAAU,EAAE,MAAM,EAAE;QAE7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,oCAAoC;QACpC,MAAM,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAEnD,OAAO,CAAC,GAAG,CAAC,wCAAwC,GAAG,WAAW,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC;QAExF,MAAM,MAAM,GAAwB;YAClC,QAAQ,EAAE,EAAE;YACZ,UAAU,EAAE;gBACV,YAAY,EAAE,MAAM;gBACpB,eAAe,EAAE;oBACf,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,CAAC;oBACb,MAAM,EAAE,CAAC;oBACT,QAAQ,EAAE,CAAC;iBACZ;aACF;YACD,cAAc,EAAE,CAAC;YACjB,MAAM,EAAE,EAAE;SACX,CAAC;QAEF,IAAI,CAAC;YACH,+EAA+E;YAC/E,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,iCAAe,CAAC,QAAQ,CAAC;YAClE,OAAO,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;gBACtC,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;oBACjC,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC9D,CAAC;qBAAM,CAAC;oBACN,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9E,OAAO,CAAC,KAAK,CAAC,6CAA6C,GAAG,GAAG,EAAE,YAAY,CAAC,CAAC;YACjF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjC,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC/C,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAChC,GAAW,EACX,MAAoB,EACpB,MAA2B;QAE3B,OAAO,CAAC,GAAG,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;QAEvE,wCAAwC;QACxC,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC;gBACvC,8EAA8E;gBAC9E,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;gBACzF,OAAO,CAAC,GAAG,CAAC,0CAA0C,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;gBACzF,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,MAAM,kCAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACpE,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,UAAU,CAAC,eAAe,GAAG,eAAe,CAAC;gBAEpD,6CAA6C;gBAC7C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBAC1D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC;oBACvC,8EAA8E;oBAC9E,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;oBACzF,OAAO,CAAC,GAAG,CAAC,+CAA+C,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;oBAC9F,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACzD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,SAAS,CAAC;gBAE3C,0EAA0E;gBAC1E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,EAAE,CAAC;gBACvE,IAAI,YAAY,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;oBAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,kCAAkC,CAAC,eAAe,CAAC,CAAC;oBACjF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC/B,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;oBACnF,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;oBACnE,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;gBACnE,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,yCAAyC,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;gBACxF,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC9G,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,kBAAkB,GAAG,MAAM,oCAAmB,CAAC,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACvF,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,UAAU,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;gBAE1D,mCAAmC;gBACnC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3E,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,SAAS,CAAC;oBAE3C,0EAA0E;oBAC1E,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,EAAE,CAAC;oBACvE,IAAI,YAAY,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;wBAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,kCAAkC,CAAC,eAAe,CAAC,CAAC;wBACjF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC/B,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;wBACnF,CAAC;6BAAM,CAAC;4BACN,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;wBACnE,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;oBACnE,CAAC;oBAED,OAAO,CAAC,GAAG,CAAC,8CAA8C,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;oBAC7F,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC9G,CAAC;QAED,sEAAsE;QACtE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,EAAE,CAAC;YAEvE,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAE1E,KAAK,MAAM,YAAY,IAAI,cAAc,EAAE,CAAC;oBAC1C,IAAI,CAAC;wBACH,6BAA6B;wBAC7B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;wBAC1D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC3B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC;4BACvC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;4BAC7D,OAAO,CAAC,GAAG,CAAC,2CAA2C,YAAY,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;4BAC3G,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;wBACrC,CAAC;wBAED,2BAA2B;wBAC3B,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC;wBACzD,MAAM,iBAAiB,GAAG,MAAM,oCAAmB,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;wBACxF,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACjC,MAAM,CAAC,UAAU,CAAC,kBAAkB,GAAG;gCACrC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,kBAAkB,IAAI,EAAE,CAAC;gCAC/C,GAAG,iBAAiB;6BACrB,CAAC;4BAEF,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;gCACxC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;gCAC7D,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCAC/B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,SAAS,CAAC;oCAC3C,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;oCACjE,OAAO,CAAC,GAAG,CAAC,+CAA+C,YAAY,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;oCAC/G,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gCACrC,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;oBAAC,OAAO,QAAQ,EAAE,CAAC;wBAClB,OAAO,CAAC,GAAG,CAAC,gDAAgD,YAAY,GAAG,EAAE,QAAQ,CAAC,CAAC;oBACzF,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAChH,CAAC;QAED,qCAAqC;QACrC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC3D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC;gBACxC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9D,OAAO,CAAC,GAAG,CAAC,mDAAmD,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;gBAClG,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,4DAA4D;QAC5D,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;YAC9E,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACvE,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC,CAAC,kCAAkC;gBAC3E,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;gBACpE,OAAO,CAAC,GAAG,CAAC,sDAAsD,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;gBACrG,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAChH,CAAC;QAED,kCAAkC;QAClC,OAAO,CAAC,GAAG,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAC5B,GAAW,EACX,MAAoB,EACpB,MAA2B;QAE3B,OAAO,CAAC,GAAG,CAAC,mCAAmC,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC,CAAC;QAE5E,IAAI,CAAC;YACH,IAAI,QAAQ,GAAuB,EAAE,CAAC;YAEtC,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC1B,KAAK,KAAK;oBACR,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;oBACxC,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC;oBACvC,MAAM;gBAER,KAAK,SAAS;oBACZ,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;oBAC5C,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,SAAS,CAAC;oBAC3C,MAAM;gBAER,KAAK,MAAM;oBACT,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;oBACjD,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC;oBACxC,MAAM;YACV,CAAC;YAED,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,8BAA8B,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;YACnG,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAErC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,uBAAuB,YAAY,EAAE,CAAC,CAAC;YAC9E,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAY,EAAC,GAAG,CAAC,CAAC;QACzC,MAAM,UAAU,GAAuB,EAAE,CAAC;QAE1C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3C,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;oBACjC,SAAS;gBACX,CAAC;gBAED,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,IAAI,CAAC,IAAI;oBACd,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,WAAW;oBACX,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,OAAO,EAAE,IAAI,CAAC,cAAc;oBAC5B,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,UAAU,EAAE,GAAG;oBACf,MAAM,EAAE,KAAK;oBACb,gBAAgB,EAAE,KAAK;oBACvB,QAAQ,EAAE;wBACR,YAAY,EAAE,IAAI,CAAC,IAAI;wBACvB,SAAS,EAAE,GAAG;qBACf;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;gBACpE,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACxC,iEAAiE;QACjE,2DAA2D;QAC3D,MAAM,cAAc,GAAG,MAAM,oCAAmB,CAAC,YAAY,CAAC,GAAG,EAAE;YACjE,YAAY,EAAE,KAAK,EAAG,iDAAiD;YACvE,UAAU,EAAE,IAAI,CAAC,oBAAoB;YACrC,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QAEH,MAAM,UAAU,GAAuB,EAAE,CAAC;QAE1C,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,IAAI,IAAI,IAAI,EAAE,CAAC;gBAEhD,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,GAAG,CAAC;oBAC/D,WAAW;oBACX,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC3D,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;oBAClC,MAAM,EAAE,SAAS;oBACjB,gBAAgB,EAAE,SAAS;oBAC3B,QAAQ,EAAE;wBACR,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI;wBACrB,aAAa,EAAE,GAAG;qBACnB;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,mDAAmD,EAAE,KAAK,CAAC,CAAC;gBACzE,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CAAC,GAAW,EAAE,MAAoB;QAC3D,MAAM,cAAc,GAAmB,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAExE,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC,wBAAwB,CAAC,GAAG,EAAE,cAAc,EAAE;YAC9F,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,IAAI,CAAC;SACrD,CAAC,CAAC;QAEH,MAAM,UAAU,GAAuB,EAAE,CAAC;QAE1C,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,CAAC;gBAExD,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7D,WAAW;oBACX,OAAO,EAAE,OAAO,CAAC,WAAW;oBAC5B,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC7D,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,MAAM,EAAE,MAAM;oBACd,gBAAgB,EAAE,YAAY;oBAC9B,QAAQ,EAAE;wBACR,gBAAgB,EAAE,OAAO,CAAC,MAAM;wBAChC,UAAU,EAAE,GAAG;qBAChB;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,kDAAkD,EAAE,KAAK,CAAC,CAAC;gBACxE,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,mBAAmB,CAAC,GAAW,EAAE,MAAoB;QACjE,MAAM,iBAAiB,GAAG,IAAA,yCAAoB,GAAE,CAAC;QAEjD,MAAM,gBAAgB,GAA4B;YAChD,OAAO,EAAE,KAAK;YACd,UAAU,EAAE,IAAI;YAChB,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC,mBAAmB,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;QAC7F,MAAM,UAAU,GAAuB,EAAE,CAAC;QAE1C,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,CAAC;gBAExD,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7D,WAAW;oBACX,OAAO,EAAE,OAAO,CAAC,WAAW;oBAC5B,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC7D,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,MAAM,EAAE,MAAM;oBACd,gBAAgB,EAAE,YAAY;oBAC9B,QAAQ,EAAE;wBACR,gBAAgB,EAAE,YAAY;wBAC9B,kBAAkB,EAAE,IAAI;wBACxB,UAAU,EAAE,GAAG;qBAChB;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,wDAAwD,EAAE,KAAK,CAAC,CAAC;gBAC9E,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;;;;;;OAQG;IACK,gBAAgB,CACtB,QAA4B,EAC5B,MAAoB,EACpB,UAA0C,EAAE;QAE5C,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAE3C,sEAAsE;gBACtE,IAAI,IAAA,uCAAsB,EAAC,IAAI,CAAC,EAAE,CAAC;oBACjC,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,2CAA2C;gBAC3C,IAAI,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;oBAC7B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;wBACvC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,sDAAsD,OAAO,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;4BAC9F,OAAO,KAAK,CAAC;wBACf,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,wEAAwE;gBACxE,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;oBAC7B,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,uDAAuD;gBACvD,IAAI,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;oBAC9B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;wBACxC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;4BACvC,OAAO,IAAI,CAAC;wBACd,CAAC;oBACH,CAAC;oBACD,OAAO,CAAC,GAAG,CAAC,6DAA6D,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;oBACxF,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,oDAAoD,OAAO,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;gBACxF,OAAO,IAAI,CAAC,CAAC,+BAA+B;YAC9C,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,IAAY,EAAE,OAAe;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAErC,qBAAqB;QACrB,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,6CAA6C;QAC7C,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY;YACtD,OAAO,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,SAAS,KAAK,MAAM,CAAC;QACpE,CAAC;QAED,sCAAsC;QACtC,MAAM,YAAY,GAAG,YAAY;aAC9B,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,gCAAgC;aACtE,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB;QAE3C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,YAAY,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;QACxD,OAAO,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,MAAoB;QAC9C,MAAM,cAAc,GAAmB,EAAE,CAAC;QAE1C,IAAI,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,CAAC;YACnC,cAAc,CAAC,SAAS,GAAG;gBACzB,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,YAAY;gBACxD,cAAc,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,cAAc;gBAC5D,aAAa,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,aAAa;gBAC1D,gBAAgB,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,gBAAgB;aACjE,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;YACjC,cAAc,CAAC,OAAO,GAAG;gBACvB,cAAc,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,cAAc;gBAC1D,cAAc,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,cAAc;gBAC1D,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC1F,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC3F,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;YAChC,cAAc,CAAC,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC;QACrD,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,GAAW;QACrC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC;YAEpE,OAAO,QAAQ;iBACZ,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;iBACrB,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;iBACzC,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;iBACvE,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,kBAAkB,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW,EAAE,WAAmB;QACjD,OAAO,gBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,MAA2B;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,CAAC,cAAc,GAAG,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;QAEvE,0BAA0B;QAC1B,MAAM,CAAC,UAAU,CAAC,eAAe,GAAG;YAClC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;YACjC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,MAAM;YACnE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,QAAQ,EAAE,CAAC,CAAC,4CAA4C;SACzD,CAAC;QAEF,0CAA0C;QAC1C,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC5B,MAAM,cAAc,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;YACnD,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,GAAG;gBAAE,OAAO,cAAc,CAAC;YAC1D,OAAO,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,gBAAgB;QAChB,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAEtE,OAAO,CAAC,GAAG,CAAC,0CAA0C,MAAM,CAAC,QAAQ,CAAC,MAAM,gBAAgB,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;QACvH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB,CAC1B,QAA4B,EAC5B,cAAsB,EAAE,EACxB,UAGI,EAAE;QAEN,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAE7C,MAAM,SAAS,GAAG,QAAQ;aACvB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,6CAA6C;aAChG,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAEzB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;YACtE,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,+BAA+B,SAAS,CAAC,MAAM,uCAAuC,WAAW,GAAG,CAAC,CAAC;QAElH,MAAM,KAAK,GAAG,IAAA,iBAAM,EAAC,WAAW,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,OAAO,CAAC,UAAU,CACtB,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CACtB,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,gBAAgB,GAAG,MAAM,sBAAsB,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAClF,IAAI,gBAAgB,EAAE,CAAC;oBACrB,OAAO,CAAC,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC;oBAC3C,OAAO,CAAC,OAAO,GAAG,gBAAgB,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;oBAC9D,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;oBAC7D,OAAO,CAAC,QAAQ,GAAG;wBACjB,GAAG,OAAO,CAAC,QAAQ;wBACnB,oBAAoB,EAAE,IAAI;wBAC1B,gBAAgB,EAAE,gBAAgB,CAAC,gBAAgB;wBACnD,SAAS,EAAE,gBAAgB,CAAC,SAAS;wBACrC,WAAW,EAAE,gBAAgB,CAAC,WAAW;qBAC1C,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,+CAA+C,OAAO,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACrF,CAAC;oBAAS,CAAC;gBACT,SAAS,EAAE,CAAC;gBACZ,OAAO,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACpD,CAAC;QACH,CAAC,CAAC,CACH,CACF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,mDAAmD,SAAS,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAChG,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,cAAc,CAAC,MAAW;QAC/B,IAAI,CAAC;YACH,OAAO,0BAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,OAAC,CAAC,QAAQ,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClG,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,GAAW;QAO9B,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,eAAe,GAAG,MAAM,kCAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAEpE,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,qBAAqB,GAAG,CAAC,CAAC;QAE9B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,oCAAmB,CAAC,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC7E,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YAEjC,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,aAAa,GAAG,MAAM,oCAAmB,CAAC,gBAAgB,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;gBACtH,qBAAqB,GAAG,aAAa,CAAC,MAAM,CAAC;YAC/C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,kCAAkC;QACpC,CAAC;QAED,OAAO;YACL,eAAe,EAAE,WAAW,CAAC,OAAO;YACpC,UAAU,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC;YACtC,UAAU;YACV,YAAY,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAClF,qBAAqB;SACtB,CAAC;IACJ,CAAC;CACF;AA73BD,gDA63BC;AAED,0BAA0B;AACb,QAAA,wBAAwB,GAAG,IAAI,kBAAkB,EAAE,CAAC"}
@@ -0,0 +1,143 @@
1
+ /**
2
+ * @package @tyroneross/scraper-testing
3
+ * Core types for web scraper testing
4
+ */
5
+ export interface ScrapedArticle {
6
+ url: string;
7
+ title: string;
8
+ publishedDate?: Date | string;
9
+ description?: string;
10
+ fullContent?: string;
11
+ fullContentMarkdown?: string;
12
+ fullContentText?: string;
13
+ confidence: number;
14
+ source: 'link-text' | 'meta-data' | 'structured-data';
15
+ qualityScore?: number;
16
+ metadata?: Record<string, any>;
17
+ }
18
+ export interface ScraperTestResult {
19
+ url: string;
20
+ detectedType: 'rss' | 'sitemap' | 'html' | 'unknown';
21
+ confidence: 'high' | 'medium' | 'low';
22
+ articles: ScrapedArticle[];
23
+ extractionStats: {
24
+ attempted: number;
25
+ successful: number;
26
+ failed: number;
27
+ filtered: number;
28
+ totalDiscovered?: number;
29
+ afterDenyFilter?: number;
30
+ afterContentValidation?: number;
31
+ afterQualityFilter?: number;
32
+ };
33
+ processingTime: number;
34
+ errors: string[];
35
+ timestamp: string;
36
+ discoveredSitemaps?: string[];
37
+ discoveredFeeds?: {
38
+ url: string;
39
+ title?: string;
40
+ type?: string;
41
+ }[];
42
+ }
43
+ export interface ScraperTestRequest {
44
+ url: string;
45
+ sourceType?: 'auto' | 'rss' | 'sitemap' | 'html';
46
+ maxArticles?: number;
47
+ extractFullContent?: boolean;
48
+ denyPaths?: string[];
49
+ qualityThreshold?: number;
50
+ }
51
+ export interface ScraperTestProps {
52
+ onTestComplete?: (result: ScraperTestResult) => void;
53
+ onTestStart?: (url: string) => void;
54
+ onError?: (error: Error) => void;
55
+ className?: string;
56
+ defaultUrl?: string;
57
+ plugins?: ScraperPlugin[];
58
+ }
59
+ export interface ScraperResultsProps {
60
+ result: ScraperTestResult | null;
61
+ loading?: boolean;
62
+ error?: string | null;
63
+ className?: string;
64
+ progress?: ProgressState | null;
65
+ }
66
+ /**
67
+ * Progress state for streaming scraper updates
68
+ */
69
+ export type ProgressStage = 'rss_check' | 'sitemap_discovery' | 'subdomain_check' | 'content_extraction' | 'quality_filtering' | 'complete';
70
+ export interface ProgressState {
71
+ stage: ProgressStage;
72
+ message: string;
73
+ percent: number;
74
+ details?: string;
75
+ }
76
+ /**
77
+ * Plugin system for extending scraper functionality
78
+ * Allows users to add their own LLM-based enhancements
79
+ */
80
+ export interface ScraperPlugin {
81
+ name: string;
82
+ version: string;
83
+ /**
84
+ * Called before scraping starts
85
+ * Useful for validation, rate limiting, or pre-processing
86
+ */
87
+ beforeScrape?: (url: string) => Promise<void>;
88
+ /**
89
+ * Called after all articles are scraped
90
+ * Useful for batch processing or re-ranking
91
+ */
92
+ afterScrape?: (articles: ScrapedArticle[]) => Promise<ScrapedArticle[]>;
93
+ /**
94
+ * Called for each article individually
95
+ * Useful for adding AI-based quality scores or classifications
96
+ */
97
+ enhanceArticle?: (article: ScrapedArticle) => Promise<ScrapedArticle>;
98
+ /**
99
+ * Called to determine if an article should be filtered out
100
+ * Return true to keep the article, false to filter it out
101
+ */
102
+ filterArticle?: (article: ScrapedArticle) => Promise<boolean>;
103
+ }
104
+ /**
105
+ * Quality scoring configuration
106
+ */
107
+ export interface QualityScoreConfig {
108
+ contentWeight?: number;
109
+ dateWeight?: number;
110
+ authorWeight?: number;
111
+ schemaWeight?: number;
112
+ readingTimeWeight?: number;
113
+ threshold?: number;
114
+ }
115
+ /**
116
+ * Content validation result
117
+ */
118
+ export interface ContentValidation {
119
+ isValid: boolean;
120
+ score: number;
121
+ reasons: string[];
122
+ }
123
+ /**
124
+ * Extracted content structure
125
+ */
126
+ export interface ExtractedContent {
127
+ title?: string;
128
+ byline?: string;
129
+ content?: string;
130
+ textContent?: string;
131
+ length?: number;
132
+ excerpt?: string;
133
+ siteName?: string;
134
+ publishedTime?: string;
135
+ lang?: string;
136
+ readingTime?: number;
137
+ structured?: {
138
+ jsonLd?: any;
139
+ openGraph?: Record<string, string>;
140
+ twitter?: Record<string, string>;
141
+ };
142
+ }
143
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../lib/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,WAAW,GAAG,WAAW,GAAG,iBAAiB,CAAC;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,KAAK,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;IACrD,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,sBAAsB,CAAC,EAAE,MAAM,CAAC;QAChC,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACpE;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACrD,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACjC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,WAAW,GACX,mBAAmB,GACnB,iBAAiB,GACjB,oBAAoB,GACpB,mBAAmB,GACnB,UAAU,CAAC;AAEf,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9C;;;OAGG;IACH,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAExE;;;OAGG;IACH,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IAEtE;;;OAGG;IACH,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAC/D;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE;QACX,MAAM,CAAC,EAAE,GAAG,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH"}
@@ -0,0 +1,7 @@
1
+ "use strict";
2
+ /**
3
+ * @package @tyroneross/scraper-testing
4
+ * Core types for web scraper testing
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../lib/types.ts"],"names":[],"mappings":";AAAA;;;GAGG"}
@@ -0,0 +1,62 @@
1
+ export interface ExtractedContent {
2
+ url: string;
3
+ title: string;
4
+ content: string;
5
+ textContent: string;
6
+ excerpt?: string;
7
+ byline?: string;
8
+ publishedTime?: Date;
9
+ siteName?: string;
10
+ lang?: string;
11
+ structured?: {
12
+ jsonLd?: any;
13
+ openGraph?: Record<string, string>;
14
+ twitterCard?: Record<string, string>;
15
+ microdata?: any[];
16
+ };
17
+ wordCount: number;
18
+ readingTime: number;
19
+ confidence: number;
20
+ extractionMethod: 'readability' | 'fallback' | 'structured';
21
+ extractedAt: Date;
22
+ errors?: string[];
23
+ }
24
+ export declare class ContentExtractor {
25
+ private readonly userAgent;
26
+ private readonly timeout;
27
+ private readonly maxContentSize;
28
+ private readonly minContentLength;
29
+ private readonly wordsPerMinute;
30
+ private readonly ssrfProtection;
31
+ constructor();
32
+ /**
33
+ * Extract content from a URL
34
+ */
35
+ extractContent(url: string): Promise<ExtractedContent | null>;
36
+ /**
37
+ * Extract content from multiple URLs with configurable concurrency
38
+ */
39
+ extractBatch(urls: string[], options?: {
40
+ concurrency?: number;
41
+ onProgress?: (completed: number, total: number, url: string) => void;
42
+ }): Promise<(ExtractedContent | null)[]>;
43
+ private fetchContent;
44
+ private extractFromHTML;
45
+ private extractWithReadability;
46
+ private extractWithFallback;
47
+ private extractStructuredData;
48
+ private extractPublishedTime;
49
+ private extractSiteName;
50
+ private extractLanguage;
51
+ private countWords;
52
+ /**
53
+ * Validate extracted content quality
54
+ */
55
+ validateContent(content: ExtractedContent): {
56
+ isValid: boolean;
57
+ issues: string[];
58
+ score: number;
59
+ };
60
+ }
61
+ export declare const globalContentExtractor: ContentExtractor;
62
+ //# sourceMappingURL=content-extractor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"content-extractor.d.ts","sourceRoot":"","sources":["../../../lib/web-scrapers/content-extractor.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IAGd,UAAU,CAAC,EAAE;QACX,MAAM,CAAC,EAAE,GAAG,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACrC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC;KACnB,CAAC;IAGF,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IAGnB,gBAAgB,EAAE,aAAa,GAAG,UAAU,GAAG,YAAY,CAAC;IAC5D,WAAW,EAAE,IAAI,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAQD,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiF;IAC3G,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoB;IACnD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAO;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAO;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;;IAiDhD;;OAEG;IACG,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAgDnE;;OAEG;IACG,YAAY,CAChB,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;KACjE,GACL,OAAO,CAAC,CAAC,gBAAgB,GAAG,IAAI,CAAC,EAAE,CAAC;YA8BzB,YAAY;YA8CZ,eAAe;IAyC7B,OAAO,CAAC,sBAAsB;IA2C9B,OAAO,CAAC,mBAAmB;IAoF3B,OAAO,CAAC,qBAAqB;IAkF7B,OAAO,CAAC,oBAAoB;IAkC5B,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,UAAU;IAKlB;;OAEG;IACH,eAAe,CAAC,OAAO,EAAE,gBAAgB,GAAG;QAC1C,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;KACf;CA6CF;AAGD,eAAO,MAAM,sBAAsB,kBAAyB,CAAC"}