@tyroneross/blog-scraper 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +254 -279
- package/dist/lib/circuit-breaker.d.ts +29 -0
- package/dist/lib/circuit-breaker.d.ts.map +1 -0
- package/dist/lib/circuit-breaker.js +89 -0
- package/dist/lib/circuit-breaker.js.map +1 -0
- package/dist/lib/content-extractor.d.ts +13 -0
- package/dist/lib/content-extractor.d.ts.map +1 -0
- package/dist/lib/content-extractor.js +75 -0
- package/dist/lib/content-extractor.js.map +1 -0
- package/dist/lib/formatters/html-to-markdown.d.ts +21 -0
- package/dist/lib/formatters/html-to-markdown.d.ts.map +1 -0
- package/dist/lib/formatters/html-to-markdown.js +146 -0
- package/dist/lib/formatters/html-to-markdown.js.map +1 -0
- package/dist/lib/formatters/text-cleaner.d.ts +44 -0
- package/dist/lib/formatters/text-cleaner.d.ts.map +1 -0
- package/dist/lib/formatters/text-cleaner.js +143 -0
- package/dist/lib/formatters/text-cleaner.js.map +1 -0
- package/dist/lib/index.d.ts +96 -0
- package/dist/lib/index.d.ts.map +1 -0
- package/dist/lib/index.js +184 -0
- package/dist/lib/index.js.map +1 -0
- package/dist/lib/quality-scorer.d.ts +83 -0
- package/dist/lib/quality-scorer.d.ts.map +1 -0
- package/dist/lib/quality-scorer.js +376 -0
- package/dist/lib/quality-scorer.js.map +1 -0
- package/dist/lib/rss-utils.d.ts +31 -0
- package/dist/lib/rss-utils.d.ts.map +1 -0
- package/dist/lib/rss-utils.js +175 -0
- package/dist/lib/rss-utils.js.map +1 -0
- package/dist/lib/scraping-rate-limiter.d.ts +52 -0
- package/dist/lib/scraping-rate-limiter.d.ts.map +1 -0
- package/dist/lib/scraping-rate-limiter.js +238 -0
- package/dist/lib/scraping-rate-limiter.js.map +1 -0
- package/dist/lib/source-orchestrator.d.ts +306 -0
- package/dist/lib/source-orchestrator.d.ts.map +1 -0
- package/dist/lib/source-orchestrator.js +840 -0
- package/dist/lib/source-orchestrator.js.map +1 -0
- package/dist/lib/types.d.ts +143 -0
- package/dist/lib/types.d.ts.map +1 -0
- package/dist/lib/types.js +7 -0
- package/dist/lib/types.js.map +1 -0
- package/dist/lib/web-scrapers/content-extractor.d.ts +62 -0
- package/dist/lib/web-scrapers/content-extractor.d.ts.map +1 -0
- package/dist/lib/web-scrapers/content-extractor.js +531 -0
- package/dist/lib/web-scrapers/content-extractor.js.map +1 -0
- package/dist/lib/web-scrapers/html-scraper.d.ts +74 -0
- package/dist/lib/web-scrapers/html-scraper.d.ts.map +1 -0
- package/dist/lib/web-scrapers/html-scraper.js +598 -0
- package/dist/lib/web-scrapers/html-scraper.js.map +1 -0
- package/dist/lib/web-scrapers/playwright-scraper.d.ts +57 -0
- package/dist/lib/web-scrapers/playwright-scraper.d.ts.map +1 -0
- package/dist/lib/web-scrapers/playwright-scraper.js +355 -0
- package/dist/lib/web-scrapers/playwright-scraper.js.map +1 -0
- package/dist/lib/web-scrapers/robots-checker.d.ts +42 -0
- package/dist/lib/web-scrapers/robots-checker.d.ts.map +1 -0
- package/dist/lib/web-scrapers/robots-checker.js +285 -0
- package/dist/lib/web-scrapers/robots-checker.js.map +1 -0
- package/dist/lib/web-scrapers/rss-discovery.d.ts +62 -0
- package/dist/lib/web-scrapers/rss-discovery.d.ts.map +1 -0
- package/dist/lib/web-scrapers/rss-discovery.js +384 -0
- package/dist/lib/web-scrapers/rss-discovery.js.map +1 -0
- package/dist/lib/web-scrapers/sitemap-parser.d.ts +65 -0
- package/dist/lib/web-scrapers/sitemap-parser.d.ts.map +1 -0
- package/dist/lib/web-scrapers/sitemap-parser.js +430 -0
- package/dist/lib/web-scrapers/sitemap-parser.js.map +1 -0
- package/package.json +54 -33
- package/dist/index.d.mts +0 -949
- package/dist/index.d.ts +0 -949
- package/dist/index.js +0 -3236
- package/dist/index.mjs +0 -3165
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"source-orchestrator.js","sourceRoot":"","sources":["../../lib/source-orchestrator.ts"],"names":[],"mappings":";;;;;;AAAA,6BAAwB;AACxB,oDAA4B;AAC5B,sDAA6B;AAC7B,2CAAoD;AACpD,gEAAkF;AAClF,kEAAkF;AAClF,8DAA4F;AAC5F,wEAAsF;AACtF,kEAA8D;AAC9D,0EAAkG;AAClG,qDAA0D;AAE1D,mBAAmB;AACnB,MAAM,iBAAiB,GAAG,IAAI,0BAAW,EAAE,CAAC;AAC5C,MAAM,sBAAsB,GAAG,IAAI,oCAAgB,EAAE,CAAC;AACtD,MAAM,mBAAmB,GAAG,IAAI,8BAAa,EAAE,CAAC;AAChD,uDAAoD;AAEpD,8BAA8B;AACjB,QAAA,sBAAsB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC7C,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACrB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,WAAW,EAAE,OAAC,CAAC,IAAI,EAAE;IACrB,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE;IAChB,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IACvD,gBAAgB,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,oBAAoB,CAAC,CAAC;IAChF,QAAQ,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC;AAIU,QAAA,kBAAkB,GAAG,OAAC,CAAC,MAAM,CAAC;IACzC,UAAU,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,UAAU,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,SAAS,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnD,UAAU,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAClC,YAAY,EAAE,OAAC,CAAC,MAAM,CAAC;QACrB,SAAS,EAAE,OAAC,CAAC,MAAM,CAAC;YAClB,YAAY,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC5C,cAAc,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC9C,aAAa,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC7C,gBAAgB,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;SACjD,CAAC,CAAC,QAAQ,EAAE;QACb,OAAO,EAAE,OAAC,CAAC,MAAM,CAAC;YAChB,cAAc,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACrC,cAAc,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACrC,eAAe,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC/C,eAAe,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;SAChD,CAAC,CAAC,QAAQ,EAAE;QACb,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,eAAe,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACtC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SAChC,CAAC,CAAC,QAAQ,EAAE;KACd,CAAC,CAAC,QAAQ,EAAE;CACd,CAAC,CAAC;AAuBH,MAAa,kBAAkB;IAA/B;QACmB,yBAAoB,GAAG,IAAI,CAAC;QAC5B,oBAAe,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;QAEnE;;WAEG;QACc,wBAAmB,GAAG;YACrC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU;YACnD,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,WAAW;YACnD,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc;SAC3D,CAAC;QAEF;;;WAGG;QACc,mBAAc,GAAG;YAChC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO;YAC5C,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU;SAC9C,CAAC;IAy2BJ,CAAC;IAv2BC;;;OAGG;IACK,uBAAuB,CAAC,GAAW,EAAE,MAAoB;QAC/D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAE3C,8DAA8D;YAC9D,IAAI,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBACnD,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;wBACjC,oDAAoD;wBACpD,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;4BAC/B,OAAO,CAAC,GAAG,CAAC,qDAAqD,WAAW,IAAI,CAAC,CAAC;4BAClF,OAAO;gCACL,GAAG,MAAM;gCACT,UAAU,EAAE,CAAC,GAAG,WAAW,IAAI,EAAE,GAAG,WAAW,EAAE,CAAC;6BACnD,CAAC;wBACJ,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBACjE,OAAO,CAAC,GAAG,CAAC,qDAAqD,UAAU,IAAI,CAAC,CAAC;oBACjF,OAAO;wBACL,GAAG,MAAM;wBACT,UAAU,EAAE,CAAC,GAAG,UAAU,IAAI,EAAE,UAAU,CAAC;qBAC5C,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,4BAA4B;QAC9B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACK,kCAAkC,CAAC,OAA0B;QACnE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE7C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAClC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAE7D,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;oBAEnD,iDAAiD;oBACjD,IAAI,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC5C,SAAS,CAAC,KAAK,CAAC,gDAAgD,CAAC,EAAE,CAAC;wBACtE,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,oBAAoB;YACtB,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAEzB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,kDAAkD,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;OAGG;IACK,uBAAuB,CAC7B,QAA4B,EAC5B,eAAyB;QAEzB,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,8CAA8C;QAC9C,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAElE,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAE3C,+DAA+D;gBAC/D,IAAI,IAAA,uCAAsB,EAAC,IAAI,CAAC,EAAE,CAAC;oBACjC,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YAC3E,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,sBAAsB,CAAC,MAAc;QACjD,+CAA+C;QAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChD,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAE1C,OAAO,CAAC,GAAG,CAAC,qDAAqD,UAAU,KAAK,CAAC,CAAC;QAElF,sCAAsC;QACtC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,YAAY,GAAG,WAAW,SAAS,IAAI,UAAU,EAAE,CAAC;YAE1D,IAAI,CAAC;gBACH,kDAAkD;gBAClD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;gBAE7D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE;oBACzC,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,YAAY,EAAE,6EAA6E;qBAC5F;iBACF,CAAC,CAAC;gBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;gBAExB,IAAI,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACtE,OAAO,CAAC,GAAG,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;oBACtE,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;YAChD,CAAC;QACH,CAAC;QAED,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,GAAW,EACX,SAAuB,EAAE,UAAU,EAAE,MAAM,EAAE;QAE7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,oCAAoC;QACpC,MAAM,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAEnD,OAAO,CAAC,GAAG,CAAC,wCAAwC,GAAG,WAAW,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC;QAExF,MAAM,MAAM,GAAwB;YAClC,QAAQ,EAAE,EAAE;YACZ,UAAU,EAAE;gBACV,YAAY,EAAE,MAAM;gBACpB,eAAe,EAAE;oBACf,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,CAAC;oBACb,MAAM,EAAE,CAAC;oBACT,QAAQ,EAAE,CAAC;iBACZ;aACF;YACD,cAAc,EAAE,CAAC;YACjB,MAAM,EAAE,EAAE;SACX,CAAC;QAEF,IAAI,CAAC;YACH,+EAA+E;YAC/E,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,iCAAe,CAAC,QAAQ,CAAC;YAClE,OAAO,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;gBACtC,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;oBACjC,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC9D,CAAC;qBAAM,CAAC;oBACN,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9E,OAAO,CAAC,KAAK,CAAC,6CAA6C,GAAG,GAAG,EAAE,YAAY,CAAC,CAAC;YACjF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjC,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC/C,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAChC,GAAW,EACX,MAAoB,EACpB,MAA2B;QAE3B,OAAO,CAAC,GAAG,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;QAEvE,wCAAwC;QACxC,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC;gBACvC,8EAA8E;gBAC9E,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;gBACzF,OAAO,CAAC,GAAG,CAAC,0CAA0C,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;gBACzF,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,MAAM,kCAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACpE,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,UAAU,CAAC,eAAe,GAAG,eAAe,CAAC;gBAEpD,6CAA6C;gBAC7C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBAC1D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC;oBACvC,8EAA8E;oBAC9E,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;oBACzF,OAAO,CAAC,GAAG,CAAC,+CAA+C,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;oBAC9F,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACzD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,SAAS,CAAC;gBAE3C,0EAA0E;gBAC1E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,EAAE,CAAC;gBACvE,IAAI,YAAY,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;oBAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,kCAAkC,CAAC,eAAe,CAAC,CAAC;oBACjF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC/B,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;oBACnF,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;oBACnE,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;gBACnE,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,yCAAyC,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;gBACxF,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC9G,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,kBAAkB,GAAG,MAAM,oCAAmB,CAAC,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACvF,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,UAAU,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;gBAE1D,mCAAmC;gBACnC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3E,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,SAAS,CAAC;oBAE3C,0EAA0E;oBAC1E,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,EAAE,CAAC;oBACvE,IAAI,YAAY,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;wBAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,kCAAkC,CAAC,eAAe,CAAC,CAAC;wBACjF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC/B,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;wBACnF,CAAC;6BAAM,CAAC;4BACN,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;wBACnE,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;oBACnE,CAAC;oBAED,OAAO,CAAC,GAAG,CAAC,8CAA8C,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;oBAC7F,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC9G,CAAC;QAED,sEAAsE;QACtE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,EAAE,CAAC;YAEvE,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAE1E,KAAK,MAAM,YAAY,IAAI,cAAc,EAAE,CAAC;oBAC1C,IAAI,CAAC;wBACH,6BAA6B;wBAC7B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;wBAC1D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC3B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC;4BACvC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;4BAC7D,OAAO,CAAC,GAAG,CAAC,2CAA2C,YAAY,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;4BAC3G,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;wBACrC,CAAC;wBAED,2BAA2B;wBAC3B,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC;wBACzD,MAAM,iBAAiB,GAAG,MAAM,oCAAmB,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;wBACxF,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACjC,MAAM,CAAC,UAAU,CAAC,kBAAkB,GAAG;gCACrC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,kBAAkB,IAAI,EAAE,CAAC;gCAC/C,GAAG,iBAAiB;6BACrB,CAAC;4BAEF,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;gCACxC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;gCAC7D,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCAC/B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,SAAS,CAAC;oCAC3C,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;oCACjE,OAAO,CAAC,GAAG,CAAC,+CAA+C,YAAY,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;oCAC/G,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gCACrC,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;oBAAC,OAAO,QAAQ,EAAE,CAAC;wBAClB,OAAO,CAAC,GAAG,CAAC,gDAAgD,YAAY,GAAG,EAAE,QAAQ,CAAC,CAAC;oBACzF,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAChH,CAAC;QAED,qCAAqC;QACrC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC3D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC;gBACxC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9D,OAAO,CAAC,GAAG,CAAC,mDAAmD,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;gBAClG,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,4DAA4D;QAC5D,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;YAC9E,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACvE,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC,CAAC,kCAAkC;gBAC3E,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;gBACpE,OAAO,CAAC,GAAG,CAAC,sDAAsD,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;gBACrG,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAChH,CAAC;QAED,kCAAkC;QAClC,OAAO,CAAC,GAAG,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAC5B,GAAW,EACX,MAAoB,EACpB,MAA2B;QAE3B,OAAO,CAAC,GAAG,CAAC,mCAAmC,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC,CAAC;QAE5E,IAAI,CAAC;YACH,IAAI,QAAQ,GAAuB,EAAE,CAAC;YAEtC,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC1B,KAAK,KAAK;oBACR,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;oBACxC,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC;oBACvC,MAAM;gBAER,KAAK,SAAS;oBACZ,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;oBAC5C,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,SAAS,CAAC;oBAC3C,MAAM;gBAER,KAAK,MAAM;oBACT,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;oBACjD,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC;oBACxC,MAAM;YACV,CAAC;YAED,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,8BAA8B,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;YACnG,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAErC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,uBAAuB,YAAY,EAAE,CAAC,CAAC;YAC9E,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAY,EAAC,GAAG,CAAC,CAAC;QACzC,MAAM,UAAU,GAAuB,EAAE,CAAC;QAE1C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3C,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;oBACjC,SAAS;gBACX,CAAC;gBAED,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,IAAI,CAAC,IAAI;oBACd,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,WAAW;oBACX,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,OAAO,EAAE,IAAI,CAAC,cAAc;oBAC5B,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,UAAU,EAAE,GAAG;oBACf,MAAM,EAAE,KAAK;oBACb,gBAAgB,EAAE,KAAK;oBACvB,QAAQ,EAAE;wBACR,YAAY,EAAE,IAAI,CAAC,IAAI;wBACvB,SAAS,EAAE,GAAG;qBACf;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;gBACpE,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACxC,iEAAiE;QACjE,2DAA2D;QAC3D,MAAM,cAAc,GAAG,MAAM,oCAAmB,CAAC,YAAY,CAAC,GAAG,EAAE;YACjE,YAAY,EAAE,KAAK,EAAG,iDAAiD;YACvE,UAAU,EAAE,IAAI,CAAC,oBAAoB;YACrC,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QAEH,MAAM,UAAU,GAAuB,EAAE,CAAC;QAE1C,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,IAAI,IAAI,IAAI,EAAE,CAAC;gBAEhD,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,GAAG,CAAC;oBAC/D,WAAW;oBACX,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC3D,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;oBAClC,MAAM,EAAE,SAAS;oBACjB,gBAAgB,EAAE,SAAS;oBAC3B,QAAQ,EAAE;wBACR,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI;wBACrB,aAAa,EAAE,GAAG;qBACnB;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,mDAAmD,EAAE,KAAK,CAAC,CAAC;gBACzE,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CAAC,GAAW,EAAE,MAAoB;QAC3D,MAAM,cAAc,GAAmB,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAExE,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC,wBAAwB,CAAC,GAAG,EAAE,cAAc,EAAE;YAC9F,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,IAAI,CAAC;SACrD,CAAC,CAAC;QAEH,MAAM,UAAU,GAAuB,EAAE,CAAC;QAE1C,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,CAAC;gBAExD,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7D,WAAW;oBACX,OAAO,EAAE,OAAO,CAAC,WAAW;oBAC5B,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC7D,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,MAAM,EAAE,MAAM;oBACd,gBAAgB,EAAE,YAAY;oBAC9B,QAAQ,EAAE;wBACR,gBAAgB,EAAE,OAAO,CAAC,MAAM;wBAChC,UAAU,EAAE,GAAG;qBAChB;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,kDAAkD,EAAE,KAAK,CAAC,CAAC;gBACxE,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,mBAAmB,CAAC,GAAW,EAAE,MAAoB;QACjE,MAAM,iBAAiB,GAAG,IAAA,yCAAoB,GAAE,CAAC;QAEjD,MAAM,gBAAgB,GAA4B;YAChD,OAAO,EAAE,KAAK;YACd,UAAU,EAAE,IAAI;YAChB,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC,mBAAmB,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;QAC7F,MAAM,UAAU,GAAuB,EAAE,CAAC;QAE1C,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,CAAC;gBAExD,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7D,WAAW;oBACX,OAAO,EAAE,OAAO,CAAC,WAAW;oBAC5B,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC7D,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,MAAM,EAAE,MAAM;oBACd,gBAAgB,EAAE,YAAY;oBAC9B,QAAQ,EAAE;wBACR,gBAAgB,EAAE,YAAY;wBAC9B,kBAAkB,EAAE,IAAI;wBACxB,UAAU,EAAE,GAAG;qBAChB;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,wDAAwD,EAAE,KAAK,CAAC,CAAC;gBAC9E,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;;;;;;OAQG;IACK,gBAAgB,CACtB,QAA4B,EAC5B,MAAoB,EACpB,UAA0C,EAAE;QAE5C,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAE3C,sEAAsE;gBACtE,IAAI,IAAA,uCAAsB,EAAC,IAAI,CAAC,EAAE,CAAC;oBACjC,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,2CAA2C;gBAC3C,IAAI,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;oBAC7B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;wBACvC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,sDAAsD,OAAO,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;4BAC9F,OAAO,KAAK,CAAC;wBACf,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,wEAAwE;gBACxE,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;oBAC7B,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,uDAAuD;gBACvD,IAAI,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;oBAC9B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;wBACxC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;4BACvC,OAAO,IAAI,CAAC;wBACd,CAAC;oBACH,CAAC;oBACD,OAAO,CAAC,GAAG,CAAC,6DAA6D,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;oBACxF,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,oDAAoD,OAAO,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;gBACxF,OAAO,IAAI,CAAC,CAAC,+BAA+B;YAC9C,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,IAAY,EAAE,OAAe;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAErC,qBAAqB;QACrB,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,6CAA6C;QAC7C,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY;YACtD,OAAO,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,SAAS,KAAK,MAAM,CAAC;QACpE,CAAC;QAED,sCAAsC;QACtC,MAAM,YAAY,GAAG,YAAY;aAC9B,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,gCAAgC;aACtE,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB;QAE3C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,YAAY,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;QACxD,OAAO,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,MAAoB;QAC9C,MAAM,cAAc,GAAmB,EAAE,CAAC;QAE1C,IAAI,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,CAAC;YACnC,cAAc,CAAC,SAAS,GAAG;gBACzB,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,YAAY;gBACxD,cAAc,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,cAAc;gBAC5D,aAAa,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,aAAa;gBAC1D,gBAAgB,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,gBAAgB;aACjE,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;YACjC,cAAc,CAAC,OAAO,GAAG;gBACvB,cAAc,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,cAAc;gBAC1D,cAAc,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,cAAc;gBAC1D,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC1F,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC3F,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;YAChC,cAAc,CAAC,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC;QACrD,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,GAAW;QACrC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC;YAEpE,OAAO,QAAQ;iBACZ,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;iBACrB,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;iBACzC,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;iBACvE,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,kBAAkB,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW,EAAE,WAAmB;QACjD,OAAO,gBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,MAA2B;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,CAAC,cAAc,GAAG,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;QAEvE,0BAA0B;QAC1B,MAAM,CAAC,UAAU,CAAC,eAAe,GAAG;YAClC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;YACjC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,MAAM;YACnE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,QAAQ,EAAE,CAAC,CAAC,4CAA4C;SACzD,CAAC;QAEF,0CAA0C;QAC1C,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC5B,MAAM,cAAc,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;YACnD,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,GAAG;gBAAE,OAAO,cAAc,CAAC;YAC1D,OAAO,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,gBAAgB;QAChB,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAEtE,OAAO,CAAC,GAAG,CAAC,0CAA0C,MAAM,CAAC,QAAQ,CAAC,MAAM,gBAAgB,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;QACvH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB,CAC1B,QAA4B,EAC5B,cAAsB,EAAE,EACxB,UAGI,EAAE;QAEN,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAE7C,MAAM,SAAS,GAAG,QAAQ;aACvB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,6CAA6C;aAChG,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAEzB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;YACtE,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,+BAA+B,SAAS,CAAC,MAAM,uCAAuC,WAAW,GAAG,CAAC,CAAC;QAElH,MAAM,KAAK,GAAG,IAAA,iBAAM,EAAC,WAAW,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,OAAO,CAAC,UAAU,CACtB,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CACtB,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,gBAAgB,GAAG,MAAM,sBAAsB,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAClF,IAAI,gBAAgB,EAAE,CAAC;oBACrB,OAAO,CAAC,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC;oBAC3C,OAAO,CAAC,OAAO,GAAG,gBAAgB,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;oBAC9D,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;oBAC7D,OAAO,CAAC,QAAQ,GAAG;wBACjB,GAAG,OAAO,CAAC,QAAQ;wBACnB,oBAAoB,EAAE,IAAI;wBAC1B,gBAAgB,EAAE,gBAAgB,CAAC,gBAAgB;wBACnD,SAAS,EAAE,gBAAgB,CAAC,SAAS;wBACrC,WAAW,EAAE,gBAAgB,CAAC,WAAW;qBAC1C,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,+CAA+C,OAAO,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACrF,CAAC;oBAAS,CAAC;gBACT,SAAS,EAAE,CAAC;gBACZ,OAAO,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACpD,CAAC;QACH,CAAC,CAAC,CACH,CACF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,mDAAmD,SAAS,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAChG,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,cAAc,CAAC,MAAW;QAC/B,IAAI,CAAC;YACH,OAAO,0BAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,OAAC,CAAC,QAAQ,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClG,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,GAAW;QAO9B,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,eAAe,GAAG,MAAM,kCAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAEpE,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,qBAAqB,GAAG,CAAC,CAAC;QAE9B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,oCAAmB,CAAC,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC7E,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YAEjC,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,aAAa,GAAG,MAAM,oCAAmB,CAAC,gBAAgB,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;gBACtH,qBAAqB,GAAG,aAAa,CAAC,MAAM,CAAC;YAC/C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,kCAAkC;QACpC,CAAC;QAED,OAAO;YACL,eAAe,EAAE,WAAW,CAAC,OAAO;YACpC,UAAU,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC;YACtC,UAAU;YACV,YAAY,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAClF,qBAAqB;SACtB,CAAC;IACJ,CAAC;CACF;AA73BD,gDA63BC;AAED,0BAA0B;AACb,QAAA,wBAAwB,GAAG,IAAI,kBAAkB,EAAE,CAAC"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @package @tyroneross/scraper-testing
|
|
3
|
+
* Core types for web scraper testing
|
|
4
|
+
*/
|
|
5
|
+
export interface ScrapedArticle {
|
|
6
|
+
url: string;
|
|
7
|
+
title: string;
|
|
8
|
+
publishedDate?: Date | string;
|
|
9
|
+
description?: string;
|
|
10
|
+
fullContent?: string;
|
|
11
|
+
fullContentMarkdown?: string;
|
|
12
|
+
fullContentText?: string;
|
|
13
|
+
confidence: number;
|
|
14
|
+
source: 'link-text' | 'meta-data' | 'structured-data';
|
|
15
|
+
qualityScore?: number;
|
|
16
|
+
metadata?: Record<string, any>;
|
|
17
|
+
}
|
|
18
|
+
export interface ScraperTestResult {
|
|
19
|
+
url: string;
|
|
20
|
+
detectedType: 'rss' | 'sitemap' | 'html' | 'unknown';
|
|
21
|
+
confidence: 'high' | 'medium' | 'low';
|
|
22
|
+
articles: ScrapedArticle[];
|
|
23
|
+
extractionStats: {
|
|
24
|
+
attempted: number;
|
|
25
|
+
successful: number;
|
|
26
|
+
failed: number;
|
|
27
|
+
filtered: number;
|
|
28
|
+
totalDiscovered?: number;
|
|
29
|
+
afterDenyFilter?: number;
|
|
30
|
+
afterContentValidation?: number;
|
|
31
|
+
afterQualityFilter?: number;
|
|
32
|
+
};
|
|
33
|
+
processingTime: number;
|
|
34
|
+
errors: string[];
|
|
35
|
+
timestamp: string;
|
|
36
|
+
discoveredSitemaps?: string[];
|
|
37
|
+
discoveredFeeds?: {
|
|
38
|
+
url: string;
|
|
39
|
+
title?: string;
|
|
40
|
+
type?: string;
|
|
41
|
+
}[];
|
|
42
|
+
}
|
|
43
|
+
export interface ScraperTestRequest {
|
|
44
|
+
url: string;
|
|
45
|
+
sourceType?: 'auto' | 'rss' | 'sitemap' | 'html';
|
|
46
|
+
maxArticles?: number;
|
|
47
|
+
extractFullContent?: boolean;
|
|
48
|
+
denyPaths?: string[];
|
|
49
|
+
qualityThreshold?: number;
|
|
50
|
+
}
|
|
51
|
+
export interface ScraperTestProps {
|
|
52
|
+
onTestComplete?: (result: ScraperTestResult) => void;
|
|
53
|
+
onTestStart?: (url: string) => void;
|
|
54
|
+
onError?: (error: Error) => void;
|
|
55
|
+
className?: string;
|
|
56
|
+
defaultUrl?: string;
|
|
57
|
+
plugins?: ScraperPlugin[];
|
|
58
|
+
}
|
|
59
|
+
export interface ScraperResultsProps {
|
|
60
|
+
result: ScraperTestResult | null;
|
|
61
|
+
loading?: boolean;
|
|
62
|
+
error?: string | null;
|
|
63
|
+
className?: string;
|
|
64
|
+
progress?: ProgressState | null;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Progress state for streaming scraper updates
|
|
68
|
+
*/
|
|
69
|
+
export type ProgressStage = 'rss_check' | 'sitemap_discovery' | 'subdomain_check' | 'content_extraction' | 'quality_filtering' | 'complete';
|
|
70
|
+
export interface ProgressState {
|
|
71
|
+
stage: ProgressStage;
|
|
72
|
+
message: string;
|
|
73
|
+
percent: number;
|
|
74
|
+
details?: string;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Plugin system for extending scraper functionality
|
|
78
|
+
* Allows users to add their own LLM-based enhancements
|
|
79
|
+
*/
|
|
80
|
+
export interface ScraperPlugin {
|
|
81
|
+
name: string;
|
|
82
|
+
version: string;
|
|
83
|
+
/**
|
|
84
|
+
* Called before scraping starts
|
|
85
|
+
* Useful for validation, rate limiting, or pre-processing
|
|
86
|
+
*/
|
|
87
|
+
beforeScrape?: (url: string) => Promise<void>;
|
|
88
|
+
/**
|
|
89
|
+
* Called after all articles are scraped
|
|
90
|
+
* Useful for batch processing or re-ranking
|
|
91
|
+
*/
|
|
92
|
+
afterScrape?: (articles: ScrapedArticle[]) => Promise<ScrapedArticle[]>;
|
|
93
|
+
/**
|
|
94
|
+
* Called for each article individually
|
|
95
|
+
* Useful for adding AI-based quality scores or classifications
|
|
96
|
+
*/
|
|
97
|
+
enhanceArticle?: (article: ScrapedArticle) => Promise<ScrapedArticle>;
|
|
98
|
+
/**
|
|
99
|
+
* Called to determine if an article should be filtered out
|
|
100
|
+
* Return true to keep the article, false to filter it out
|
|
101
|
+
*/
|
|
102
|
+
filterArticle?: (article: ScrapedArticle) => Promise<boolean>;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Quality scoring configuration
|
|
106
|
+
*/
|
|
107
|
+
export interface QualityScoreConfig {
|
|
108
|
+
contentWeight?: number;
|
|
109
|
+
dateWeight?: number;
|
|
110
|
+
authorWeight?: number;
|
|
111
|
+
schemaWeight?: number;
|
|
112
|
+
readingTimeWeight?: number;
|
|
113
|
+
threshold?: number;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Content validation result
|
|
117
|
+
*/
|
|
118
|
+
export interface ContentValidation {
|
|
119
|
+
isValid: boolean;
|
|
120
|
+
score: number;
|
|
121
|
+
reasons: string[];
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Extracted content structure
|
|
125
|
+
*/
|
|
126
|
+
export interface ExtractedContent {
|
|
127
|
+
title?: string;
|
|
128
|
+
byline?: string;
|
|
129
|
+
content?: string;
|
|
130
|
+
textContent?: string;
|
|
131
|
+
length?: number;
|
|
132
|
+
excerpt?: string;
|
|
133
|
+
siteName?: string;
|
|
134
|
+
publishedTime?: string;
|
|
135
|
+
lang?: string;
|
|
136
|
+
readingTime?: number;
|
|
137
|
+
structured?: {
|
|
138
|
+
jsonLd?: any;
|
|
139
|
+
openGraph?: Record<string, string>;
|
|
140
|
+
twitter?: Record<string, string>;
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../lib/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,WAAW,GAAG,WAAW,GAAG,iBAAiB,CAAC;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,KAAK,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;IACrD,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,sBAAsB,CAAC,EAAE,MAAM,CAAC;QAChC,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACpE;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACrD,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACjC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,WAAW,GACX,mBAAmB,GACnB,iBAAiB,GACjB,oBAAoB,GACpB,mBAAmB,GACnB,UAAU,CAAC;AAEf,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9C;;;OAGG;IACH,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAExE;;;OAGG;IACH,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IAEtE;;;OAGG;IACH,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAC/D;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE;QACX,MAAM,CAAC,EAAE,GAAG,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../lib/types.ts"],"names":[],"mappings":";AAAA;;;GAGG"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
export interface ExtractedContent {
|
|
2
|
+
url: string;
|
|
3
|
+
title: string;
|
|
4
|
+
content: string;
|
|
5
|
+
textContent: string;
|
|
6
|
+
excerpt?: string;
|
|
7
|
+
byline?: string;
|
|
8
|
+
publishedTime?: Date;
|
|
9
|
+
siteName?: string;
|
|
10
|
+
lang?: string;
|
|
11
|
+
structured?: {
|
|
12
|
+
jsonLd?: any;
|
|
13
|
+
openGraph?: Record<string, string>;
|
|
14
|
+
twitterCard?: Record<string, string>;
|
|
15
|
+
microdata?: any[];
|
|
16
|
+
};
|
|
17
|
+
wordCount: number;
|
|
18
|
+
readingTime: number;
|
|
19
|
+
confidence: number;
|
|
20
|
+
extractionMethod: 'readability' | 'fallback' | 'structured';
|
|
21
|
+
extractedAt: Date;
|
|
22
|
+
errors?: string[];
|
|
23
|
+
}
|
|
24
|
+
export declare class ContentExtractor {
|
|
25
|
+
private readonly userAgent;
|
|
26
|
+
private readonly timeout;
|
|
27
|
+
private readonly maxContentSize;
|
|
28
|
+
private readonly minContentLength;
|
|
29
|
+
private readonly wordsPerMinute;
|
|
30
|
+
private readonly ssrfProtection;
|
|
31
|
+
constructor();
|
|
32
|
+
/**
|
|
33
|
+
* Extract content from a URL
|
|
34
|
+
*/
|
|
35
|
+
extractContent(url: string): Promise<ExtractedContent | null>;
|
|
36
|
+
/**
|
|
37
|
+
* Extract content from multiple URLs with configurable concurrency
|
|
38
|
+
*/
|
|
39
|
+
extractBatch(urls: string[], options?: {
|
|
40
|
+
concurrency?: number;
|
|
41
|
+
onProgress?: (completed: number, total: number, url: string) => void;
|
|
42
|
+
}): Promise<(ExtractedContent | null)[]>;
|
|
43
|
+
private fetchContent;
|
|
44
|
+
private extractFromHTML;
|
|
45
|
+
private extractWithReadability;
|
|
46
|
+
private extractWithFallback;
|
|
47
|
+
private extractStructuredData;
|
|
48
|
+
private extractPublishedTime;
|
|
49
|
+
private extractSiteName;
|
|
50
|
+
private extractLanguage;
|
|
51
|
+
private countWords;
|
|
52
|
+
/**
|
|
53
|
+
* Validate extracted content quality
|
|
54
|
+
*/
|
|
55
|
+
validateContent(content: ExtractedContent): {
|
|
56
|
+
isValid: boolean;
|
|
57
|
+
issues: string[];
|
|
58
|
+
score: number;
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
export declare const globalContentExtractor: ContentExtractor;
|
|
62
|
+
//# sourceMappingURL=content-extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content-extractor.d.ts","sourceRoot":"","sources":["../../../lib/web-scrapers/content-extractor.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IAGd,UAAU,CAAC,EAAE;QACX,MAAM,CAAC,EAAE,GAAG,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACrC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC;KACnB,CAAC;IAGF,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IAGnB,gBAAgB,EAAE,aAAa,GAAG,UAAU,GAAG,YAAY,CAAC;IAC5D,WAAW,EAAE,IAAI,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAQD,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiF;IAC3G,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoB;IACnD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAO;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAO;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;;IAiDhD;;OAEG;IACG,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAgDnE;;OAEG;IACG,YAAY,CAChB,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;KACjE,GACL,OAAO,CAAC,CAAC,gBAAgB,GAAG,IAAI,CAAC,EAAE,CAAC;YA8BzB,YAAY;YA8CZ,eAAe;IAyC7B,OAAO,CAAC,sBAAsB;IA2C9B,OAAO,CAAC,mBAAmB;IAoF3B,OAAO,CAAC,qBAAqB;IAkF7B,OAAO,CAAC,oBAAoB;IAkC5B,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,UAAU;IAKlB;;OAEG;IACH,eAAe,CAAC,OAAO,EAAE,gBAAgB,GAAG;QAC1C,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;KACf;CA6CF;AAGD,eAAO,MAAM,sBAAsB,kBAAyB,CAAC"}
|