dsh-web-search-pro 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/LOGIN.md +67 -0
- package/README.md +123 -0
- package/cordis.patch.yml +21 -0
- package/lib/browser-service.d.ts +68 -0
- package/lib/browser-service.js +8 -0
- package/lib/browser-service.js.map +1 -0
- package/lib/config.d.ts +91 -0
- package/lib/config.js +72 -0
- package/lib/config.js.map +1 -0
- package/lib/deps.d.ts +34 -0
- package/lib/deps.js +90 -0
- package/lib/deps.js.map +1 -0
- package/lib/engines.d.ts +71 -0
- package/lib/engines.js +562 -0
- package/lib/engines.js.map +1 -0
- package/lib/extract.d.ts +40 -0
- package/lib/extract.js +243 -0
- package/lib/extract.js.map +1 -0
- package/lib/fetch.d.ts +42 -0
- package/lib/fetch.js +175 -0
- package/lib/fetch.js.map +1 -0
- package/lib/index.d.ts +23 -0
- package/lib/index.js +117 -0
- package/lib/index.js.map +1 -0
- package/lib/memory-cache.d.ts +18 -0
- package/lib/memory-cache.js +42 -0
- package/lib/memory-cache.js.map +1 -0
- package/lib/platform-search.d.ts +34 -0
- package/lib/platform-search.js +61 -0
- package/lib/platform-search.js.map +1 -0
- package/lib/playwright.d.ts +71 -0
- package/lib/playwright.js +165 -0
- package/lib/playwright.js.map +1 -0
- package/lib/router.d.ts +66 -0
- package/lib/router.js +332 -0
- package/lib/router.js.map +1 -0
- package/lib/store.d.ts +117 -0
- package/lib/store.js +223 -0
- package/lib/store.js.map +1 -0
- package/lib/tools.d.ts +28 -0
- package/lib/tools.js +542 -0
- package/lib/tools.js.map +1 -0
- package/lib/util.d.ts +73 -0
- package/lib/util.js +220 -0
- package/lib/util.js.map +1 -0
- package/package.json +59 -0
- package/scripts/save-login.mjs +77 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engines.js","sourceRoot":"","sources":["../src/engines.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAE1F,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAA2B,MAAM,sBAAsB,CAAA;AAiBxG,MAAM,OAAO,WAAY,SAAQ,KAAK;IACE;IAAuB;IAA7D,YAAY,OAAe,EAAW,IAAY,EAAW,YAAY,IAAI;QAC3E,KAAK,CAAC,OAAO,CAAC,CAAA;QADsB,SAAI,GAAJ,IAAI,CAAQ;QAAW,cAAS,GAAT,SAAS,CAAO;QAE3E,IAAI,CAAC,IAAI,GAAG,aAAa,CAAA;IAC3B,CAAC;CACF;AAmBD,SAAS,WAAW,CAAI,OAAmB,EAAE,EAAU,EAAE,KAAa;IACpE,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,GAAG,YAAY,EAAE,gBAAgB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACnG,OAAO,CAAC,IAAI,CACV,CAAC,CAAC,EAAE,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,CAAC,CAAC,EAC1C,CAAC,CAAC,EAAE,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA,CAAC,CAAC,CAC1C,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,UAAU,CAAC,IAAgB;IACzC,OAAO;QACL,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,yBAAyB;QAChC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ;QAC7C,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,WAAW,CAAC,0BAA0B,EAAE,oBAAoB,CAAC,CAAA;YACtF,MAAM,MAAM,GAAoB,MAAM,WAAW,CAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,EACrD,MAAM,EACN,aAAa,CACd,CAAA;YACD,OAAO,EAAE,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QAC7G,CAAC;KACF,CAAA;AACH,CAAC;AAED,gFAAgF;AAEhF,MAAM,UAAU,SAAS,CAAC,IAAgB;IACxC,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAA;IAC3D,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,KAAK;QACZ,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;QACzC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,2BAA2B,EAAE;gBACrD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,WAAW,EAAE,GAAG,EAAG,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBACpE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC3H,MAAM;gBACN,SAAS,EAAE,MAAM;aAClB,CAAC,CAAA;YACF,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,WAAW,CAAC,qBAAqB,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;YACtF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAoG,CAAA;YACpI,MAAM,OAAO,GAAsB,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAChE,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE;gBAChB,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;gBACpC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClF,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE;aAC3D,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YACjC,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,SAAS;IACvB,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,YAAY;QACnB,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI;QACrB,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,sCAAsC,GAAG,kBAAkB,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;YAC5H,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,WAAW,CAAC,kBAAkB,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;YACnF,MAAM,OAAO,GAAsB,EAAE,CAAA;YACrC,MAAM,OAAO,GAAG,uJAAuJ,CAAA;YACvK,IAAI,CAAyB,CAAA;YAC7B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;gBAC1B,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;gBACtC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;gBAC1C,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;gBACzD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;oBAAE,SAAQ;gBAC5D,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;gBACtG,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAK;YACpC,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,WAAW,CAAC,sDAAsD,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;YACxH,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,UAAU;IACxB,OAAO;QACL,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,MAAM;QACb,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI;QACrB,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,OAAO,CACvB,gCAAgC,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,EACzG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAC9B,CAAA;YACD,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,WAAW,CAAC,YAAY,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;YAC7E,MAAM,OAAO,GAAsB,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;YAC5D,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,WAAW,CAAC,0BAA0B,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;YAC5F,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,KAAK,GAAG,EAAE;IAC9C,MAAM,OAAO,GAAsB,EAAE,CAAA;IACrC,MAAM,MAAM,GAAG,mDAAmD,CAAA;IAClE,IAAI,CAAyB,CAAA;IAC7B,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QACxB,MAAM,IAAI,GAAG,CAAC,GAAW,EAAsB,EAAE;YAC/C,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,GAAG,GAAG,4BAA4B,GAAG,GAAG,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC3F,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QACtD,CAAC,CAAA;QACD,MAAM,SAAS,GAAG,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACzG,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;QAC3B,oDAAoD;QACpD,MAAM,OAAO,GAAG,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACxD,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;QAC9H,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAA;QAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAA;QACvE,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAQ;QAClD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,IAAI;YACT,GAAG,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;YAC7C,GAAG,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YACtF,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;SAC3C,CAAC,CAAA;QACF,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAK;IACpC,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,MAAM,CAAC,GAAG,4BAA4B,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,gBAAgB,CAAC,IAAgB;IAC/C,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAA;IAC7D,OAAO;QACL,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,SAAS;QAChB,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI;QACrB,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,OAAO,GAA2B,EAAE,CAAA;YAC1C,MAAM,CAAC,GAAG,GAAG,EAAE,CAAA;YACf,IAAI,CAAC;gBAAE,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,GAAG,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,uBAAuB,GAAG,kBAAkB,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;YACtH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC;gBAAE,MAAM,IAAI,WAAW,CAAC,+DAA+D,EAAE,oBAAoB,EAAE,KAAK,CAAC,CAAA;YACjJ,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,WAAW,CAAC,mBAAmB,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;YACpF,MAAM,OAAO,GAAsB,EAAE,CAAA;YACrC,MAAM,MAAM,GAAG,oEAAoE,CAAA;YACnF,IAAI,CAAyB,CAAA;YAC7B,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5C,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;gBACtB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,SAAQ;gBACxC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG;oBACH,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE;oBAC5D,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;iBAC7E,CAAC,CAAA;gBACF,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAK;YACpC,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,2DAA2D;gBAC3D,MAAM,IAAI,WAAW,CAAC,oCAAoC,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;YACnF,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,YAAY,CAAC,IAAgB;IAC3C,OAAO;QACL,EAAE,EAAE,QAAQ;QACZ,KAAK,EAAE,QAAQ;QACf,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;QAC/B,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,6DAA6D,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;YAC1M,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,MAAM,IAAI,WAAW,CAAC,oBAAoB,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAA;YACjH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAA6G,CAAA;YAC/I,MAAM,OAAO,GAAsB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gBAC9C,MAAM,KAAK,GAAG,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAA;gBACvE,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;gBACtD,OAAO;oBACL,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,qBAAqB,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACxD,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;oBAC1C,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,GAAG,KAAK,GAAG,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;iBACxG,CAAA;YACH,CAAC,CAAC,CAAA;YACF,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,gBAAgB,CAAC,IAAgB;IAC/C,OAAO;QACL,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW;QACrC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;QAC/B,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,qBAAqB,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;YACjK,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,MAAM,IAAI,WAAW,CAAC,yBAAyB,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAA;YACtH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAA+E,CAAA;YACjH,MAAM,OAAO,GAAsB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAChD,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE;gBAChB,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,UAAU,CAAC,aAAa,GAAG,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,IAAI,YAAY,EAAE;gBACvI,GAAG,CAAC,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE;aACvF,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC5C,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAgB;IACjD,OAAO;QACL,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe;QAC3C,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;QAC/B,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,0CAA0C,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;YACxL,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,MAAM,IAAI,WAAW,CAAC,0BAA0B,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAA;YACvH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAwH,CAAA;YAC1J,MAAM,OAAO,GAAsB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAChD,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE;gBAChB,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;gBACpC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;aACtK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC5C,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,cAAc,CAAC,IAAgB;IAC7C,OAAO;QACL,EAAE,EAAE,UAAU;QACd,KAAK,EAAE,eAAe;QACtB,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;QAC/B,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;YACxI,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,MAAM,IAAI,WAAW,CAAC,sBAAsB,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAA;YACnH,IAAI,IAAS,CAAA;YACb,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAQ,CAAA;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,WAAW,CAAC,2BAA2B,EAAE,cAAc,CAAC,CAAA;YACpE,CAAC;YACD,MAAM,KAAK,GAAoG,IAAI,EAAE,IAAI,IAAI,EAAE,CAAA;YAC/H,MAAM,OAAO,GAAsB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrE,GAAG,EAAE,iCAAiC,GAAG,CAAC,CAAC,IAAI;gBAC/C,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;gBACpC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;aAC3K,CAAC,CAAC,CAAA;YACH,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,UAAU;IACxB,OAAO;QACL,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,eAAe;QACtB,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI;QACrB,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,sCAAsC,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;YAC7J,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,WAAW,CAAC,cAAc,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;YAC/E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAEjC,CAAA;YACD,yEAAyE;YACzE,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;gBACxC,CAAC,CAAC,MAAM,CAAC,IAAI;gBACb,CAAC,CAAC,CAAE,MAAM,CAAC,IAAyC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAA;YACnE,MAAM,OAAO,GAAuB,OAA4I,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gBACvL,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAA;gBACnB,MAAM,GAAG,GAAG,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,yBAAyB,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;gBACxE,MAAM,OAAO,GAAG,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,CAAA;gBACnH,OAAO;oBACL,GAAG,EAAE,GAAG,IAAI,EAAE;oBACd,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;oBACrC,GAAG,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;oBAC5I,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;iBACnD,CAAA;YACH,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YAChC,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,aAAa,CAAC,IAAgB;IAC5C,OAAO;QACL,EAAE,EAAE,SAAS;QACb,KAAK,EAAE,kBAAkB;QACzB,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;QAC/B,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;YAC7B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC,UAAU,GAAG,CAAC,GAAG,GAAG,GAAG,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,SAAS,EAAE,qEAAqE,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;YAC1O,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,MAAM,IAAI,WAAW,CAAC,iBAAiB,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAA;YAC9G,MAAM,OAAO,GAAsB,EAAE,CAAA;YACrC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC7C,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBAC9D,IAAI,CAAC,EAAE,IAAI,CAAC,KAAK;oBAAE,SAAQ;gBAC3B,MAAM,IAAI,GAAa,EAAE,CAAA;gBACzB,IAAI,OAAO;oBAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBAC/B,IAAI,KAAK,IAAI,KAAK,KAAK,MAAM;oBAAE,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAA;gBAC1D,IAAI,QAAQ;oBAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBACjC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,kCAAkC,GAAG,EAAE;oBAC5C,KAAK;oBACL,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;iBACpD,CAAC,CAAA;gBACF,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;oBAAE,MAAK;YAChC,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,WAAW,CAAC,4BAA4B,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;YAC9F,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,iBAAiB,GAA2B;IAChD,WAAW,EAAE,aAAa;IAC1B,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,QAAQ;IAChB,SAAS,EAAE,WAAW;IACtB,QAAQ,EAAE,UAAU;CACrB,CAAA;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB,EAAE,IAAgB;IAC9D,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAC3C,OAAO;QACL,EAAE,EAAE,UAAU,GAAG,QAAQ;QACzB,KAAK,EAAE,UAAU,GAAG,QAAQ;QAC5B,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO;QACrF,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,WAAW,CAAC,0CAA0C,GAAG,QAAQ,EAAE,oBAAoB,EAAE,KAAK,CAAC,CAAA;YACxI,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;YAC/G,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACnB,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,OAAO,GAAG,GAAG,CAAC,IAAI,CAAA;gBACxE,MAAM,IAAI,WAAW,CAAC,UAAU,GAAG,QAAQ,GAAG,+CAA+C,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,oBAAoB,EAAE,KAAK,CAAC,CAAA;YACjJ,CAAC;YACD,IAAI,IAAI,GAAU,EAAE,CAAA;YACpB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;gBACtC,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAiC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAU,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAC3K,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC;oBAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAU,CAAA;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAC5E,CAAC;YACD,MAAM,OAAO,GAAsB,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBACvE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC5C,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;gBAC5F,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;aAC3K,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC5C,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,WAAW,CAAC,UAAU,GAAG,QAAQ,GAAG,gCAAgC,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAC3H,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,gBAAgB,CAAC,QAAgB,EAAE,IAAgB;IACjE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO;YACL,EAAE,EAAE,oBAAoB;YACxB,KAAK,EAAE,yBAAyB;YAChC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW;YAC1H,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;gBAC/B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;gBACxH,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;oBAAE,MAAM,IAAI,WAAW,CAAC,yBAAyB,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAA;gBACtH,MAAM,OAAO,GAAsB,EAAE,CAAA;gBACrC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7C,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;oBAC1C,IAAI,CAAC,CAAC;wBAAE,SAAQ;oBAChB,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBACvD,IAAI,KAAK;wBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBACnE,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;wBAAE,MAAK;gBACpC,CAAC;gBACD,OAAO,EAAE,OAAO,EAAE,CAAA;YACpB,CAAC;SACF,CAAA;IACH,CAAC;IACD,OAAO;QACL,EAAE,EAAE,aAAa,GAAG,QAAQ;QAC5B,KAAK,EAAE,cAAc,GAAG,QAAQ;QAChC,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK;QACtB,KAAK,CAAC,MAAM;YACV,MAAM,IAAI,WAAW,CAAC,iCAAiC,GAAG,QAAQ,EAAE,oBAAoB,EAAE,KAAK,CAAC,CAAA;QAClG,CAAC;KACF,CAAA;AACH,CAAC;AAED,8EAA8E;AAE9E,MAAM,UAAU,WAAW;IACzB,OAAO;QACL,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO;QAC3B,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI;QACrB,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,GAAG,GAAG,MAAM,OAAO,CACvB,qDAAqD,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,EACjI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAC9B,CAAA;YACD,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,WAAW,CAAC,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;YAC9E,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;YACzC,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,WAAW,CAAC,2BAA2B,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;YAC7F,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO;QACL,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ;QAC7B,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI;QACrB,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAC1C,MAAM,OAAO,GAAG,MAAM,OAAO,CAC3B,4EAA4E,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,UAAU,GAAG,CAAC,GAAG,eAAe,EAC3I,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAC9B,CAAA;YACD,IAAI,CAAC,OAAO,CAAC,EAAE;gBAAE,MAAM,IAAI,WAAW,CAAC,sBAAsB,GAAG,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;YAC/F,MAAM,GAAG,GAAc,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAS,EAAE,aAAa,EAAE,MAAM,IAAI,EAAE,CAAA;YACpF,IAAI,CAAC,GAAG,CAAC,MAAM;gBAAE,MAAM,IAAI,WAAW,CAAC,4BAA4B,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;YAC1F,MAAM,OAAO,GAAsB,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,kCAAkC,GAAG,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,SAAS,GAAG,EAAE,EAAE,CAAC,CAAC,CAAA;YACjI,MAAM,QAAQ,GAAG,MAAM,OAAO,CAC5B,wFAAwF,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EACxG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAC9B,CAAA;YACD,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAS,EAAE,MAAM,IAAI,EAAE,CAAA;gBAC/D,OAAO;oBACL,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;wBACpB,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAA;wBACtB,OAAO;4BACL,GAAG,EAAE,kCAAkC,GAAG,EAAE,GAAG,GAAG;4BAClD,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,GAAG,EAAE,EAAE;4BACxE,GAAG,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;yBAC5D,CAAA;oBACH,CAAC,CAAC;iBACH,CAAA;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,8EAA8E;AAE9E,MAAM,UAAU,oBAAoB,CAAC,EAAU,EAAE,IAAwB,EAAE,IAAgB;IACzF,MAAM,UAAU,GAAuB;QACrC,EAAE,EAAE,SAAS,GAAG,EAAE;QAClB,KAAK,EAAE,IAAI,CAAC,IAAI;QAChB,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG;QACnB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;KACxC,CAAA;IACD,OAAO;QACL,EAAE,EAAE,SAAS,GAAG,EAAE;QAClB,KAAK,EAAE,IAAI,CAAC,IAAI,GAAG,QAAQ;QAC3B,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO;QAC/B,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,WAAW,CAAC,yDAAyD,EAAE,oBAAoB,EAAE,KAAK,CAAC,CAAA;YAChI,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YACnE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YAC7E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;YAC7F,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,WAAW,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,GAAG,4DAA4D,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YACtJ,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,8EAA8E;AAE9E,MAAM,UAAU,wBAAwB,CAAC,QAAgB,EAAE,IAAgB;IACzE,MAAM,OAAO,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAA;IAC/C,OAAO;QACL,EAAE,EAAE,aAAa,GAAG,QAAQ;QAC5B,KAAK,EAAE,CAAC,OAAO,EAAE,KAAK,IAAI,QAAQ,CAAC,GAAG,eAAe;QACrD,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO;QAC5C,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAC/B,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,WAAW,CAAC,6CAA6C,GAAG,QAAQ,EAAE,oBAAoB,EAAE,KAAK,CAAC,CAAA;YAC3I,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAA;YAC/C,MAAM,IAAI,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,QAAQ,IAAI,EAAE,EAAoB,CAAA;YAChE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;YAC1F,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,IAAI,WAAW,CACnB,OAAO,CAAC,KAAK,GAAG,kJAAkJ,GAAG,QAAQ,GAAG,WAAW,EAC3L,cAAc,EACd,KAAK,CACN,CAAA;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,+EAA+E;AAE/E,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,MAAM,GAAG,GAAG;QACnB,SAAS,EAAE,GAAG,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1C,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM;YAChC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;YAC7D,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,WAAW,CAAC,WAAW,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;YAC5E,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;YACzC,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,WAAW,CAAC,uBAAuB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAC1F,OAAO,EAAE,OAAO,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;AACH,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,IAAgB;IAChE,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAA;QAC1C,KAAK,aAAa,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAA;QACnD,KAAK,eAAe,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAA;QACvD,KAAK,UAAU,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9C,KAAK,SAAS,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAA;QAC5C,KAAK,MAAM,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;QAClC,KAAK,aAAa,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAA;QAC/D,KAAK,SAAS,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;QAC1F,KAAK,QAAQ,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAA;QACrD,KAAK,WAAW,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAA;QAC3D,KAAK,UAAU,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAA;QACzD,yFAAyF;QACzF,KAAK,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA;QACpC,KAAK,QAAQ,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;QACtC,KAAK,OAAO,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;QAC9D,KAAK,OAAO,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;QAC9D,KAAK,QAAQ,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAA;QAChE,KAAK,OAAO,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;QAC9D,KAAK,QAAQ,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAA;QAChE,KAAK,UAAU,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAA;QACpE,OAAO,CAAC,CAAC,OAAO,EAAE,CAAA;IACpB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAU,CAAA;AAC5I,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,eAAe,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAU,CAAA"}
|
package/lib/extract.d.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Userscript-style page extraction ("脚本猫/油猴" inspired): declarative
|
|
3
|
+
* per-hostname rules (content + removal selectors) applied over a jsdom
|
|
4
|
+
* parse, with generic readability fallbacks and a DOM→text walker.
|
|
5
|
+
* Built-in rules cover common Chinese/global sites; users can add persistent
|
|
6
|
+
* rules through the web_rule tool (stored in SQLite).
|
|
7
|
+
* @module web-search-pro/extract
|
|
8
|
+
*/
|
|
9
|
+
export interface ExtractRule {
|
|
10
|
+
hostname: string;
|
|
11
|
+
contentSelectors: string[];
|
|
12
|
+
removeSelectors?: string[];
|
|
13
|
+
}
|
|
14
|
+
/** Built-in extraction rules, applied after user rules (user rules win). */
|
|
15
|
+
export declare const BUILTIN_RULES: ExtractRule[];
|
|
16
|
+
/** Normalize a hostname for matching (strip www. and port). */
|
|
17
|
+
export declare function normalizeHost(hostname: string): string;
|
|
18
|
+
/**
|
|
19
|
+
* Find the best rule for a hostname: longest suffix match against user +
|
|
20
|
+
* built-in rules.
|
|
21
|
+
*/
|
|
22
|
+
export declare function matchRule(hostname: string, rules: readonly ExtractRule[]): ExtractRule | undefined;
|
|
23
|
+
export interface ExtractResult {
|
|
24
|
+
title: string;
|
|
25
|
+
text: string;
|
|
26
|
+
usedRule: string | undefined;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Extract readable text + title from an HTML document.
|
|
30
|
+
* @param html - raw HTML.
|
|
31
|
+
* @param url - page URL (for hostname rule matching).
|
|
32
|
+
* @param rules - merged rule list (user rules first, built-ins appended).
|
|
33
|
+
* @param maxChars - output cap.
|
|
34
|
+
*/
|
|
35
|
+
export declare function extractText(html: string, url: string, rules: readonly ExtractRule[], maxChars?: number): ExtractResult;
|
|
36
|
+
/** Extract titles + links from HTML search result lists (generic). */
|
|
37
|
+
export declare function extractResultLinks(html: string): {
|
|
38
|
+
title: string;
|
|
39
|
+
url: string;
|
|
40
|
+
}[];
|
package/lib/extract.js
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Userscript-style page extraction ("脚本猫/油猴" inspired): declarative
|
|
3
|
+
* per-hostname rules (content + removal selectors) applied over a jsdom
|
|
4
|
+
* parse, with generic readability fallbacks and a DOM→text walker.
|
|
5
|
+
* Built-in rules cover common Chinese/global sites; users can add persistent
|
|
6
|
+
* rules through the web_rule tool (stored in SQLite).
|
|
7
|
+
* @module web-search-pro/extract
|
|
8
|
+
*/
|
|
9
|
+
import { jsdom } from "./util.js";
|
|
10
|
+
/** Backtick character (kept in a constant so fenced code blocks stay readable). */
|
|
11
|
+
const BT = String.fromCharCode(96);
|
|
12
|
+
/** Built-in extraction rules, applied after user rules (user rules win). */
|
|
13
|
+
export const BUILTIN_RULES = [
|
|
14
|
+
{ hostname: 'zhihu.com', contentSelectors: ['.RichContent-inner', '.RichText', 'article'] },
|
|
15
|
+
{ hostname: 'bilibili.com', contentSelectors: ['#v_desc', '.article-content', '.article-container', 'article'] },
|
|
16
|
+
{ hostname: 'xiaohongshu.com', contentSelectors: ['#detail-desc', '.note-content', '.note-text', 'article'] },
|
|
17
|
+
{ hostname: 'weibo.com', contentSelectors: ['.WB_text', '.woo-box-flex', 'article'] },
|
|
18
|
+
{ hostname: 'weibo.cn', contentSelectors: ['.ctt', 'article'] },
|
|
19
|
+
{ hostname: 'github.com', contentSelectors: ['article.markdown-body', '.markdown-body', 'main'] },
|
|
20
|
+
{ hostname: 'juejin.cn', contentSelectors: ['.article-content', 'article'] },
|
|
21
|
+
{ hostname: 'csdn.net', contentSelectors: ['#content_views', 'article'] },
|
|
22
|
+
{ hostname: 'medium.com', contentSelectors: ['article'] },
|
|
23
|
+
{ hostname: 'wikipedia.org', contentSelectors: ['#mw-content-text'] },
|
|
24
|
+
{ hostname: 'stackoverflow.com', contentSelectors: ['.post-text', 'article'] },
|
|
25
|
+
{ hostname: 'douban.com', contentSelectors: ['.note', '#link-report', 'article'] },
|
|
26
|
+
{ hostname: 'youtube.com', contentSelectors: ['#description-inline-expander', '#description'] },
|
|
27
|
+
{ hostname: 'v2ex.com', contentSelectors: ['.topic_content', '.markdown_body'] },
|
|
28
|
+
{ hostname: '36kr.com', contentSelectors: ['article'] },
|
|
29
|
+
{ hostname: 'sspai.com', contentSelectors: ['article'] },
|
|
30
|
+
{ hostname: 'ithome.com', contentSelectors: ['#paragraph', 'article'] },
|
|
31
|
+
{ hostname: 'huxiu.com', contentSelectors: ['#article_content', 'article'] },
|
|
32
|
+
];
|
|
33
|
+
/** Normalize a hostname for matching (strip www. and port). */
|
|
34
|
+
export function normalizeHost(hostname) {
|
|
35
|
+
return hostname.replace(/^www\./i, '').toLowerCase();
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Find the best rule for a hostname: longest suffix match against user +
|
|
39
|
+
* built-in rules.
|
|
40
|
+
*/
|
|
41
|
+
export function matchRule(hostname, rules) {
|
|
42
|
+
const host = normalizeHost(hostname);
|
|
43
|
+
let best;
|
|
44
|
+
let bestLen = -1;
|
|
45
|
+
for (const rule of rules) {
|
|
46
|
+
const r = normalizeHost(rule.hostname);
|
|
47
|
+
if ((host === r || host.endsWith('.' + r)) && r.length > bestLen) {
|
|
48
|
+
best = rule;
|
|
49
|
+
bestLen = r.length;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return best;
|
|
53
|
+
}
|
|
54
|
+
/** Elements treated as block separators during text walking. */
|
|
55
|
+
const BLOCK_TAGS = new Set([
|
|
56
|
+
'P', 'DIV', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'LI', 'UL', 'OL', 'TABLE', 'TR',
|
|
57
|
+
'BLOCKQUOTE', 'SECTION', 'ARTICLE', 'HEADER', 'FOOTER', 'FIGURE', 'FIGCAPTION',
|
|
58
|
+
'PRE', 'BR', 'HR', 'TD', 'TH', 'MAIN', 'ASIDE', 'DL', 'DT', 'DD',
|
|
59
|
+
]);
|
|
60
|
+
/** Elements skipped entirely during extraction. */
|
|
61
|
+
const SKIP_TAGS = new Set([
|
|
62
|
+
'SCRIPT', 'STYLE', 'NOSCRIPT', 'SVG', 'IFRAME', 'TEMPLATE', 'VIDEO', 'AUDIO',
|
|
63
|
+
'BUTTON', 'INPUT', 'SELECT', 'TEXTAREA', 'NAV', 'FORM', 'META', 'LINK',
|
|
64
|
+
]);
|
|
65
|
+
/** Convert one DOM element subtree to readable text. */
|
|
66
|
+
function elementToText(node, depth) {
|
|
67
|
+
if (depth > 40)
|
|
68
|
+
return '';
|
|
69
|
+
if (node.nodeType === 3) { // text node
|
|
70
|
+
return node.textContent ?? '';
|
|
71
|
+
}
|
|
72
|
+
if (node.nodeType !== 1)
|
|
73
|
+
return '';
|
|
74
|
+
const tag = (node.tagName ?? '').toUpperCase();
|
|
75
|
+
if (SKIP_TAGS.has(tag))
|
|
76
|
+
return '';
|
|
77
|
+
if (tag === 'PRE') {
|
|
78
|
+
const code = (node.textContent ?? '').replace(/^\n+|\n+$/g, '');
|
|
79
|
+
return '\n' + BT.repeat(3) + '\n' + code + '\n' + BT.repeat(3) + '\n';
|
|
80
|
+
}
|
|
81
|
+
if (tag === 'CODE') {
|
|
82
|
+
return BT + (node.textContent ?? '') + BT;
|
|
83
|
+
}
|
|
84
|
+
if (tag === 'A') {
|
|
85
|
+
const text = (node.textContent ?? '').trim();
|
|
86
|
+
const href = node.getAttribute?.('href') ?? '';
|
|
87
|
+
if (!text)
|
|
88
|
+
return '';
|
|
89
|
+
if (/^https?:\/\//i.test(href) && href.length < 300)
|
|
90
|
+
return '[' + text + '](' + href + ')';
|
|
91
|
+
return text;
|
|
92
|
+
}
|
|
93
|
+
if (tag === 'IMG') {
|
|
94
|
+
const alt = node.getAttribute?.('alt');
|
|
95
|
+
return alt ? '[图片: ' + alt + ']' : '';
|
|
96
|
+
}
|
|
97
|
+
let out = '';
|
|
98
|
+
const children = node.childNodes ?? [];
|
|
99
|
+
for (let i = 0; i < children.length; i++) {
|
|
100
|
+
out += elementToText(children[i], depth + 1);
|
|
101
|
+
}
|
|
102
|
+
if (BLOCK_TAGS.has(tag))
|
|
103
|
+
out = '\n' + out + '\n';
|
|
104
|
+
return out;
|
|
105
|
+
}
|
|
106
|
+
/** Pick the main content element: rule selectors first, then generic heuristics. */
|
|
107
|
+
function pickContent(document, rule) {
|
|
108
|
+
if (rule && rule.contentSelectors.length) {
|
|
109
|
+
for (const sel of rule.contentSelectors) {
|
|
110
|
+
try {
|
|
111
|
+
const el = document.querySelector(sel);
|
|
112
|
+
if (el && (el.textContent ?? '').trim().length > 40)
|
|
113
|
+
return el;
|
|
114
|
+
}
|
|
115
|
+
catch { /* invalid selector */ }
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
const candidates = ['article', 'main', '[role="main"]', '[itemprop="articleBody"]', '.article-content', '.post-content', '.rich_media_content', '#js_content'];
|
|
119
|
+
for (const sel of candidates) {
|
|
120
|
+
try {
|
|
121
|
+
const el = document.querySelector(sel);
|
|
122
|
+
if (el && (el.textContent ?? '').trim().length > 60)
|
|
123
|
+
return el;
|
|
124
|
+
}
|
|
125
|
+
catch { /* invalid selector */ }
|
|
126
|
+
}
|
|
127
|
+
const body = document.body;
|
|
128
|
+
if (body) {
|
|
129
|
+
const text = (body.textContent ?? '').trim();
|
|
130
|
+
if (text.length > 60)
|
|
131
|
+
return body;
|
|
132
|
+
}
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
/** Try JSON-LD articleBody as the extraction source. */
|
|
136
|
+
function jsonLdBody(document) {
|
|
137
|
+
const scripts = document.querySelectorAll('script[type="application/ld+json"]');
|
|
138
|
+
for (const s of scripts) {
|
|
139
|
+
try {
|
|
140
|
+
const data = JSON.parse(s.textContent ?? '');
|
|
141
|
+
const list = Array.isArray(data) ? data : [data];
|
|
142
|
+
for (const item of list) {
|
|
143
|
+
const body = item?.articleBody;
|
|
144
|
+
if (typeof body === 'string' && body.trim().length > 60)
|
|
145
|
+
return body;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
catch { /* not JSON */ }
|
|
149
|
+
}
|
|
150
|
+
return undefined;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Extract readable text + title from an HTML document.
|
|
154
|
+
* @param html - raw HTML.
|
|
155
|
+
* @param url - page URL (for hostname rule matching).
|
|
156
|
+
* @param rules - merged rule list (user rules first, built-ins appended).
|
|
157
|
+
* @param maxChars - output cap.
|
|
158
|
+
*/
|
|
159
|
+
export function extractText(html, url, rules, maxChars = 200_000) {
|
|
160
|
+
let dom;
|
|
161
|
+
try {
|
|
162
|
+
dom = new jsdom(html, { url });
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return { title: '', text: stripRough(html).slice(0, maxChars), usedRule: undefined };
|
|
166
|
+
}
|
|
167
|
+
const document = dom.window.document;
|
|
168
|
+
let title = document.title?.trim() ?? '';
|
|
169
|
+
const og = document.querySelector('meta[property="og:title"]');
|
|
170
|
+
if (og?.content && og.content.trim().length > title.length)
|
|
171
|
+
title = og.content.trim();
|
|
172
|
+
const h1 = document.querySelector('h1');
|
|
173
|
+
if (h1 && !title && (h1.textContent ?? '').trim())
|
|
174
|
+
title = h1.textContent.trim();
|
|
175
|
+
let host = '';
|
|
176
|
+
try {
|
|
177
|
+
host = new URL(url).hostname;
|
|
178
|
+
}
|
|
179
|
+
catch { /* ignore */ }
|
|
180
|
+
const rule = matchRule(host, rules);
|
|
181
|
+
let text = '';
|
|
182
|
+
const ld = jsonLdBody(document);
|
|
183
|
+
if (ld) {
|
|
184
|
+
text = ld;
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
const content = pickContent(document, rule);
|
|
188
|
+
if (content) {
|
|
189
|
+
if (rule?.removeSelectors) {
|
|
190
|
+
for (const sel of rule.removeSelectors) {
|
|
191
|
+
try {
|
|
192
|
+
content.querySelectorAll(sel).forEach((el) => el.remove());
|
|
193
|
+
}
|
|
194
|
+
catch { /* ignore */ }
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
if (!rule) {
|
|
198
|
+
for (const sel of ['nav', 'aside', 'footer', 'form', 'script', 'style']) {
|
|
199
|
+
try {
|
|
200
|
+
content.querySelectorAll(sel).forEach((el) => el.remove());
|
|
201
|
+
}
|
|
202
|
+
catch { /* ignore */ }
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
text = elementToText(content, 0);
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
text = '';
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
const cleaned = text
|
|
212
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
213
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
214
|
+
.trim();
|
|
215
|
+
return { title, text: cleaned.slice(0, maxChars), usedRule: rule?.hostname };
|
|
216
|
+
}
|
|
217
|
+
/** Minimal tag stripping for non-HTML fallback paths. */
|
|
218
|
+
function stripRough(html) {
|
|
219
|
+
return html
|
|
220
|
+
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
|
|
221
|
+
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
|
|
222
|
+
.replace(/<[^>]*>/g, ' ')
|
|
223
|
+
.replace(/\s+/g, ' ')
|
|
224
|
+
.trim();
|
|
225
|
+
}
|
|
226
|
+
/** Extract titles + links from HTML search result lists (generic). */
|
|
227
|
+
export function extractResultLinks(html) {
|
|
228
|
+
const out = [];
|
|
229
|
+
const anchorRe = /<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
|
|
230
|
+
let m;
|
|
231
|
+
while ((m = anchorRe.exec(html)) !== null) {
|
|
232
|
+
const href = m[1];
|
|
233
|
+
const inner = stripTags(m[2]).replace(/\s+/g, ' ').trim();
|
|
234
|
+
if (!/^https?:\/\//i.test(href) || inner.length < 4)
|
|
235
|
+
continue;
|
|
236
|
+
out.push({ title: inner, url: href });
|
|
237
|
+
if (out.length >= 50)
|
|
238
|
+
break;
|
|
239
|
+
}
|
|
240
|
+
return out;
|
|
241
|
+
}
|
|
242
|
+
import { stripTags } from "./util.js";
|
|
243
|
+
//# sourceMappingURL=extract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AAEjC,mFAAmF;AACnF,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;AAQlC,4EAA4E;AAC5E,MAAM,CAAC,MAAM,aAAa,GAAkB;IAC1C,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,oBAAoB,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE;IAC3F,EAAE,QAAQ,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC,SAAS,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE;IAChH,EAAE,QAAQ,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,cAAc,EAAE,eAAe,EAAE,YAAY,EAAE,SAAS,CAAC,EAAE;IAC7G,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,UAAU,EAAE,eAAe,EAAE,SAAS,CAAC,EAAE;IACrF,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE;IAC/D,EAAE,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE;IACjG,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,kBAAkB,EAAE,SAAS,CAAC,EAAE;IAC5E,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,gBAAgB,EAAE,SAAS,CAAC,EAAE;IACzE,EAAE,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,SAAS,CAAC,EAAE;IACzD,EAAE,QAAQ,EAAE,eAAe,EAAE,gBAAgB,EAAE,CAAC,kBAAkB,CAAC,EAAE;IACrE,EAAE,QAAQ,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE;IAC9E,EAAE,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,OAAO,EAAE,cAAc,EAAE,SAAS,CAAC,EAAE;IAClF,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,8BAA8B,EAAE,cAAc,CAAC,EAAE;IAC/F,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,EAAE;IAChF,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,SAAS,CAAC,EAAE;IACvD,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,SAAS,CAAC,EAAE;IACxD,EAAE,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE;IACvE,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,kBAAkB,EAAE,SAAS,CAAC,EAAE;CAC7E,CAAA;AAED,+DAA+D;AAC/D,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,OAAO,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,KAA6B;IACvE,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAA;IACpC,IAAI,IAA6B,CAAA;IACjC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAA;IAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YACjE,IAAI,GAAG,IAAI,CAAA;YACX,OAAO,GAAG,CAAC,CAAC,MAAM,CAAA;QACpB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI;IAC/E,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY;IAC9E,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;CACjE,CAAC,CAAA;AAEF,mDAAmD;AACnD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO;IAC5E,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CACvE,CAAC,CAAA;AAEF,wDAAwD;AACxD,SAAS,aAAa,CAAC,IAAS,EAAE,KAAa;IAC7C,IAAI,KAAK,GAAG,EAAE;QAAE,OAAO,EAAE,CAAA;IACzB,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC,YAAY;QACrC,OAAO,IAAI,CAAC,WAAW,IAAI,EAAE,CAAA;IAC/B,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAClC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;IAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAA;IACjC,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAA;QAC/D,OAAO,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;IACvE,CAAC;IACD,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,GAAG,EAAE,CAAA;IAC3C,CAAC;IACD,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QAC9C,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAA;QACpB,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,CAAA;QAC1F,OAAO,IAAI,CAAA;IACb,CAAC;IACD,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAA;QACtC,OAAO,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IACvC,CAAC;IACD,IAAI,GAAG,GAAG,EAAE,CAAA;IACZ,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAA;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,GAAG,IAAI,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;IAC9C,CAAC;IACD,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,GAAG,GAAG,IAAI,GAAG,GAAG,GAAG,IAAI,CAAA;IAChD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,oFAAoF;AACpF,SAAS,WAAW,CAAC,QAAa,EAAE,IAA6B;IAC/D,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC;QACzC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;gBACtC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE;oBAAE,OAAO,EAAE,CAAA;YAChE,CAAC;YAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,0BAA0B,EAAE,kBAAkB,EAAE,eAAe,EAAE,qBAAqB,EAAE,aAAa,CAAC,CAAA;IAC9J,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;YACtC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE;gBAAE,OAAO,EAAE,CAAA;QAChE,CAAC;QAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;IAC1B,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAC5C,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,IAAI,CAAA;IACnC,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,wDAAwD;AACxD,SAAS,UAAU,CAAC,QAAa;IAC/B,MAAM,OAAO,GAAG,QAAQ,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CAAA;IAC/E,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;YAC5C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAChD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,IAAI,EAAE,WAAW,CAAA;gBAC9B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE;oBAAE,OAAO,IAAI,CAAA;YACtE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,cAAc,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,GAAW,EAAE,KAA6B,EAAE,QAAQ,GAAG,OAAO;IACtG,IAAI,GAAQ,CAAA;IACZ,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAA;IACtF,CAAC;IACD,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAA;IACpC,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IACxC,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,2BAA2B,CAAC,CAAA;IAC9D,IAAI,EAAE,EAAE,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QAAE,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACrF,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;IACvC,IAAI,EAAE,IAAI,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QAAE,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAA;IAEhF,IAAI,IAAI,GAAG,EAAE,CAAA;IACb,IAAI,CAAC;QAAC,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAA;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnC,IAAI,IAAI,GAAG,EAAE,CAAA;IACb,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IAC/B,IAAI,EAAE,EAAE,CAAC;QACP,IAAI,GAAG,EAAE,CAAA;IACX,CAAC;SAAM,CAAC;QACN,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;QAC3C,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,IAAI,EAAE,eAAe,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;oBACvC,IAAI,CAAC;wBAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBAChG,CAAC;YACH,CAAC;YACD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,KAAK,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;oBACxE,IAAI,CAAC;wBAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBAChG,CAAC;YACH,CAAC;YACD,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QAClC,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,IAAI;SACjB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,IAAI,EAAE,CAAA;IACT,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;AAC9E,CAAC;AAED,yDAAyD;AACzD,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,IAAI;SACR,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;SAC3C,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;SACzC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAA;AACX,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,GAAG,GAAqC,EAAE,CAAA;IAChD,MAAM,QAAQ,GAAG,8CAA8C,CAAA;IAC/D,IAAI,CAAyB,CAAA;IAC7B,OAAO,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;QAClB,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QAC1D,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAQ;QAC7D,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QACrC,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE;YAAE,MAAK;IAC7B,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA"}
|
package/lib/fetch.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Enhanced page fetch pipeline (agent-reach Jina reader + userscript-style
|
|
3
|
+
* extraction + playwright fallback), with page snapshot persistence.
|
|
4
|
+
* @module web-search-pro/fetch
|
|
5
|
+
*/
|
|
6
|
+
import type { Store } from './store.ts';
|
|
7
|
+
import type { ResolvedConfig } from './config.ts';
|
|
8
|
+
import type { BrowserService } from './browser-service.ts';
|
|
9
|
+
import type { ExtractRule } from './extract.ts';
|
|
10
|
+
export type FetchMode = 'auto' | 'jina' | 'http' | 'playwright';
|
|
11
|
+
export interface FetchOptions {
|
|
12
|
+
mode: FetchMode;
|
|
13
|
+
signal: AbortSignal | undefined;
|
|
14
|
+
maxChars: number;
|
|
15
|
+
fresh: boolean;
|
|
16
|
+
persist: boolean;
|
|
17
|
+
}
|
|
18
|
+
export interface FetchResult {
|
|
19
|
+
url: string;
|
|
20
|
+
title?: string;
|
|
21
|
+
text: string;
|
|
22
|
+
source: string;
|
|
23
|
+
fromCache: boolean;
|
|
24
|
+
statusCode?: number;
|
|
25
|
+
usedRule?: string;
|
|
26
|
+
}
|
|
27
|
+
/** Validate and normalize a URL for fetching. */
|
|
28
|
+
export declare function normalizeUrl(raw: string): string;
|
|
29
|
+
/** All rules: user (DB) first, then built-ins; user rules win on ties. */
|
|
30
|
+
export declare function mergedRules(store: Store): ExtractRule[];
|
|
31
|
+
export declare class FetchService {
|
|
32
|
+
private readonly store;
|
|
33
|
+
private readonly config;
|
|
34
|
+
private readonly browser;
|
|
35
|
+
private readonly memory;
|
|
36
|
+
constructor(store: Store, config: ResolvedConfig | (() => ResolvedConfig), browser: BrowserService);
|
|
37
|
+
private cfg;
|
|
38
|
+
fetchPage(url: string, opts: FetchOptions): Promise<FetchResult>;
|
|
39
|
+
private fetchJina;
|
|
40
|
+
private fetchHttp;
|
|
41
|
+
private fetchPlaywright;
|
|
42
|
+
}
|
package/lib/fetch.js
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Enhanced page fetch pipeline (agent-reach Jina reader + userscript-style
|
|
3
|
+
* extraction + playwright fallback), with page snapshot persistence.
|
|
4
|
+
* @module web-search-pro/fetch
|
|
5
|
+
*/
|
|
6
|
+
import { extractText, BUILTIN_RULES } from "./extract.js";
|
|
7
|
+
import { httpGet, capText } from "./util.js";
|
|
8
|
+
import { LruCache } from "./memory-cache.js";
|
|
9
|
+
/** Validate and normalize a URL for fetching. */
|
|
10
|
+
export function normalizeUrl(raw) {
|
|
11
|
+
const trimmed = raw.trim();
|
|
12
|
+
if (!/^https?:\/\//i.test(trimmed)) {
|
|
13
|
+
throw new Error('url must be an http(s) URL');
|
|
14
|
+
}
|
|
15
|
+
let parsed;
|
|
16
|
+
try {
|
|
17
|
+
parsed = new URL(trimmed);
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
throw new Error('url is not a valid URL');
|
|
21
|
+
}
|
|
22
|
+
return parsed.href;
|
|
23
|
+
}
|
|
24
|
+
/** All rules: user (DB) first, then built-ins; user rules win on ties. */
|
|
25
|
+
export function mergedRules(store) {
|
|
26
|
+
const dbRules = store.listRules().map(r => ({
|
|
27
|
+
hostname: r.hostname,
|
|
28
|
+
contentSelectors: r.content.split(/[,\n]/).map(s => s.trim()).filter(Boolean),
|
|
29
|
+
...r.remove ? { removeSelectors: r.remove.split(/[,\n]/).map(s => s.trim()).filter(Boolean) } : {},
|
|
30
|
+
}));
|
|
31
|
+
return [...dbRules, ...BUILTIN_RULES];
|
|
32
|
+
}
|
|
33
|
+
export class FetchService {
|
|
34
|
+
store;
|
|
35
|
+
config;
|
|
36
|
+
browser;
|
|
37
|
+
memory = new LruCache(128);
|
|
38
|
+
constructor(store, config, browser) {
|
|
39
|
+
this.store = store;
|
|
40
|
+
this.config = config;
|
|
41
|
+
this.browser = browser;
|
|
42
|
+
}
|
|
43
|
+
cfg() {
|
|
44
|
+
return typeof this.config === 'function' ? this.config() : this.config;
|
|
45
|
+
}
|
|
46
|
+
async fetchPage(url, opts) {
|
|
47
|
+
const normalized = normalizeUrl(url);
|
|
48
|
+
const maxChars = Math.min(Math.max(opts.maxChars, 1_000), 500_000);
|
|
49
|
+
if (!opts.fresh) {
|
|
50
|
+
const hot = this.memory.get('page|' + normalized, this.cfg().ttlSeconds * 1000);
|
|
51
|
+
if (hot)
|
|
52
|
+
return { ...hot, fromCache: true };
|
|
53
|
+
const cached = this.store.getPage(normalized, this.cfg().ttlSeconds);
|
|
54
|
+
if (cached && cached.text) {
|
|
55
|
+
const page = {
|
|
56
|
+
url: normalized,
|
|
57
|
+
...cached.title ? { title: cached.title } : {},
|
|
58
|
+
text: capText(cached.text, maxChars),
|
|
59
|
+
source: 'cache:' + (cached.source ?? 'unknown'),
|
|
60
|
+
fromCache: true,
|
|
61
|
+
...cached.status !== undefined ? { statusCode: cached.status } : {},
|
|
62
|
+
};
|
|
63
|
+
this.memory.set('page|' + normalized, page);
|
|
64
|
+
return page;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const rules = mergedRules(this.store);
|
|
68
|
+
let result;
|
|
69
|
+
if (opts.mode === 'auto' || opts.mode === 'jina') {
|
|
70
|
+
try {
|
|
71
|
+
result = await this.fetchJina(normalized, opts, maxChars);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
if (opts.mode === 'jina')
|
|
75
|
+
throw error;
|
|
76
|
+
if (opts.signal?.aborted)
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (!result && (opts.mode === 'auto' || opts.mode === 'http')) {
|
|
81
|
+
try {
|
|
82
|
+
result = await this.fetchHttp(normalized, opts, maxChars, rules);
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
if (opts.mode === 'http')
|
|
86
|
+
throw error;
|
|
87
|
+
if (opts.signal?.aborted)
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (!result && (opts.mode === 'auto' || opts.mode === 'playwright')) {
|
|
92
|
+
if (this.cfg().playwright.enabled) {
|
|
93
|
+
result = await this.fetchPlaywright(normalized, opts, maxChars, rules);
|
|
94
|
+
}
|
|
95
|
+
else if (opts.mode === 'playwright') {
|
|
96
|
+
throw new Error('playwright backend is disabled in config');
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
if (!result) {
|
|
100
|
+
throw new Error('all fetch backends failed for ' + normalized);
|
|
101
|
+
}
|
|
102
|
+
this.memory.set('page|' + normalized, result);
|
|
103
|
+
if (opts.persist) {
|
|
104
|
+
this.store.savePage({
|
|
105
|
+
url: normalized,
|
|
106
|
+
...result.title ? { title: result.title } : {},
|
|
107
|
+
text: result.text,
|
|
108
|
+
...result.statusCode !== undefined ? { status: result.statusCode } : {},
|
|
109
|
+
source: result.source,
|
|
110
|
+
});
|
|
111
|
+
this.store.recordQuery({
|
|
112
|
+
kind: 'fetch',
|
|
113
|
+
url: normalized,
|
|
114
|
+
query: result.title ?? normalized,
|
|
115
|
+
engine: result.source,
|
|
116
|
+
status: 'ok',
|
|
117
|
+
detail: JSON.stringify({ textLength: result.text.length, usedRule: result.usedRule }),
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
return result;
|
|
121
|
+
}
|
|
122
|
+
async fetchJina(url, opts, maxChars) {
|
|
123
|
+
const cfg = this.cfg();
|
|
124
|
+
const headers = {};
|
|
125
|
+
const key = cfg.jinaApiKey || process.env[cfg.jinaApiKeyEnv];
|
|
126
|
+
if (key)
|
|
127
|
+
headers['authorization'] = 'Bearer ' + key;
|
|
128
|
+
headers['x-respond-with'] = 'markdown';
|
|
129
|
+
const res = await httpGet('https://r.jina.ai/' + url, { headers, signal: opts.signal, timeoutMs: 30_000 });
|
|
130
|
+
if (res.status === 401 && !key)
|
|
131
|
+
throw new Error('jina reader requires an API key (set jinaApiKey or $JINA_API_KEY)');
|
|
132
|
+
if (!res.ok)
|
|
133
|
+
throw new Error('jina reader HTTP ' + res.status);
|
|
134
|
+
const text = res.text;
|
|
135
|
+
// Jina returns "# Title\n\ncontent"; peel the first H1 as title when present.
|
|
136
|
+
let title;
|
|
137
|
+
let body = text;
|
|
138
|
+
const m = /^#\s+(.+?)\s*\n/.exec(text);
|
|
139
|
+
if (m) {
|
|
140
|
+
title = m[1].trim();
|
|
141
|
+
body = text.slice(m[0].length);
|
|
142
|
+
}
|
|
143
|
+
return { url, ...title ? { title } : {}, text: capText(body.trim(), maxChars), source: 'jina', fromCache: false };
|
|
144
|
+
}
|
|
145
|
+
async fetchHttp(url, opts, maxChars, rules) {
|
|
146
|
+
const res = await httpGet(url, { signal: opts.signal, timeoutMs: 30_000 });
|
|
147
|
+
const contentType = res.contentType ?? '';
|
|
148
|
+
const isHtml = /html|xml/i.test(contentType) || /<\s*!doctype|<!DOCTYPE|(<html[\s>])/i.test(res.text.slice(0, 2000));
|
|
149
|
+
if (isHtml) {
|
|
150
|
+
const extracted = extractText(res.text, res.finalUrl, rules, maxChars);
|
|
151
|
+
return {
|
|
152
|
+
url: res.finalUrl,
|
|
153
|
+
...extracted.title ? { title: extracted.title } : {},
|
|
154
|
+
text: extracted.text || capText(res.text.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim(), maxChars),
|
|
155
|
+
source: 'http',
|
|
156
|
+
fromCache: false,
|
|
157
|
+
statusCode: res.status,
|
|
158
|
+
...extracted.usedRule ? { usedRule: extracted.usedRule } : {},
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
return { url: res.finalUrl, text: capText(res.text, maxChars), source: 'http', fromCache: false, statusCode: res.status };
|
|
162
|
+
}
|
|
163
|
+
async fetchPlaywright(url, opts, maxChars, rules) {
|
|
164
|
+
const rendered = await this.browser.render(url, rules, { signal: opts.signal, maxChars });
|
|
165
|
+
return {
|
|
166
|
+
url,
|
|
167
|
+
...rendered.title ? { title: rendered.title } : {},
|
|
168
|
+
text: rendered.text,
|
|
169
|
+
source: 'playwright',
|
|
170
|
+
fromCache: false,
|
|
171
|
+
...rendered.usedRule ? { usedRule: rendered.usedRule } : {},
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=fetch.js.map
|
package/lib/fetch.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AACzD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAsB5C,iDAAiD;AACjD,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAC1B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;IAC/C,CAAC;IACD,IAAI,MAAW,CAAA;IACf,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAA;AACpB,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,WAAW,CAAC,KAAY;IACtC,MAAM,OAAO,GAAkB,KAAK,CAAC,SAAS,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACzD,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;QAC7E,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;KACnG,CAAC,CAAC,CAAA;IACH,OAAO,CAAC,GAAG,OAAO,EAAE,GAAG,aAAa,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IAIJ;IACA;IACA;IALF,MAAM,GAAG,IAAI,QAAQ,CAAc,GAAG,CAAC,CAAA;IAExD,YACmB,KAAY,EACZ,MAA+C,EAC/C,OAAuB;QAFvB,UAAK,GAAL,KAAK,CAAO;QACZ,WAAM,GAAN,MAAM,CAAyC;QAC/C,YAAO,GAAP,OAAO,CAAgB;IACvC,CAAC;IAEI,GAAG;QACT,OAAO,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;IACxE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,IAAkB;QAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,CAAA;QAElE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,GAAG,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC,CAAA;YAC/E,IAAI,GAAG;gBAAE,OAAO,EAAE,GAAG,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,CAAA;YACpE,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAgB;oBACxB,GAAG,EAAE,UAAU;oBACf,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;oBAC9C,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC;oBACpC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC;oBAC/C,SAAS,EAAE,IAAI;oBACf,GAAG,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE;iBACpE,CAAA;gBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,GAAG,UAAU,EAAE,IAAI,CAAC,CAAA;gBAC3C,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,MAA+B,CAAA;QAEnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;YAC3D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;oBAAE,MAAM,KAAK,CAAA;gBACrC,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAM,KAAK,CAAA;YACvC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,CAAC;YAC9D,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAA;YAClE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;oBAAE,MAAM,KAAK,CAAA;gBACrC,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAM,KAAK,CAAA;YACvC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;YACpE,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;gBAClC,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAA;YACxE,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACtC,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;YAC7D,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,CAAA;QAChE,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,GAAG,UAAU,EAAE,MAAM,CAAC,CAAA;QAC7C,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,GAAG,EAAE,UAAU;gBACf,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;gBAC9C,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE;gBACvE,MAAM,EAAE,MAAM,CAAC,MAAM;aACtB,CAAC,CAAA;YACF,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;gBACrB,IAAI,EAAE,OAAO;gBACb,GAAG,EAAE,UAAU;gBACf,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,UAAU;gBACjC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;aACtF,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,IAAkB,EAAE,QAAgB;QACvE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,OAAO,GAA2B,EAAE,CAAA;QAC1C,MAAM,GAAG,GAAG,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;QAC5D,IAAI,GAAG;YAAE,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,GAAG,GAAG,CAAA;QACnD,OAAO,CAAC,gBAAgB,CAAC,GAAG,UAAU,CAAA;QACtC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,oBAAoB,GAAG,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;QAC1G,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAA;QACpH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,GAAG,CAAC,MAAM,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAA;QACrB,8EAA8E;QAC9E,IAAI,KAAyB,CAAA;QAC7B,IAAI,IAAI,GAAG,IAAI,CAAA;QACf,MAAM,CAAC,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtC,IAAI,CAAC,EAAE,CAAC;YACN,KAAK,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAA;YACpB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAA;QACjC,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;IACnH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,IAAkB,EAAE,QAAgB,EAAE,KAAoB;QAC7F,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;QAC1E,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,IAAI,EAAE,CAAA;QACzC,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,sCAAsC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAA;QACpH,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;YACtE,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;gBACpD,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC;gBACxG,MAAM,EAAE,MAAM;gBACd,SAAS,EAAE,KAAK;gBAChB,UAAU,EAAE,GAAG,CAAC,MAAM;gBACtB,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;aAC9D,CAAA;QACH,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,CAAA;IAC3H,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW,EAAE,IAAkB,EAAE,QAAgB,EAAE,KAAoB;QACnG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;QACzF,OAAO;YACL,GAAG;YACH,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE;YAClD,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,YAAY;YACpB,SAAS,EAAE,KAAK;YAChB,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;SAC5D,CAAA;IACH,CAAC;CACF"}
|
package/lib/index.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* web-search-pro — 增强型、可持久化的扩展网页搜索插件 for DeepSeek Harness.
|
|
3
|
+
*
|
|
4
|
+
* - Multi-backend search routing with automatic fallback (agent-reach style):
|
|
5
|
+
* ctx.web seam / Exa / DuckDuckGo / Bing / Jina + platform backends
|
|
6
|
+
* (gh, bili-cli, yt-dlp, sov2ex, opencli, agent-reach).
|
|
7
|
+
* - Persistent SQLite store (MediaCrawler style): search queries + results,
|
|
8
|
+
* page snapshots, and user-extended per-site extraction rules survive
|
|
9
|
+
* restarts and are reused within a configurable TTL.
|
|
10
|
+
* - Userscript-style per-site extraction rules ("脚本猫/油猴" style) applied
|
|
11
|
+
* by the fetch pipeline (Jina Reader → HTTP+extraction → Playwright).
|
|
12
|
+
* - Optional ctx.web provider registration so the built-in web_search /
|
|
13
|
+
* web_fetch tools can route through this plugin.
|
|
14
|
+
*
|
|
15
|
+
* @module web-search-pro
|
|
16
|
+
*/
|
|
17
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
18
|
+
import { Config } from './config.ts';
|
|
19
|
+
export declare const name = "web-search-pro";
|
|
20
|
+
export declare const inject: string[];
|
|
21
|
+
export { Config };
|
|
22
|
+
export type { Config as WebSearchProConfig } from './config.ts';
|
|
23
|
+
export declare function apply(ctx: Context, config: Config): void;
|