@xapp/arachne-web-service 1.32.0 → 1.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ArachneWebService.d.ts +52 -3
- package/lib/ArachneWebService.js +56 -9
- package/lib/ArachneWebService.js.map +1 -1
- package/lib/index.d.ts +2 -0
- package/lib/index.js +2 -0
- package/lib/index.js.map +1 -1
- package/lib/llm/LLMProvider.d.ts +48 -7
- package/lib/llm/LLMProvider.js +66 -12
- package/lib/llm/LLMProvider.js.map +1 -1
- package/lib/services/ServiceExtractor.d.ts +180 -0
- package/lib/services/ServiceExtractor.js +474 -0
- package/lib/services/ServiceExtractor.js.map +1 -0
- package/lib/services/readServicePages.d.ts +37 -0
- package/lib/services/readServicePages.js +58 -0
- package/lib/services/readServicePages.js.map +1 -0
- package/lib/services/selectServicePages.d.ts +34 -0
- package/lib/services/selectServicePages.js +177 -0
- package/lib/services/selectServicePages.js.map +1 -0
- package/lib/taxonomy/ServiceTaxonomy.d.ts +5 -2
- package/lib/taxonomy/ServiceTaxonomy.js +10 -7
- package/lib/taxonomy/ServiceTaxonomy.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ServiceExtractor.js","sourceRoot":"","sources":["../../src/services/ServiceExtractor.ts"],"names":[],"mappings":";;;;;;;;;;;;AAsNA,sCAcC;AApOD,kCAAkC;AAClC,mDAAqC;AAGrC,iEAA8D;AAG9D;;GAEG;AACU,QAAA,iBAAiB,GAAG,cAAc,CAAC;AAEhD;;GAEG;AACU,QAAA,wBAAwB,GAAG,KAAK,CAAC;AAE9C,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC,MAAM,eAAe,GAAG,CAAC,CAAC;AAwI1B,MAAM,eAAe,GAAG;IACpB,aAAa,EAAE,gBAAgB;IAC/B,cAAc,EAAE,gBAAgB;IAChC,kBAAkB,EAAE,oBAAoB;IACxC,IAAI,EAAE,MAAM;IACZ,SAAS,EAAE,WAAW;CAChB,CAAC;AAEX,MAAM,UAAU,GAAG;IACf,QAAQ,EAAE,UAAU;IACpB,sBAAsB,EAAE,sBAAsB;IAC9C,8BAA8B,EAAE,8BAA8B;IAC9D,SAAS,EAAE,WAAW;CAChB,CAAC;AAEX,MAAM,UAAU,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAW,CAAC;AAkC7G;;;GAGG;AACH,SAAgB,aAAa,CAAC,IAAY;IACtC,OAAO,IAAI;SACN,SAAS,CAAC,MAAM,CAAC;QAClB,mGAAmG;SAClG,OAAO,CAAC,oBAAoB,EAAE,GAAG,CAAC;SAClC,OAAO,CAAC,8BAA8B,EAAE,IAAI,CAAC;SAC7C,OAAO,CAAC,0BAA0B,EAAE,IAAI,CAAC;SACzC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE;SACN,WAAW,EAAE,CAAC;AACvB,CAAC;AAED,SAAS,cAAc,CAAC,IAAc;IAClC,OAAO;QACH,IAAI,EAAE,OAAO;QACb,KAAK,EAAE;YACH,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;YAChF,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC;YAC5B,oBAAoB,EAAE,KAAK;SAC9B;KACJ,CAAC;AACN,CAAC;AAED,SAAS,YAAY,CAAC,UAAkC;IACpD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;AAC1G,CAAC;AAED,SAAS,OAAO,CAAC,UAAkC;IAC/C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;AAC9D,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;AAEpE,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAE1F,SAAS,UAAU,CAAC,CAAU;IAC1B,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,SAAS,SAAS,CAAC,CAAU,EAAE,KAA6B;IACxD,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AACpG,CAAC;AAED,SAAS,aAAa,CAAC,CAAM;IACzB,OAAO,CACH,CAAC,CAAC,CAAC;QACH,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC;QACzB,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC;QACtB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC;QACjC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAClC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CACzB,CAAC;AACN,CAAC;AAED,SAAS,kBAAkB,CAAC,CAAM;IAC9B,MAAM,SAAS,GAAG,CAAC,CAAM,EAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACnF,OAAO,CACH,CAAC,CAAC,CAAC;QACH,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAClG,SAAS,CAAC,CAAC,CAAC,iBAAiB,EAAE,SAAS,CAAC;QACzC,SAAS,CAAC,CAAC,CAAC,mBAAmB,EAAE,SAAS,CAAC;QAC3C,SAAS,CACL,CAAC,CAAC,KAAK,EACP,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAC9G;QACD,SAAS,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;QAChE,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3D,SAAS,CAAC,CAAC,CAAC,eAAe,EAAE,SAAS,CAAC;QACvC,SAAS,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,CACrE,CAAC;AACN,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAEjE;;;;;;GAMG;AACH,MAAa,gBAAgB;IAKzB,YAAY,KAA4D;;QACpE,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,MAAA,KAAK,CAAC,QAAQ,mCAAI,IAAI,iCAAe,EAAE,CAAC;IAC5D,CAAC;IAED;;;OAGG;IACG,OAAO;6DACT,KAAwB,EACxB,UAAsC,EAAE;;YAExC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACnB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,EACtC,CAAC,MAAA,OAAO,CAAC,cAAc,mCAAI,gCAAwB,CAAC,GAAG,eAAe,CACzE,CAAC;YACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;gBACrD,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,sFAAsF;YACtF,MAAM,MAAM,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC/E,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACtE,MAAM,QAAQ,GAAoB,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,CAAC,CAAkB,EAAW,EAAE;;gBAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBAClC,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA,MAAA,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,0CAAE,QAAQ,CAAC,OAAO,CAAC,CAAA,CAAC;YACxE,CAAC,CAAC;YACF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,KAAa,EAAE,QAA2B,EAAiC,EAAE;gBACzG,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC;oBACjE,OAAO,SAAS,CAAC;gBACrB,CAAC;gBACD,OAAO,IAAI,CAAC;YAChB,CAAC,CAAC;YAEF,MAAM,MAAM,GAAsB;gBAC9B,eAAe,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBACtC,gBAAgB,EAAE,yBAAiB;gBACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;gBAC5B,eAAe,EAAE,EAAE;gBACnB,QAAQ,EAAE,EAAE;gBACZ,gBAAgB,EAAE,EAAE;gBACpB,kBAAkB,EAAE,EAAE;gBACtB,KAAK,EAAE,EAAE;gBACT,cAAc,EAAE,EAAE;gBAClB,cAAc,EAAE,EAAE;gBAClB,kBAAkB,EAAE,EAAE;gBACtB,IAAI,EAAE,EAAE;gBACR,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,EAAE;gBACZ,oBAAoB,EAAE,EAAE;gBACxB,4BAA4B,EAAE,EAAE;gBAChC,SAAS,EAAE,EAAE;gBACb,cAAc,EAAE,EAAE;gBAClB,WAAW,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE;gBACpE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBACnC,QAAQ;gBACR,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACxC,CAAC;YAEF,IAAI,MAAM,GAAG,CAAC,MAAA,OAAO,CAAC,MAAM,mCAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5E,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,MAAM,CAAC,YAAY,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;gBACxE,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YACtG,CAAC;iBAAM,CAAC;gBACJ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CACzB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAC3B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAC3B,mBAAmB,EACnB,aAAa,EACb,iBAAiB,CACpB,CAAC;gBACF,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,OAAO,SAAS,CAAC;gBACrB,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;oBAC/C,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;oBAC5D,OAAO,MAAM,CAAC;gBAClB,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAChD,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;qBAClD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;qBACrE,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;gBACpC,MAAM,CAAC,YAAY,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;gBACtG,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC/C,KAAK;oBACL,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC;oBACpC,QAAQ,EAAE,EAAE;iBACf,CAAC,CAAC,CAAC;gBACJ,MAAM,GAAG,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,SAAS,CAAC,CAAC;YAClD,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CACzB,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,CACrB,CAAC;YACF,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YAExC,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,aAAa,MAAM,CAAC,QAAQ,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SACjE,MAAM,CAAC,MACX,oBAAoB,QAAQ,CAAC,MAAM,SAAS,CAC/C,CAAC;YAEF,OAAO,MAAM,CAAC;QAClB,CAAC;KAAA;IAEO,SAAS,CACb,MAAwB,EACxB,MAAyB,EACzB,OAAqG;QAErG,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEhD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/E,MAAM,IAAI,GAAG,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,GAAG,GAAG,MAAM;gBACd,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE;gBAC9E,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;oBACzB,CAAC,CAAC,SAAS,CAAC;YAChB,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAC7E,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACpE,CAAC;gBACD,SAAS;YACb,CAAC;YACD,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC3F,SAAS;YACb,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YACrE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,SAAS;YACb,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC;YAClE,IAAI,QAAQ,EAAE,CAAC;gBACX,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC/E,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,QAAQ,CAAC,IAAI,+BAChB,IAAI,EAAE,GAAG,CAAC,IAAI,EACd,KAAK,EAAE,GAAG,CAAC,KAAK,IACb,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAC7C,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EACrC,QAAQ,IACV,CAAC;YACP,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,MAAwB,EAAE,IAAkB,EAAQ,EAAE;YAC/E,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3D,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YACjD,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC;QACpG,MAAM,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,MAAM,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC;QAC1G,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC;QAE9F,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;gBACtF,SAAS;YACb,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC5D,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;YAClG,CAAC;QACL,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,eAAe,CAAC,SAAS,CAAC,IAAoC,CAAC,CAAC;YAC9E,IACI,CAAC,KAAK;gBACN,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,CAAC,IAAoC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,EACnG,CAAC;gBACC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACjB,KAAK,EAAE,KAAK,aAAL,KAAK,cAAL,KAAK,GAAI,YAAY;oBAC5B,KAAK,EAAE,SAAS,CAAC,KAAK;oBACtB,MAAM,EAAE,iBAAiB;iBAC5B,CAAC,CAAC;gBACH,SAAS;YACb,CAAC;YACD,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;QAC1C,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAA+B,CAAC,CAAC;YAC/D,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;gBACvF,SAAS;YACb,CAAC;YACD,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAA+B,CAAC,CAAC;YAC/D,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;gBAC7F,SAAS;YACb,CAAC;YACD,IAAI,CAAC,eAAe,KAAK,EAAE,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;IAED;;OAEG;IACK,GAAG,CAAC,KAAwB,EAAE,WAAmB;QACrD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1E,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC;YACxD,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,oGAAoG;QACpG,IAAI,SAAS,GAAG,WAAW,CAAC;QAC5B,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACnD,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;gBACnB,MAAM;YACV,CAAC;YACD,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,iCAAM,CAAC,KAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,CAAC;IAEO,UAAU,CAAC,KAAwB,EAAE,KAAc;QACvD,OAAO,KAAK;aACP,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;aACnG,IAAI,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IAEO,cAAc,CAAC,KAAwB;QAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,OAAO;;UAEL,MAAM;;;;qEAIqD,oBAAoB;;;EAGvF,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,mBAAmB,CAAC,EAAE,CAAC;IAC5C,CAAC;IAEO,cAAc,CAAC,KAAwB;QAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAClD,OAAO,YAAY,CAAC;YAChB,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,MAAM,EAAE,MAAM,CAAC,EAAE;YAC5D,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,gBAAgB,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;YAC5E,QAAQ,EAAE,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;SACpD,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACK,OAAO,CAAC,MAAgB;;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QACnD,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACzF,KAAK,MAAM,MAAM,IAAI,MAAA,KAAK,CAAC,QAAQ,mCAAI,EAAE,EAAE,CAAC;gBACxC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAA,MAAM,CAAC,aAAa,mCAAI,EAAE,EAAE,CAAC,CAAC;gBACtF,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,WAAC,OAAA,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,MAAA,CAAC,CAAC,QAAQ,mCAAI,EAAE,EAAE,CAAC,CAAA,EAAA,CAAC,CAAC;YACzH,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,WAAC,OAAA,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,MAAA,CAAC,CAAC,QAAQ,mCAAI,EAAE,EAAE,CAAC,CAAA,EAAA,CAAC,CAAC;QACpI,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,aAAa,CAAC,KAAwB,EAAE,OAAuB;QACnE,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,OAAO;aACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;aAC3G,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,gBAAgB;aAClD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;aAC1C,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,gBAAgB;aACrD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;aAC1C,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC;aAC9C,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aAC1D,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhB,OAAO;;;;;;;;;;;;;EAab,WAAW;;;EAGX,KAAK;;;EAGL,UAAU;;;EAGV,QAAQ;;EAER,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;IACvB,CAAC;IAEO,aAAa,CAAC,KAAwB,EAAE,OAAuB;QACnE,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,CAAC,QAAgC,EAAE,EAAU,EAAE,CAC1D,OAAO,iCAAM,KAAK,KAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,IAAG,CAAC;QAC/D,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAE7E,OAAO,YAAY,CAAC;YAChB,QAAQ,EAAE,OAAO,CAAC;gBACd,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE;gBAC1D,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9B,QAAQ;aACX,CAAC;YACF,iBAAiB,EAAE,MAAM,EAAE;YAC3B,mBAAmB,EAAE,MAAM,EAAE;YAC7B,KAAK,EAAE,OAAO,CAAC;gBACX,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE;gBAC3C,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC1B,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9B,QAAQ;aACX,CAAC;YACF,UAAU,EAAE,OAAO,CAAC;gBAChB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE;gBAC5D,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE;gBAChD,QAAQ;aACX,CAAC;YACF,KAAK,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YAC1E,eAAe,EAAE,MAAM,EAAE;YACzB,YAAY,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;SACpF,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACW,GAAG,CACb,MAAc,EACd,UAAkB,EAClB,SAAiB,EACjB,KAA6B,EAC7B,KAAa;;YAEb,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACtC,MAAM;oBACN,UAAU;oBACV,SAAS;oBACT,WAAW,EAAE,CAAC;oBACd,KAAK,EAAE,GAAG,KAAK,IAAI,OAAO,EAAE;iBAC/B,CAAC,CAAC;gBACH,IAAI,MAAe,CAAC;gBACpB,IAAI,CAAC;oBACD,MAAM,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC/D,CAAC;gBAAC,WAAM,CAAC;oBACL,MAAM,GAAG,SAAS,CAAC;gBACvB,CAAC;gBACD,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChB,OAAO,MAAM,CAAC;gBAClB,CAAC;gBACD,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,GAAG,KAAK,YAAY,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,2BAA2B,eAAe,OAAO,EAAE,CAC3G,CAAC;YACN,CAAC;YACD,OAAO,SAAS,CAAC;QACrB,CAAC;KAAA;CACJ;AAjZD,4CAiZC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,QAAyB;IAK3C,MAAM,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC;IAC9B,OAAO;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACnC,KAAK,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5C,UAAU,EAAE;YACR,aAAa,EAAE,CAAC,CAAC,cAAc;YAC/B,cAAc,EAAE,CAAC,CAAC,eAAe;YACjC,kBAAkB,EAAE,CAAC,CAAC,mBAAmB;YACzC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,SAAS,EAAE,CAAC,CAAC,UAAU;SAC1B;KACJ,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { SubpageTab } from "../utils/subpageDetection";
|
|
2
|
+
import { ServicePage } from "./selectServicePages";
|
|
3
|
+
import { ServicePageText } from "./ServiceExtractor";
|
|
4
|
+
export interface ReadServicePagesProps {
|
|
5
|
+
/**
|
|
6
|
+
* Opens a new tab with the crawler's page setup.
|
|
7
|
+
*/
|
|
8
|
+
openTab: () => Promise<SubpageTab>;
|
|
9
|
+
homepageUrl: string;
|
|
10
|
+
/**
|
|
11
|
+
* Pages to read, in order. The homepage is not visited again.
|
|
12
|
+
*/
|
|
13
|
+
targets: ServicePage[];
|
|
14
|
+
/**
|
|
15
|
+
* Converts a page's HTML to the markdown sent to the LLM.
|
|
16
|
+
*/
|
|
17
|
+
toMarkdown: (html: string) => string;
|
|
18
|
+
/**
|
|
19
|
+
* Budget for each page.
|
|
20
|
+
*/
|
|
21
|
+
pageTimeoutMs: number;
|
|
22
|
+
/**
|
|
23
|
+
* No page is started after this time (epoch ms).
|
|
24
|
+
*/
|
|
25
|
+
deadline: number;
|
|
26
|
+
/**
|
|
27
|
+
* HTML of pages already loaded during the crawl, by URL. These are not visited again.
|
|
28
|
+
*/
|
|
29
|
+
loaded?: Map<string, string>;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Loads each target on its own tab, in order, and returns its text.
|
|
33
|
+
*
|
|
34
|
+
* A page that errors, times out or redirects off-site is skipped. Pages are not started once the deadline is
|
|
35
|
+
* near.
|
|
36
|
+
*/
|
|
37
|
+
export declare function readServicePages(props: ReadServicePagesProps): Promise<ServicePageText[]>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.readServicePages = readServicePages;
|
|
13
|
+
/*! Copyright (c) 2026, XAPP AI */
|
|
14
|
+
const arachne_utils_1 = require("@xapp/arachne-utils");
|
|
15
|
+
const stentor_logger_1 = require("stentor-logger");
|
|
16
|
+
const subpageDetection_1 = require("../utils/subpageDetection");
|
|
17
|
+
/**
|
|
18
|
+
* Loads each target on its own tab, in order, and returns its text.
|
|
19
|
+
*
|
|
20
|
+
* A page that errors, times out or redirects off-site is skipped. Pages are not started once the deadline is
|
|
21
|
+
* near.
|
|
22
|
+
*/
|
|
23
|
+
function readServicePages(props) {
|
|
24
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
25
|
+
const { openTab, homepageUrl, targets, toMarkdown, pageTimeoutMs, deadline, loaded } = props;
|
|
26
|
+
const pages = [];
|
|
27
|
+
for (const [index, target] of targets.entries()) {
|
|
28
|
+
if (target.kind === "homepage") {
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
const cached = loaded === null || loaded === void 0 ? void 0 : loaded.get(target.url);
|
|
32
|
+
if (cached !== undefined) {
|
|
33
|
+
pages.push({ url: target.url, kind: target.kind, markdown: toMarkdown(cached) });
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
const remaining = deadline - Date.now();
|
|
37
|
+
if (remaining < subpageDetection_1.MIN_SUBPAGE_BUDGET_MS) {
|
|
38
|
+
(0, stentor_logger_1.log)().info(`Service pages: budget exhausted, not reading ${targets.length - index} more page(s).`);
|
|
39
|
+
break;
|
|
40
|
+
}
|
|
41
|
+
const budget = Math.min(pageTimeoutMs, remaining);
|
|
42
|
+
// A fresh tab per page, so a navigation that outlives its budget is cancelled by closing the tab
|
|
43
|
+
const tab = (0, subpageDetection_1.createLazyTab)(openTab);
|
|
44
|
+
try {
|
|
45
|
+
const html = yield (0, arachne_utils_1.promiseWithTimeout)(budget, () => __awaiter(this, void 0, void 0, function* () { return (0, subpageDetection_1.loadSameSitePage)(yield tab.get(), target.url, homepageUrl, budget); }), `Service page ${target.url} exceeded its budget of ${budget}ms`);
|
|
46
|
+
pages.push({ url: target.url, kind: target.kind, markdown: toMarkdown(html) });
|
|
47
|
+
}
|
|
48
|
+
catch (e) {
|
|
49
|
+
(0, stentor_logger_1.log)().warn(`Service pages: skipping ${target.url}: ${e instanceof Error ? e.message : e}`);
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
yield tab.close();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return pages;
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=readServicePages.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"readServicePages.js","sourceRoot":"","sources":["../../src/services/readServicePages.ts"],"names":[],"mappings":";;;;;;;;;;;AA0CA,4CAuCC;AAjFD,kCAAkC;AAClC,uDAAyD;AACzD,mDAAqC;AAErC,gEAA+G;AAgC/G;;;;;GAKG;AACH,SAAsB,gBAAgB,CAAC,KAA4B;;QAC/D,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;QAC7F,MAAM,KAAK,GAAsB,EAAE,CAAC;QAEpC,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBAC7B,SAAS;YACb,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACjF,SAAS;YACb,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxC,IAAI,SAAS,GAAG,wCAAqB,EAAE,CAAC;gBACpC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gDAAgD,OAAO,CAAC,MAAM,GAAG,KAAK,gBAAgB,CAAC,CAAC;gBACnG,MAAM;YACV,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;YAClD,iGAAiG;YACjG,MAAM,GAAG,GAAG,IAAA,gCAAa,EAAC,OAAO,CAAC,CAAC;YACnC,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,IAAA,kCAAkB,EACjC,MAAM,EACN,GAAS,EAAE,gDAAC,OAAA,IAAA,mCAAgB,EAAC,MAAM,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA,GAAA,EAC9E,gBAAgB,MAAM,CAAC,GAAG,2BAA2B,MAAM,IAAI,CAClE,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnF,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,2BAA2B,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC/F,CAAC;oBAAS,CAAC;gBACP,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;CAAA"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Why a page was picked, in priority order: the homepage, pages that describe services, about/FAQ/financing/careers
|
|
3
|
+
* pages (which name licences and brands), service-area pages and brand pages.
|
|
4
|
+
*/
|
|
5
|
+
export type ServicePageKind = "homepage" | "service" | "about" | "area" | "brand";
|
|
6
|
+
export interface ServicePage {
|
|
7
|
+
url: string;
|
|
8
|
+
kind: ServicePageKind;
|
|
9
|
+
}
|
|
10
|
+
export interface SelectServicePagesOptions {
|
|
11
|
+
/**
|
|
12
|
+
* Most pages to return, homepage included. Default `DEFAULT_MAX_SERVICE_PAGES`.
|
|
13
|
+
*/
|
|
14
|
+
maxPages?: number;
|
|
15
|
+
/**
|
|
16
|
+
* Pages already discovered elsewhere, e.g. from the sitemap. Added when the homepage does not link them.
|
|
17
|
+
*/
|
|
18
|
+
known?: {
|
|
19
|
+
aboutUs?: string;
|
|
20
|
+
faqPage?: string;
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export declare const DEFAULT_MAX_SERVICE_PAGES = 12;
|
|
24
|
+
/**
|
|
25
|
+
* Picks the same-site pages worth reading for the services a business offers.
|
|
26
|
+
*
|
|
27
|
+
* The homepage comes first and counts against `maxPages`. When there are more candidates than room, one about,
|
|
28
|
+
* one service-area and one brand page are kept (while services keep at least half the room) and the rest goes to
|
|
29
|
+
* service pages, breadth first.
|
|
30
|
+
*
|
|
31
|
+
* @param html The homepage HTML
|
|
32
|
+
* @param homepageUrl The homepage URL, also the base for relative links
|
|
33
|
+
*/
|
|
34
|
+
export declare function selectServicePages(html: string, homepageUrl: string, options?: SelectServicePagesOptions): ServicePage[];
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.DEFAULT_MAX_SERVICE_PAGES = void 0;
|
|
37
|
+
exports.selectServicePages = selectServicePages;
|
|
38
|
+
/*! Copyright (c) 2026, XAPP AI */
|
|
39
|
+
const cheerio = __importStar(require("cheerio"));
|
|
40
|
+
const subpageDetection_1 = require("../utils/subpageDetection");
|
|
41
|
+
exports.DEFAULT_MAX_SERVICE_PAGES = 12;
|
|
42
|
+
const KIND_ORDER = ["homepage", "service", "about", "area", "brand"];
|
|
43
|
+
const FILE = /\.(pdf|jpe?g|png|gif|svg|webp|docx?|xlsx?|zip|mp3|mp4|mov)$/i;
|
|
44
|
+
const SKIP = /\/(\d{4})\/|(^|[/_-])(blog|news|articles?|posts?|tags?|category|author|feed|wp-|privacy|terms|legal|accessibility|sitemap|login|account|cart|checkout|contact|reviews?|testimonials|gallery|coupons?|specials|promotions?)([/_-]|$)/i;
|
|
45
|
+
const AREA = /service[\s_-]?areas?|areas?[\s_-]we[\s_-]serve|locations?|cities|communities|counties|where[\s_-]we[\s_-]work|serving[\s_-]/i;
|
|
46
|
+
const BRAND = /brands?|manufacturers?|products|equipment|dealer/i;
|
|
47
|
+
const ABOUT = /about|our[\s_-]story|who[\s_-]we[\s_-]are|faqs?\b|frequently|why[\s_-]|financing|warrant|guarantee|careers|join[\s_-]our/i;
|
|
48
|
+
const SERVICE = /services?|what[\s_-]we[\s_-]do|repair|install|replac|maintenance|clean|remodel|inspection|hvac|heating|cooling|air[\s_-]condition|\bac\b|furnace|heat[\s_-]pump|boiler|duct|plumb|drain|sewer|water[\s_-]heater|tankless|leak|pipe|toilet|faucet|electric|panel|wiring|generator|ev[\s_-]charg|lighting|roof|gutter|siding|window|door|insulation|solar|pest|termite|lawn|landscap|tree|irrigation|sprinkler|pool|garage|fence|deck|concrete|paving|masonry|chimney|paint|floor|carpet|tile|kitchen|bath|handyman|appliance|mold|water[\s_-]damage|restoration|septic|well[\s_-]pump|pressure[\s_-]wash|junk|moving|locksmith|foundation|waterproof|drywall|cabinet|countertop/i;
|
|
49
|
+
function siteKey(url) {
|
|
50
|
+
return `${url.hostname.toLowerCase().replace(/^www\./, "")}${url.pathname.replace(/\/+$/, "")}${url.search}`;
|
|
51
|
+
}
|
|
52
|
+
function classify(pathname, text) {
|
|
53
|
+
if (FILE.test(pathname) || SKIP.test(pathname)) {
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
let path = pathname;
|
|
57
|
+
try {
|
|
58
|
+
path = decodeURIComponent(pathname);
|
|
59
|
+
}
|
|
60
|
+
catch (_a) {
|
|
61
|
+
// keep it encoded
|
|
62
|
+
}
|
|
63
|
+
const subject = `${path} ${text}`;
|
|
64
|
+
if (AREA.test(subject)) {
|
|
65
|
+
return "area";
|
|
66
|
+
}
|
|
67
|
+
if (BRAND.test(subject)) {
|
|
68
|
+
return "brand";
|
|
69
|
+
}
|
|
70
|
+
if (ABOUT.test(subject)) {
|
|
71
|
+
return "about";
|
|
72
|
+
}
|
|
73
|
+
if (SERVICE.test(subject)) {
|
|
74
|
+
return "service";
|
|
75
|
+
}
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Breadth over depth: shallower pages first, and within a depth one page per parent section before any section's
|
|
80
|
+
* second page.
|
|
81
|
+
*/
|
|
82
|
+
function breadthFirst(urls) {
|
|
83
|
+
var _a, _b;
|
|
84
|
+
const byDepth = new Map();
|
|
85
|
+
for (const url of urls) {
|
|
86
|
+
const segments = new URL(url).pathname.split("/").filter(Boolean);
|
|
87
|
+
const parent = segments.slice(0, -1).join("/");
|
|
88
|
+
const sections = (_a = byDepth.get(segments.length)) !== null && _a !== void 0 ? _a : new Map();
|
|
89
|
+
sections.set(parent, [...((_b = sections.get(parent)) !== null && _b !== void 0 ? _b : []), url]);
|
|
90
|
+
byDepth.set(segments.length, sections);
|
|
91
|
+
}
|
|
92
|
+
const ordered = [];
|
|
93
|
+
for (const d of [...byDepth.keys()].sort((a, b) => a - b)) {
|
|
94
|
+
const queues = [...byDepth.get(d).values()];
|
|
95
|
+
while (queues.some((q) => q.length > 0)) {
|
|
96
|
+
for (const queue of queues) {
|
|
97
|
+
const next = queue.shift();
|
|
98
|
+
if (next) {
|
|
99
|
+
ordered.push(next);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return ordered;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Picks the same-site pages worth reading for the services a business offers.
|
|
108
|
+
*
|
|
109
|
+
* The homepage comes first and counts against `maxPages`. When there are more candidates than room, one about,
|
|
110
|
+
* one service-area and one brand page are kept (while services keep at least half the room) and the rest goes to
|
|
111
|
+
* service pages, breadth first.
|
|
112
|
+
*
|
|
113
|
+
* @param html The homepage HTML
|
|
114
|
+
* @param homepageUrl The homepage URL, also the base for relative links
|
|
115
|
+
*/
|
|
116
|
+
function selectServicePages(html, homepageUrl, options = {}) {
|
|
117
|
+
var _a, _b, _c;
|
|
118
|
+
const maxPages = Math.max(1, (_a = options.maxPages) !== null && _a !== void 0 ? _a : exports.DEFAULT_MAX_SERVICE_PAGES);
|
|
119
|
+
const home = new URL(homepageUrl);
|
|
120
|
+
const seen = new Set([siteKey(home)]);
|
|
121
|
+
const candidates = {
|
|
122
|
+
service: [],
|
|
123
|
+
about: [],
|
|
124
|
+
area: [],
|
|
125
|
+
brand: [],
|
|
126
|
+
};
|
|
127
|
+
const consider = (href, text, forced) => {
|
|
128
|
+
if (!href || /^(mailto|tel|javascript):/i.test(href)) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
let url;
|
|
132
|
+
try {
|
|
133
|
+
url = new URL(href, home);
|
|
134
|
+
}
|
|
135
|
+
catch (_a) {
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
url.hash = "";
|
|
139
|
+
if (!(0, subpageDetection_1.isSameSite)(url.toString(), homepageUrl)) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
const key = siteKey(url);
|
|
143
|
+
if (seen.has(key)) {
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
const kind = forced !== null && forced !== void 0 ? forced : classify(url.pathname, text);
|
|
147
|
+
if (!kind || kind === "homepage") {
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
seen.add(key);
|
|
151
|
+
candidates[kind].push(url.toString());
|
|
152
|
+
};
|
|
153
|
+
const $ = cheerio.load(html);
|
|
154
|
+
$("a[href]").each((_, el) => {
|
|
155
|
+
consider($(el).attr("href"), $(el).text().replace(/\s+/g, " ").trim());
|
|
156
|
+
});
|
|
157
|
+
consider((_b = options.known) === null || _b === void 0 ? void 0 : _b.aboutUs, "", "about");
|
|
158
|
+
consider((_c = options.known) === null || _c === void 0 ? void 0 : _c.faqPage, "", "about");
|
|
159
|
+
candidates.service = breadthFirst(candidates.service);
|
|
160
|
+
const budget = maxPages - 1;
|
|
161
|
+
const picked = [];
|
|
162
|
+
const take = (kind, count) => {
|
|
163
|
+
for (const url of candidates[kind].splice(0, Math.max(0, Math.min(count, budget - picked.length)))) {
|
|
164
|
+
picked.push({ url, kind });
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
const reserved = ["about", "area", "brand"]
|
|
168
|
+
.filter((kind) => candidates[kind].length > 0)
|
|
169
|
+
.slice(0, Math.floor(budget / 2));
|
|
170
|
+
take("service", budget - reserved.length);
|
|
171
|
+
reserved.forEach((kind) => take(kind, 1));
|
|
172
|
+
take("service", budget);
|
|
173
|
+
["about", "area", "brand"].forEach((kind) => take(kind, budget));
|
|
174
|
+
picked.sort((a, b) => KIND_ORDER.indexOf(a.kind) - KIND_ORDER.indexOf(b.kind));
|
|
175
|
+
return [{ url: homepageUrl, kind: "homepage" }, ...picked];
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=selectServicePages.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"selectServicePages.js","sourceRoot":"","sources":["../../src/services/selectServicePages.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoHA,gDAqEC;AAzLD,kCAAkC;AAClC,iDAAmC;AAEnC,gEAAuD;AAwB1C,QAAA,yBAAyB,GAAG,EAAE,CAAC;AAE5C,MAAM,UAAU,GAAsB,CAAC,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAExF,MAAM,IAAI,GAAG,8DAA8D,CAAC;AAE5E,MAAM,IAAI,GACN,sOAAsO,CAAC;AAE3O,MAAM,IAAI,GACN,8HAA8H,CAAC;AAEnI,MAAM,KAAK,GAAG,mDAAmD,CAAC;AAElE,MAAM,KAAK,GACP,2HAA2H,CAAC;AAEhI,MAAM,OAAO,GACT,ipBAAipB,CAAC;AAEtpB,SAAS,OAAO,CAAC,GAAQ;IACrB,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;AACjH,CAAC;AAED,SAAS,QAAQ,CAAC,QAAgB,EAAE,IAAY;IAC5C,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7C,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,IAAI,IAAI,GAAG,QAAQ,CAAC;IACpB,IAAI,CAAC;QACD,IAAI,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAAC,WAAM,CAAC;QACL,kBAAkB;IACtB,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IAClC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,IAAc;;IAChC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;IACzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,mCAAI,IAAI,GAAG,EAAoB,CAAC;QAC7E,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,MAAA,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,mCAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACtC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC3B,IAAI,IAAI,EAAE,CAAC;oBACP,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;;GASG;AACH,SAAgB,kBAAkB,CAC9B,IAAY,EACZ,WAAmB,EACnB,UAAqC,EAAE;;IAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAA,OAAO,CAAC,QAAQ,mCAAI,iCAAyB,CAAC,CAAC;IAC5E,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9C,MAAM,UAAU,GAA2D;QACvE,OAAO,EAAE,EAAE;QACX,KAAK,EAAE,EAAE;QACT,IAAI,EAAE,EAAE;QACR,KAAK,EAAE,EAAE;KACZ,CAAC;IAEF,MAAM,QAAQ,GAAG,CAAC,IAAwB,EAAE,IAAY,EAAE,MAAwB,EAAQ,EAAE;QACxF,IAAI,CAAC,IAAI,IAAI,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,OAAO;QACX,CAAC;QACD,IAAI,GAAQ,CAAC;QACb,IAAI,CAAC;YACD,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9B,CAAC;QAAC,WAAM,CAAC;YACL,OAAO;QACX,CAAC;QACD,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,IAAA,6BAAU,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,CAAC,EAAE,CAAC;YAC3C,OAAO;QACX,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,OAAO;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACxB,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,MAAA,OAAO,CAAC,KAAK,0CAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IAC9C,QAAQ,CAAC,MAAA,OAAO,CAAC,KAAK,0CAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IAE9C,UAAU,CAAC,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAEtD,MAAM,MAAM,GAAG,QAAQ,GAAG,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,CAAC,IAA0C,EAAE,KAAa,EAAQ,EAAE;QAC7E,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACjG,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAW;SACjD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SAC7C,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACtC,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1C,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAE5E,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAE/E,OAAO,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,GAAG,MAAM,CAAC,CAAC;AAC/D,CAAC"}
|
|
@@ -103,13 +103,16 @@ export declare const SERVICE_TAXONOMY_V1: Taxonomy;
|
|
|
103
103
|
* Lookups over a taxonomy: canonical slugs, aliases and the family a job is scored in.
|
|
104
104
|
*/
|
|
105
105
|
export declare class ServiceTaxonomy {
|
|
106
|
-
|
|
106
|
+
readonly definition: Taxonomy;
|
|
107
107
|
readonly version: string;
|
|
108
108
|
readonly jobs: TaxonomyJobEntry[];
|
|
109
109
|
private readonly bySlug;
|
|
110
110
|
private readonly families;
|
|
111
111
|
private readonly trades;
|
|
112
|
-
|
|
112
|
+
/**
|
|
113
|
+
* @param definition The taxonomy JSON, Service Taxonomy v1 by default
|
|
114
|
+
*/
|
|
115
|
+
constructor(definition?: Taxonomy);
|
|
113
116
|
job(slug: string): TaxonomyJobEntry | undefined;
|
|
114
117
|
isFamily(slug: string): boolean;
|
|
115
118
|
isTrade(slug: string): boolean;
|
|
@@ -14,14 +14,17 @@ exports.SERVICE_TAXONOMY_V1 = v1_json_1.default;
|
|
|
14
14
|
* Lookups over a taxonomy: canonical slugs, aliases and the family a job is scored in.
|
|
15
15
|
*/
|
|
16
16
|
class ServiceTaxonomy {
|
|
17
|
-
|
|
17
|
+
/**
|
|
18
|
+
* @param definition The taxonomy JSON, Service Taxonomy v1 by default
|
|
19
|
+
*/
|
|
20
|
+
constructor(definition = exports.SERVICE_TAXONOMY_V1) {
|
|
18
21
|
var _a, _b;
|
|
19
|
-
this.
|
|
20
|
-
this.version =
|
|
22
|
+
this.definition = definition;
|
|
23
|
+
this.version = definition.version;
|
|
21
24
|
this.jobs = [];
|
|
22
25
|
this.families = new Set();
|
|
23
26
|
this.trades = new Set();
|
|
24
|
-
for (const trade of
|
|
27
|
+
for (const trade of definition.trades) {
|
|
25
28
|
this.trades.add(trade.slug);
|
|
26
29
|
for (const job of (_a = trade.jobs) !== null && _a !== void 0 ? _a : []) {
|
|
27
30
|
this.jobs.push(Object.assign(Object.assign({}, job), { trade: trade.slug }));
|
|
@@ -63,11 +66,11 @@ class ServiceTaxonomy {
|
|
|
63
66
|
if (!candidate) {
|
|
64
67
|
return undefined;
|
|
65
68
|
}
|
|
66
|
-
candidate = (_a = this.
|
|
69
|
+
candidate = (_a = this.definition.aliases[candidate]) !== null && _a !== void 0 ? _a : candidate;
|
|
67
70
|
if (!this.bySlug.has(candidate)) {
|
|
68
|
-
const prefix = Object.keys(this.
|
|
71
|
+
const prefix = Object.keys(this.definition.alias_prefixes).find((p) => candidate.startsWith(p));
|
|
69
72
|
if (prefix) {
|
|
70
|
-
candidate = this.
|
|
73
|
+
candidate = this.definition.alias_prefixes[prefix] + candidate.slice(prefix.length);
|
|
71
74
|
}
|
|
72
75
|
}
|
|
73
76
|
// Follow deprecations, guarding against a cycle
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ServiceTaxonomy.js","sourceRoot":"","sources":["../../src/taxonomy/ServiceTaxonomy.ts"],"names":[],"mappings":";;;;;;AAAA,kCAAkC;AAClC,wDAA2B;AAyG3B;;GAEG;AACU,QAAA,mBAAmB,GAAa,iBAAE,CAAC;AAEhD;;GAEG;AACH,MAAa,eAAe;IAWxB,
|
|
1
|
+
{"version":3,"file":"ServiceTaxonomy.js","sourceRoot":"","sources":["../../src/taxonomy/ServiceTaxonomy.ts"],"names":[],"mappings":";;;;;;AAAA,kCAAkC;AAClC,wDAA2B;AAyG3B;;GAEG;AACU,QAAA,mBAAmB,GAAa,iBAAE,CAAC;AAEhD;;GAEG;AACH,MAAa,eAAe;IAWxB;;OAEG;IACH,YAAqB,aAAuB,2BAAmB;;QAA1C,eAAU,GAAV,UAAU,CAAgC;QAC3D,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACf,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QAExB,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5B,KAAK,MAAM,GAAG,IAAI,MAAA,KAAK,CAAC,IAAI,mCAAI,EAAE,EAAE,CAAC;gBACjC,IAAI,CAAC,IAAI,CAAC,IAAI,iCAAM,GAAG,KAAE,KAAK,EAAE,KAAK,CAAC,IAAI,IAAG,CAAC;YAClD,CAAC;YACD,KAAK,MAAM,MAAM,IAAI,MAAA,KAAK,CAAC,QAAQ,mCAAI,EAAE,EAAE,CAAC;gBACxC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC/B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,iCAAM,GAAG,KAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,IAAG,CAAC;gBACvE,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,GAAG,CAAC,IAAY;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,QAAQ,CAAC,IAAY;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACH,eAAe,CAAC,MAAiB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CACnB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAChG,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,OAAO,CAAC,IAAY;;QAChB,IAAI,SAAS,GAAG,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,GAAG,WAAW,EAAE,CAAC;QAC3C,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,SAAS,GAAG,MAAA,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,mCAAI,SAAS,CAAC;QAE5D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAChG,IAAI,MAAM,EAAE,CAAC;gBACT,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxF,CAAC;QACL,CAAC;QAED,gDAAgD;QAChD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrC,OAAO,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,aAAa,KAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnB,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,CAAC;IACrB,CAAC;IAED;;;;OAIG;IACH,QAAQ,CAAC,IAAY;;QACjB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAE/D,OAAO,GAAG,CAAC,CAAC,CAAC,MAAA,GAAG,CAAC,MAAM,mCAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACrD,CAAC;CACJ;AArGD,0CAqGC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xapp/arachne-web-service",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.34.0",
|
|
4
4
|
"types": "lib/index",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"files": [
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
"dependencies": {
|
|
36
36
|
"@xapp/arachne": "1.18.0",
|
|
37
37
|
"@xapp/arachne-parsers": "1.25.0",
|
|
38
|
+
"google-auth-library": "11.3.2",
|
|
38
39
|
"gpt-3-encoder": "1.1.4",
|
|
39
40
|
"ico-to-png": "0.2.2",
|
|
40
41
|
"node-vibrant": "3.2.1-alpha.1",
|